@bli-cockpit/cli 0.2.49 → 0.2.51

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. package/dist/adapters/raw-evidence-claude-reader.js +108 -0
  2. package/dist/adapters/raw-evidence-codex-reader.js +147 -0
  3. package/dist/adapters/raw-evidence-collection-state.js +199 -0
  4. package/dist/adapters/raw-evidence-facts.js +338 -0
  5. package/dist/adapters/raw-evidence-git-diff-reader.js +187 -0
  6. package/dist/adapters/raw-evidence-image-reader.js +107 -0
  7. package/dist/adapters/raw-evidence-sanitize.js +56 -0
  8. package/dist/adapters/raw-evidence-transcript-file.js +182 -0
  9. package/dist/adapters/raw-evidence.js +63 -1183
  10. package/dist/commands/backfill-batches.js +34 -0
  11. package/dist/commands/backfill-candidates.js +54 -0
  12. package/dist/commands/backfill-checkpoint.js +101 -0
  13. package/dist/commands/backfill-command-line.js +70 -0
  14. package/dist/commands/backfill-evidence-outcomes.js +104 -0
  15. package/dist/commands/backfill-issues.js +265 -0
  16. package/dist/commands/backfill-output.js +75 -0
  17. package/dist/commands/backfill-plan.js +71 -0
  18. package/dist/commands/backfill-reasons.js +107 -0
  19. package/dist/commands/backfill-report.js +298 -0
  20. package/dist/commands/backfill-result.js +150 -0
  21. package/dist/commands/backfill-scan.js +274 -0
  22. package/dist/commands/backfill-scope.js +114 -0
  23. package/dist/commands/backfill-session-report.js +145 -0
  24. package/dist/commands/backfill-types.js +1 -0
  25. package/dist/commands/backfill-upload.js +212 -0
  26. package/dist/commands/backfill.js +41 -1961
  27. package/dist/commands/doctor.js +57 -0
  28. package/dist/commands/jarvis-trace.js +184 -0
  29. package/dist/commands/jarvis.js +144 -4
  30. package/dist/commands/local-args-collector.js +26 -0
  31. package/dist/commands/local-args-tower.js +21 -0
  32. package/dist/commands/local-args.js +3 -1
  33. package/dist/commands/local-help.js +19 -2
  34. package/dist/commands/local.js +3 -0
  35. package/dist/commands/memory-install-claude.js +294 -0
  36. package/dist/commands/memory-install-codex.js +205 -0
  37. package/dist/commands/memory-install-contract.js +286 -0
  38. package/dist/commands/memory-install-files.js +63 -0
  39. package/dist/commands/memory-install-skills.js +121 -0
  40. package/dist/commands/memory-install-toml.js +265 -0
  41. package/dist/commands/memory-install.js +465 -0
  42. package/dist/commands/public-root.js +1 -1
  43. package/dist/commands/sync-followups.js +105 -0
  44. package/dist/commands/sync.js +7 -1
  45. package/dist/local-state-attributed-target.js +75 -0
  46. package/dist/local-state-config.js +147 -0
  47. package/dist/local-state-files.js +59 -0
  48. package/dist/local-state-identity.js +73 -0
  49. package/dist/local-state-pairing.js +263 -0
  50. package/dist/local-state-paths.js +61 -0
  51. package/dist/local-state-session.js +68 -0
  52. package/dist/local-state-status.js +163 -0
  53. package/dist/local-state-work-context.js +190 -0
  54. package/dist/local-state.js +34 -848
  55. package/dist/tower-client.js +3 -2
  56. package/dist/tower-stream.js +57 -3
  57. package/package.json +2 -1
@@ -3,48 +3,52 @@
3
3
  * every-15-minutes sync was never running to see.
4
4
  *
5
5
  * Read `runBackfill` below as the table of contents. It is four stages, in
6
- * order, and every function in this file belongs to exactly one of them:
6
+ * order, and every part of this command lives in exactly one of them:
7
7
  *
8
- * SCAN resolveBackfillScope paired? which roots, stores and window
9
- * scanBackfillSessions read both stores into candidates
10
- * auditScannedCandidates census what could not be accounted for
11
- * PLAN planBackfillRun confirm `--all`, or answer a `--dry-run`
12
- * UPLOAD uploadBackfillBatches sync in batches under the shared lock
13
- * REPORT reportBackfillOutcome post, checkpoint, decide, assemble
8
+ * SCAN backfill-scan.ts which roots, stores and window; read both
9
+ * stores into candidates; census the gaps
10
+ * PLAN backfill-plan.ts confirm `--all`, or answer a `--dry-run`
11
+ * UPLOAD backfill-upload.ts sync in batches under the shared lock
12
+ * REPORT backfill-report.ts post, checkpoint, decide, assemble
14
13
  *
15
- * Two rules run through all four and explain most of the apparent complexity.
16
- * First, a session may never be silently dropped: everything the scan could
17
- * not account for lands on the issue ledger and everything a person reads
18
- * names its own reason. Second, the cursor is the only irreversible thing here
19
- * it advances solely through a contiguous prefix of sessions that both
20
- * earned a durable pointer and were acknowledged by the server, so a
14
+ * The supporting modules each own one thing the stages share:
15
+ *
16
+ * backfill-types.ts the shapes every stage passes along
17
+ * backfill-command-line.ts the flag surface and the retry command
18
+ * backfill-scope.ts paired? which roots, sources and window
19
+ * backfill-candidates.ts one session's identity and ordering
20
+ * backfill-batches.ts what is uploadable, cut into batches
21
+ * backfill-issues.ts the ledger of what could not be accounted for
22
+ * backfill-reasons.ts the skip-reason table an operator reads
23
+ * backfill-evidence-outcomes.ts what one upload achieved per session
24
+ * backfill-session-report.ts one attribution row per session seen
25
+ * backfill-checkpoint.ts the durable cursor and pointer writes
26
+ * backfill-result.ts the `--json` payload
27
+ * backfill-output.ts every line and prompt a person sees
28
+ *
29
+ * Two rules run through all four stages and explain most of the apparent
30
+ * complexity. First, a session may never be silently dropped: everything the
31
+ * scan could not account for lands on the issue ledger and everything a person
32
+ * reads names its own reason. Second, the cursor is the only irreversible
33
+ * thing here — it advances solely through a contiguous prefix of sessions that
34
+ * both earned a durable pointer and were acknowledged by the server, so a
21
35
  * misjudged "resolved" loses that history for good.
22
36
  */
23
- import { NO_UPLOAD_ATTEMPT_RECORDED, RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES, notUploadableAttributionStateReason } from "@bli-cockpit/telemetry-core";
24
- import crypto from "node:crypto";
25
- import fs from "node:fs/promises";
26
- import os from "node:os";
27
- import path from "node:path";
28
- import { describeError } from "../health-detail.js";
29
- import { scanAndAttributeClaudeSessions } from "../adapters/claude-attribution.js";
30
- import { defaultCodexSessionDirs, scanAndAttributeCodexSessions } from "../adapters/codex-attribution.js";
31
- import { RAW_EVIDENCE_DEFAULT_BYTE_BUDGET, RAW_EVIDENCE_DEFAULT_OBJECT_BUDGET } from "../adapters/raw-evidence.js";
32
- import { acquireBackfillLock, } from "../backfill-lock.js";
33
- import { BACKFILL_COMPLETION_RECHECK_MS, BACKFILL_COVERAGE_VERSION, emptyBackfillCursorState, prepareBackfillCursorForScope, readBackfillCursor, recordBackfillCursorObservations, recordBackfillScanCoverage, writeBackfillCompletionMarker, writeBackfillCursor, } from "../cursors/backfill-cursor.js";
34
- import { CLAUDE_CURSOR_FILENAME, readRawEvidenceCursor, writeRawEvidenceCursor, } from "../cursors/raw-evidence-cursor.js";
35
- import { getCollectorRuntimePaths, readLocalCollectorConfig, readLocalCollectorSessionFile, readLocalSessionReference, readLocalWorkContextForRepo, startLocalWorkContext } from "../local-state.js";
36
- import { DEFAULT_DISCOVERY_MAX_DEPTH, DEFAULT_DISCOVERY_MAX_REPOS, collectionRootPathAliases, discoverGitWorktreesInRootsWithStatus, } from "../repo-identity.js";
37
- import { isRawEvidenceUploadableAttributionState } from "../raw-evidence-attribution-policy.js";
38
- import { normalizeCollectionRoots } from "../root-normalization.js";
37
+ import { acquireBackfillLock } from "../backfill-lock.js";
38
+ import { writeBackfillCursor } from "../cursors/backfill-cursor.js";
39
39
  import { acquireSyncLock } from "../sync-lock.js";
40
- import { LocalUploadBlockedError, postCodexSessionReport, syncLocalAmbientEnvelope } from "../upload.js";
41
- const BACKFILL_UPLOAD_BATCH_SESSIONS = 25;
42
- const BACKFILL_MAX_CONSECUTIVE_FAILURES = 3;
43
- const ALL_BACKFILL_SINCE_MINUTES = 20 * 365 * 24 * 60;
44
- // Window a bare `cockpit backfill` uses. Wide enough to cover a new machine's
45
- // recent history and an intern who went quiet for a few weeks, narrow enough
46
- // that it is not the whole-history scan `--all` deliberately gates.
47
- export const DEFAULT_BACKFILL_SINCE_DAYS = 30;
40
+ import { DEFAULT_BACKFILL_SINCE_DAYS, backfillRetryCommand, defaultBackfillWindowNotice, } from "./backfill-command-line.js";
41
+ import { isInteractiveStdin, writeHumanBackfillResult, writeLine, } from "./backfill-output.js";
42
+ import { planBackfillRun } from "./backfill-plan.js";
43
+ import { reportBackfillOutcome } from "./backfill-report.js";
44
+ import { lockHeldResult } from "./backfill-result.js";
45
+ import { scanBackfillRun } from "./backfill-scan.js";
46
+ import { uploadBackfillBatches } from "./backfill-upload.js";
47
+ // The command's public surface. Callers (doctor, onboard, local, the CLI
48
+ // router, the BLI-3272 regression test) import these from `./backfill.js`;
49
+ // which sibling implements one is this module's business, not theirs.
50
+ export { DEFAULT_BACKFILL_SINCE_DAYS, backfillRetryCommand } from "./backfill-command-line.js";
51
+ export { buildBackfillSessionReport } from "./backfill-session-report.js";
48
52
  export async function runBackfillCommand(command, io) {
49
53
  // A bare `cockpit backfill` used to refuse and print three lines telling the
50
54
  // operator to pick a window. That put a mandatory flag on the command that
@@ -132,1928 +136,4 @@ export async function runBackfill(command, io) {
132
136
  await collectionLock.handle.release();
133
137
  await lock.handle.release();
134
138
  }
135
- }
136
- /**
137
- * Someone else is already holding a lock this run needs. Reported as blocked
138
- * with nothing done rather than as a failure — the scan is still valid, and
139
- * the retry command in it works as soon as the other run finishes.
140
- */
141
- function lockHeldResult(command, scanned, blocker) {
142
- return {
143
- ...baseBackfillResult(command, backfillResultBaseArgs(scanned)),
144
- status: "blocked",
145
- retry_command: scanned.retryCommand,
146
- failure_reason: blocker.failureReason,
147
- blocked_at: {
148
- what: blocker.what,
149
- batch_index: 0,
150
- batch_total: 0,
151
- done: 0,
152
- total: scanned.scan.candidates.length,
153
- },
154
- };
155
- }
156
- /** The `{now, dashboardUrl, sources, window, cursor, scan, reasonCounts}` bag every `baseBackfillResult` call needs. */
157
- function backfillResultBaseArgs(ctx) {
158
- return {
159
- now: ctx.now,
160
- dashboardUrl: ctx.dashboardUrl,
161
- sources: ctx.sources,
162
- window: ctx.window,
163
- cursor: ctx.cursor,
164
- scan: ctx.scan,
165
- reasonCounts: ctx.reasonCounts,
166
- };
167
- }
168
- /**
169
- * SCAN: is this machine paired, what roots/sources/window apply, and what did
170
- * the archived Codex + Claude session stores actually contain. Returns either
171
- * a terminal "not paired" result or everything PLAN/UPLOAD/REPORT need next.
172
- */
173
- async function scanBackfillRun(command, io, now) {
174
- const paths = getCollectorRuntimePaths(command.homeDir);
175
- const scope = await resolveBackfillScope(command, io, now, paths);
176
- if (scope.kind === "not_paired") {
177
- return {
178
- kind: "blocked",
179
- result: blockedBackfillResult(command, {
180
- now,
181
- dashboardUrl: scope.dashboardUrl,
182
- reason: "collector_not_paired",
183
- cursor: await readBackfillCursor(paths),
184
- }),
185
- };
186
- }
187
- const { collectionRoots, sources, worktreeDiscovery } = scope;
188
- const storedCursor = command.dryRun
189
- ? emptyBackfillCursorState()
190
- : await readBackfillCursor(paths);
191
- const scopedCursor = prepareBackfillCursorForScope(storedCursor, collectionRoots, sources);
192
- const cursor = scopedCursor.cursor;
193
- const pointerlessTerminalSessions = await loadPointerlessTerminalSessions(paths, sources);
194
- const scan = await scanBackfillSessions({
195
- command,
196
- homeDir: command.homeDir ?? os.homedir(),
197
- worktrees: worktreeDiscovery.worktrees,
198
- collectionRoots,
199
- sources,
200
- window: scope.window,
201
- cursor,
202
- pointerlessTerminalSessions,
203
- now,
204
- });
205
- const reasonCounts = await auditScannedCandidates(scan, worktreeDiscovery.incomplete_reasons);
206
- return {
207
- kind: "scanned",
208
- now,
209
- paths,
210
- dashboardUrl: scope.dashboardUrl,
211
- sources,
212
- window: scope.window,
213
- collectionRoots,
214
- worktrees: worktreeDiscovery.worktrees,
215
- retryCommand: scope.retryCommand,
216
- scopedCursor,
217
- cursor,
218
- scan,
219
- reasonCounts,
220
- oversizedCandidateKeys: oversizedBackfillCandidateKeys(scan.candidates),
221
- };
222
- }
223
- /**
224
- * Answers the four questions every later stage assumes: is this collector
225
- * paired, which approved roots and repos does it cover, which stores and how
226
- * far back was it asked for, and — on a dry run — can it even reach the
227
- * dashboard. Nothing here reads a session; a machine that is not paired stops
228
- * before any history is touched.
229
- */
230
- async function resolveBackfillScope(command, io, now, paths) {
231
- const config = await readLocalCollectorConfig(paths);
232
- const sessionFile = await readLocalCollectorSessionFile(paths);
233
- const session = await readLocalSessionReference(paths);
234
- if (session.session_state !== "valid") {
235
- return {
236
- kind: "not_paired",
237
- dashboardUrl: sessionFile.dashboard_url ?? config.dashboard_url,
238
- };
239
- }
240
- const pairedAt = parseRequiredDate(sessionFile.paired_at, "paired_at");
241
- const dashboardUrl = normalizeDashboardUrl(sessionFile.dashboard_url ?? config.dashboard_url);
242
- const roots = backfillCollectionRoots(command, config.default_repo_paths);
243
- const collectionRoots = normalizeCollectionRoots(await collectionRootPathAliases(roots));
244
- const worktreeDiscovery = await discoverBackfillWorktrees(collectionRoots, command);
245
- const window = backfillWindow(command, now, pairedAt);
246
- await validateBackfillReachability({
247
- fetchImpl: io.fetch,
248
- dashboardUrl,
249
- dryRun: command.dryRun,
250
- });
251
- return {
252
- kind: "ready",
253
- dashboardUrl,
254
- collectionRoots,
255
- worktreeDiscovery,
256
- retryCommand: backfillDiscoveryRetryCommand(command, worktreeDiscovery),
257
- sources: selectedSources(command.source),
258
- window,
259
- };
260
- }
261
- /**
262
- * Finishes the scan's ledger before anything acts on it: adds what repo
263
- * discovery could not enumerate, reads every candidate file once to find the
264
- * ones this machine cannot actually deliver, and folds all of it into the
265
- * reason table the operator sees. Mutates `scan` in place — it is the one
266
- * owner of `issues` and `retryable_candidate_keys` — and returns the table.
267
- */
268
- async function auditScannedCandidates(scan, discoveryIncompleteReasons) {
269
- addRepoDiscoveryIssues(scan.issues, discoveryIncompleteReasons);
270
- sortScanIssuesByPriority(scan.issues);
271
- const guards = await countReadOnlyGuards(scan.candidates);
272
- scan.retryable_candidate_keys = new Set([
273
- ...scan.retryable_candidate_keys,
274
- ...guards.retryable_candidate_keys,
275
- ]);
276
- addReadOnlyGuardIssues(scan.issues, guards.counts);
277
- return reasonCountsFor(scan.candidates, guards.counts, scan.issues);
278
- }
279
- /**
280
- * PLAN: given the scan, should this run actually upload anything right now?
281
- * `--all` without `--yes` needs interactive confirmation; `--dry-run` reports
282
- * what would happen and stops there. Either returns a terminal result;
283
- * anything else proceeds to UPLOAD.
284
- */
285
- async function planBackfillRun(command, io, ctx) {
286
- const { dashboardUrl, scan, reasonCounts } = ctx;
287
- if (command.all && !command.yes) {
288
- if (!command.json) {
289
- writeDryRunSummary(io, {
290
- candidates: scan.candidates,
291
- reasonCounts,
292
- dashboardUrl,
293
- dryRunOnly: false,
294
- });
295
- }
296
- const confirmed = await confirmAllBackfill(io);
297
- if (!confirmed) {
298
- return { kind: "result", result: confirmationDeclinedResult(command, ctx) };
299
- }
300
- }
301
- if (command.dryRun) {
302
- if (!command.json) {
303
- writeDryRunSummary(io, {
304
- candidates: scan.candidates,
305
- reasonCounts,
306
- dashboardUrl,
307
- dryRunOnly: true,
308
- });
309
- }
310
- return { kind: "result", result: dryRunResult(command, ctx) };
311
- }
312
- return { kind: "proceed" };
313
- }
314
- /** The operator saw the `--all` review and said no. Nothing was written; everything remains. */
315
- function confirmationDeclinedResult(command, ctx) {
316
- const base = baseBackfillResult(command, backfillResultBaseArgs(ctx));
317
- return {
318
- ...base,
319
- status: "blocked",
320
- counts: { ...base.counts, remaining: ctx.scan.candidates.length },
321
- failure_reason: "confirmation_declined",
322
- };
323
- }
324
- /**
325
- * What a `--dry-run` would have done. It completes unless the scan itself hit
326
- * something blocking, because a dry run that reports "complete" over a scan
327
- * that could not see all the history would be the confident wrong answer.
328
- */
329
- function dryRunResult(command, ctx) {
330
- const blockingIssues = blockingScanIssues(ctx.scan.issues);
331
- return {
332
- ...baseBackfillResult(command, backfillResultBaseArgs(ctx)),
333
- status: blockingIssues.length > 0 ? "partial" : "complete",
334
- dry_run: true,
335
- retry_command: ctx.retryCommand,
336
- ...(blockingIssues.length > 0
337
- ? { failure_reason: blockingIssues[0]?.reason }
338
- : {}),
339
- };
340
- }
341
- /**
342
- * UPLOAD: sync every uploadable candidate in fixed-size batches, heartbeating
343
- * the backfill lock between batches, stopping early on an exhausted budget or
344
- * three consecutive failed batches. A candidate whose main is durably
345
- * oversized-skipped (BLI-2727) is accounted for, not missing — it will never
346
- * earn a durable pointer under the current cap, and treating it as "still
347
- * missing" would fail every batch it happens to share with genuinely uploaded
348
- * siblings, so it is excluded via `oversizedCandidateKeys` throughout.
349
- */
350
- async function uploadBackfillBatches(command, io, lock, ctx) {
351
- const uploadable = uploadableCandidates(ctx.scan.candidates);
352
- const batches = buildBackfillBatches(uploadable);
353
- const rawEvidenceBudget = {
354
- remainingBytes: RAW_EVIDENCE_DEFAULT_BYTE_BUDGET,
355
- remainingObjects: RAW_EVIDENCE_DEFAULT_OBJECT_BUDGET,
356
- };
357
- const tally = emptyUploadTally();
358
- let consecutiveFailures = 0;
359
- let blockedAt;
360
- let failureReason = blockingScanIssues(ctx.scan.issues)[0]?.reason;
361
- const stoppedAt = (what, index) => ({
362
- what,
363
- batch_index: index + 1,
364
- batch_total: batches.length,
365
- done: tally.done,
366
- total: uploadable.length,
367
- });
368
- for (const [index, batch] of batches.entries()) {
369
- await lock.handle.heartbeat();
370
- const outcome = await uploadOneBackfillBatch(command, io, ctx, batch, rawEvidenceBudget);
371
- foldBatchIntoTally(tally, batch, outcome);
372
- consecutiveFailures = outcome.failed ? consecutiveFailures + 1 : 0;
373
- if (!command.json) {
374
- writeLine(io.stdout, `Uploaded ${tally.done}/${uploadable.length} (batch ${index + 1}/${batches.length})`);
375
- }
376
- await yieldToEventLoop();
377
- if (outcome.deferred) {
378
- blockedAt = stoppedAt("raw evidence budget exhausted", index);
379
- failureReason = "deferred_budget_exhausted";
380
- break;
381
- }
382
- if (consecutiveFailures >= BACKFILL_MAX_CONSECUTIVE_FAILURES) {
383
- blockedAt = stoppedAt("consecutive upload failures", index);
384
- failureReason = failedBatchReason(outcome.sync, outcome.missingDurableMain);
385
- break;
386
- }
387
- }
388
- return { ...tally, uploadable, batches, blockedAt, failureReason };
389
- }
390
- /** Uploads one batch and judges it; every judgement here is a named predicate. */
391
- async function uploadOneBackfillBatch(command, io, ctx, batch, rawEvidenceBudget) {
392
- const sync = await syncBackfillBatch({
393
- command,
394
- batch,
395
- worktrees: ctx.worktrees,
396
- codexAttribution: ctx.scan.codexAttribution,
397
- claudeAttribution: ctx.scan.claudeAttribution,
398
- rawEvidenceBudget,
399
- fetchImpl: io.fetch,
400
- });
401
- const durableInBatch = durableBackfillCandidateKeys(batch, sync);
402
- const missingDurableMain = countMissingDurableMains(batch, durableInBatch, ctx.oversizedCandidateKeys);
403
- return {
404
- sync,
405
- durableInBatch,
406
- missingDurableMain,
407
- failed: didBatchFail(sync, missingDurableMain),
408
- deferred: deferredEvidenceCount(sync) > 0,
409
- };
410
- }
411
- function emptyUploadTally() {
412
- return {
413
- syncResults: [],
414
- completedBatches: 0,
415
- failedBatches: 0,
416
- done: 0,
417
- failed: 0,
418
- deferred: 0,
419
- uploadedObjects: 0,
420
- uploadedChunks: 0,
421
- backfilledSessions: 0,
422
- durableCandidateKeys: new Set(),
423
- };
424
- }
425
- function foldBatchIntoTally(tally, batch, outcome) {
426
- tally.syncResults.push(outcome.sync);
427
- tally.done += batch.candidates.length;
428
- tally.uploadedObjects += outcome.sync.raw_evidence_uploaded_object_count;
429
- tally.uploadedChunks += outcome.sync.raw_evidence_uploaded_chunk_count;
430
- tally.failed += countSessionUploadFailures(outcome.sync);
431
- tally.deferred += deferredEvidenceCount(outcome.sync);
432
- for (const key of outcome.durableInBatch)
433
- tally.durableCandidateKeys.add(key);
434
- tally.backfilledSessions = tally.durableCandidateKeys.size;
435
- if (outcome.failed)
436
- tally.failedBatches += 1;
437
- if (!outcome.failed && !outcome.deferred)
438
- tally.completedBatches += 1;
439
- }
440
- /** Evidence the server accepted the batch for but had no budget left to store. */
441
- function deferredEvidenceCount(sync) {
442
- return (sync.raw_evidence_deferred_byte_budget +
443
- sync.raw_evidence_deferred_object_budget);
444
- }
445
- /**
446
- * Candidates in this batch that still owe a durable main transcript. An
447
- * oversized skip is excluded (BLI-2727): it can never earn a pointer under the
448
- * current cap, and counting it here would fail every batch it happens to share
449
- * with genuinely uploaded siblings.
450
- */
451
- function countMissingDurableMains(batch, durableInBatch, oversizedCandidateKeys) {
452
- return batch.candidates.filter((candidate) => !durableInBatch.has(candidateCursorKey(candidate)) &&
453
- !oversizedCandidateKeys.has(candidateCursorKey(candidate))).length;
454
- }
455
- /** A batch counts as failed if it did not upload, lost a session, or left one pointerless. */
456
- function didBatchFail(sync, missingDurableMain) {
457
- return (sync.status !== "uploaded" ||
458
- countSessionUploadFailures(sync) > 0 ||
459
- missingDurableMain > 0);
460
- }
461
- /** Names which of the three failure shapes ended the run, most specific first. */
462
- function failedBatchReason(sync, missingDurableMain) {
463
- if (sync.status === "spooled")
464
- return sync.failure_reason;
465
- if (missingDurableMain > 0)
466
- return "durable_session_pointer_missing";
467
- return "upload_failed";
468
- }
469
- /**
470
- * REPORT: post the session report, record durable pointers, advance the
471
- * cursor through the resolved contiguous prefix, decide completion, write the
472
- * all-history completion marker when this run actually finished it, and
473
- * assemble the final result. `remaining`/`completionBlocked` deliberately
474
- * treat an oversized skip differently (BLI-2727): it still counts toward
475
- * `remaining` so the JSON output never goes silent about it, but it never
476
- * blocks completion on its own — every completion-gating computation below
477
- * excludes it explicitly via `oversizedCandidateKeys`/`blockingScanIssues`.
478
- */
479
- async function reportBackfillOutcome(command, io, ctx, upload) {
480
- let { blockedAt, failureReason } = upload;
481
- const posted = await postBackfillSessionReport(command, io, ctx, upload);
482
- if (!posted.acknowledged && !blockedAt) {
483
- blockedAt = sessionReportStopPoint(upload);
484
- failureReason ??= sessionReportFailureReason(posted);
485
- }
486
- if (posted.acknowledged) {
487
- await checkpointResolvedBackfillProgress(ctx, upload);
488
- }
489
- const completion = summarizeBackfillCompletion({
490
- ctx,
491
- upload,
492
- posted,
493
- stoppedEarly: Boolean(blockedAt),
494
- failureReason,
495
- });
496
- if (completion.status === "complete" && command.all) {
497
- await writeAllHistoryCompletionMarker(ctx);
498
- }
499
- return assembleBackfillResult({
500
- command,
501
- ctx,
502
- upload,
503
- posted,
504
- completion,
505
- blockedAt,
506
- });
507
- }
508
- /**
509
- * Tells the server what this run observed about every session, uploaded or
510
- * not. Its acknowledgement — not raw-evidence durability alone — is what makes
511
- * a historical cursor position irreversible, so the caller gates the whole
512
- * checkpoint on the `acknowledged` flag returned here.
513
- */
514
- async function postBackfillSessionReport(command, io, ctx, upload) {
515
- const sessions = buildBackfillSessionReport({
516
- candidates: ctx.scan.candidates,
517
- syncResults: upload.syncResults,
518
- now: ctx.now,
519
- });
520
- if (sessions.length === 0) {
521
- return {
522
- sessions,
523
- report: emptyReport("no_sessions_observed"),
524
- acknowledged: true,
525
- };
526
- }
527
- const reportContext = await ensureBackfillReportContext({
528
- homeDir: command.homeDir,
529
- paths: ctx.paths,
530
- collectionRoots: ctx.collectionRoots,
531
- worktrees: ctx.worktrees,
532
- candidates: ctx.scan.candidates,
533
- });
534
- const report = await postCodexSessionReport({
535
- homeDir: command.homeDir,
536
- repoRoot: reportContext.repoRoot,
537
- dashboardUrl: ctx.dashboardUrl,
538
- sessions,
539
- fetch: io.fetch,
540
- now: ctx.now,
541
- });
542
- return {
543
- sessions,
544
- report,
545
- acknowledged: report.posted && report.recorded_count >= sessions.length,
546
- };
547
- }
548
- /** Where a run that uploaded cleanly but could not file its report stopped. */
549
- function sessionReportStopPoint(upload) {
550
- return {
551
- what: "session report failed",
552
- batch_index: upload.completedBatches,
553
- batch_total: upload.batches.length,
554
- done: upload.done,
555
- total: upload.uploadable.length,
556
- };
557
- }
558
- /** A partial acknowledgement is its own reason; otherwise the post's own. */
559
- function sessionReportFailureReason(posted) {
560
- return posted.report.posted &&
561
- posted.report.recorded_count < posted.sessions.length
562
- ? "session_report_ack_incomplete"
563
- : posted.report.reason;
564
- }
565
- /**
566
- * Makes this run's progress durable once the server has acknowledged it:
567
- * session pointers first, then the backfill cursor through the contiguous
568
- * resolved prefix. Raw-evidence durability is necessary but not sufficient —
569
- * both writes wait on the report acknowledgement, because a cursor advanced
570
- * past a session the server never recorded can never be walked back.
571
- */
572
- async function checkpointResolvedBackfillProgress(ctx, upload) {
573
- const { paths, scan, cursor, now } = ctx;
574
- if (upload.durableCandidateKeys.size > 0) {
575
- await recordBackfillDurableSessionPointers({
576
- paths,
577
- candidates: scan.candidates,
578
- syncResults: upload.syncResults,
579
- now,
580
- });
581
- }
582
- const cursorAdvanced = advanceBackfillCursorThroughResolvedPrefix({
583
- cursor,
584
- candidates: scan.candidates,
585
- durableCandidateKeys: upload.durableCandidateKeys,
586
- retryableCandidateKeys: scan.retryable_candidate_keys,
587
- discoveryComplete: !scan.issues.some((issue) => issue.scope === "global"),
588
- now,
589
- });
590
- if (cursorAdvanced)
591
- await writeBackfillCursor(paths, cursor);
592
- }
593
- /**
594
- * Answers "is this backfill finished, and if not, why not" as pure arithmetic
595
- * over what the scan found and what the upload resolved — no side effects, so
596
- * the one place BLI-2727's oversized-skip rule is applied is readable in full.
597
- * `remaining` is a reporting total that still counts oversized skips so the
598
- * JSON never goes silent about them; every `*Blocking` count excludes them,
599
- * because a file too large for the current cap is a permanent labeled fact
600
- * that no rerun can resolve and completion must not wait on forever.
601
- */
602
- function summarizeBackfillCompletion(options) {
603
- const { scan, oversizedCandidateKeys } = options.ctx;
604
- const { upload, posted } = options;
605
- const unresolvedUploadableKeys = new Set(upload.uploadable
606
- .filter((candidate) => !upload.durableCandidateKeys.has(candidateCursorKey(candidate)))
607
- .map(candidateCursorKey));
608
- const unresolvedUploadableBlocking = [...unresolvedUploadableKeys].filter((key) => !oversizedCandidateKeys.has(key)).length;
609
- const unresolvedRetryableBlocking = [...scan.retryable_candidate_keys].filter((key) => !oversizedCandidateKeys.has(key)).length;
610
- const remaining = countRemainingHistory({
611
- scan,
612
- unresolvedUploadableKeys,
613
- posted,
614
- });
615
- const failed = Math.max(upload.failed, unresolvedUploadableBlocking);
616
- const blockingIssues = blockingScanIssues(scan.issues);
617
- const completionBlocked = blockingIssues.length > 0 ||
618
- unresolvedUploadableBlocking > 0 ||
619
- unresolvedRetryableBlocking > 0 ||
620
- upload.deferred > 0 ||
621
- failed > 0 ||
622
- !posted.acknowledged;
623
- let failureReason = options.failureReason;
624
- if (!failureReason && completionBlocked) {
625
- const retryableCandidateReason = scan.candidates.find((candidate) => scan.retryable_candidate_keys.has(candidateCursorKey(candidate)) &&
626
- !oversizedCandidateKeys.has(candidateCursorKey(candidate)))?.reason;
627
- failureReason =
628
- blockingIssues[0]?.reason ??
629
- (unresolvedUploadableBlocking > 0
630
- ? "durable_session_pointer_missing"
631
- : (retryableCandidateReason ?? "backfill_incomplete"));
632
- }
633
- return {
634
- remaining,
635
- failed,
636
- status: options.stoppedEarly || completionBlocked ? "partial" : "complete",
637
- failureReason,
638
- };
639
- }
640
- /**
641
- * How much history a rerun still has to do. A reporting total, not a
642
- * completion gate: it counts every unresolved candidate — oversized skips
643
- * included, so `--json` never goes silent about them (BLI-2727) — plus what
644
- * selection dropped, what discovery never saw, and the report itself when the
645
- * server has not acknowledged it.
646
- */
647
- function countRemainingHistory(options) {
648
- const unresolvedKnownKeys = new Set([
649
- ...options.unresolvedUploadableKeys,
650
- ...options.scan.retryable_candidate_keys,
651
- ]);
652
- const unseenGlobalFailures = options.scan.issues
653
- .filter((issue) => issue.scope === "global")
654
- .reduce((total, issue) => total + issue.count, 0);
655
- const reportRetryable = options.posted.acknowledged
656
- ? 0
657
- : // The cursor is intentionally all-or-nothing for the report. Even
658
- // chunks already accepted by the server are retried idempotently when
659
- // another required chunk lacks an acknowledgement.
660
- Math.max(1, options.posted.sessions.length);
661
- return (unresolvedKnownKeys.size +
662
- options.scan.omitted_candidate_count +
663
- unseenGlobalFailures +
664
- reportRetryable);
665
- }
666
- /**
667
- * Proof, for doctor and status, that archived history is covered. Only an
668
- * `--all` run earns it: a bounded `--since-days` run may complete its
669
- * requested window, but that is not proof of an all-history backfill.
670
- */
671
- async function writeAllHistoryCompletionMarker(ctx) {
672
- const { paths, cursor, sources, scopedCursor, scan, oversizedCandidateKeys, now } = ctx;
673
- recordBackfillScanCoverage(cursor, sources, now, now);
674
- await writeBackfillCursor(paths, cursor);
675
- const oversizedCandidates = scan.candidates.filter((candidate) => oversizedCandidateKeys.has(candidateCursorKey(candidate)));
676
- await writeBackfillCompletionMarker(paths, {
677
- schema_version: "cockpit-backfill-complete.v2",
678
- coverage_version: BACKFILL_COVERAGE_VERSION,
679
- collection_scope_id: scopedCursor.collection_scope_id,
680
- sources,
681
- completed_at: now.toISOString(),
682
- revalidate_after: new Date(now.getTime() + BACKFILL_COMPLETION_RECHECK_MS).toISOString(),
683
- cursor,
684
- ...(oversizedCandidates.length > 0
685
- ? {
686
- oversized_skips: {
687
- reason: "file_too_large",
688
- count: oversizedCandidates.length,
689
- byte_sizes: oversizedCandidates.map((candidate) => candidate.byte_size),
690
- },
691
- }
692
- : {}),
693
- });
694
- }
695
- /** The `--json` payload: every stage's numbers merged onto the scan's baseline. */
696
- function assembleBackfillResult(options) {
697
- const { command, ctx, upload, posted, completion } = options;
698
- const base = baseBackfillResult(command, backfillResultBaseArgs(ctx));
699
- return {
700
- ...base,
701
- status: completion.status,
702
- retry_command: ctx.retryCommand,
703
- counts: {
704
- ...base.counts,
705
- backfilled: upload.backfilledSessions,
706
- failed: completion.failed,
707
- deferred: upload.deferred,
708
- remaining: completion.remaining,
709
- },
710
- batches: {
711
- total: upload.batches.length,
712
- completed: upload.completedBatches,
713
- failed: upload.failedBatches,
714
- },
715
- report: posted.report,
716
- server_acknowledged: {
717
- codex_session_report_recorded_count: posted.report.recorded_count,
718
- raw_evidence_uploaded_object_count: upload.uploadedObjects,
719
- raw_evidence_uploaded_chunk_count: upload.uploadedChunks,
720
- },
721
- blocked_at: options.blockedAt,
722
- failure_reason: completion.failureReason,
723
- };
724
- }
725
- function defaultBackfillWindowNotice() {
726
- return [
727
- `No window given — backfilling the last ${DEFAULT_BACKFILL_SINCE_DAYS} days.`,
728
- "The effective start is capped at the collector paired_at timestamp.",
729
- "Use `cockpit backfill --since-days N` for a different window, or `cockpit backfill --all` for the full local history (review a dry-run first; add `--yes` on headless agent runs).",
730
- ].join("\n");
731
- }
732
- /**
733
- * Produces a copyable retry that preserves the requested history window and
734
- * every discovery/selection override. Values that need quoting use syntax
735
- * accepted by the supported native shells: POSIX shells on macOS and
736
- * PowerShell on Windows.
737
- */
738
- export function backfillRetryCommand(command) {
739
- const parts = ["cockpit", "backfill"];
740
- if (command.all) {
741
- parts.push("--all", "--yes");
742
- }
743
- else if (command.sinceDays !== undefined) {
744
- parts.push("--since-days", String(command.sinceDays));
745
- }
746
- if (command.source)
747
- parts.push("--source", command.source);
748
- if (command.maxFiles !== undefined) {
749
- parts.push("--max-files", String(command.maxFiles));
750
- }
751
- if (command.maxDepth !== undefined) {
752
- parts.push("--max-depth", String(command.maxDepth));
753
- }
754
- if (command.maxRepos !== undefined) {
755
- parts.push("--max-repos", String(command.maxRepos));
756
- }
757
- if (command.repoRoot) {
758
- parts.push("--workspace", quoteCliArgument(command.repoRoot));
759
- }
760
- if (command.dryRun)
761
- parts.push("--dry-run");
762
- return parts.join(" ");
763
- }
764
- function backfillDiscoveryRetryCommand(command, discovery) {
765
- const retry = { ...command };
766
- if (discovery.incomplete_reasons.includes("max_depth_reached")) {
767
- retry.maxDepth =
768
- (command.maxDepth ?? DEFAULT_DISCOVERY_MAX_DEPTH) + 1;
769
- }
770
- if (discovery.incomplete_reasons.includes("max_worktrees_reached")) {
771
- retry.maxRepos =
772
- (command.maxRepos ?? DEFAULT_DISCOVERY_MAX_REPOS) * 2;
773
- }
774
- return backfillRetryCommand(retry);
775
- }
776
- function quoteCliArgument(value) {
777
- if (/^[a-z0-9_./:\\-]+$/iu.test(value))
778
- return value;
779
- const escaped = process.platform === "win32"
780
- ? value.replaceAll("'", "''")
781
- : value.replaceAll("'", `'\"'\"'`);
782
- return `'${escaped}'`;
783
- }
784
- function backfillCollectionRoots(command, savedRoots) {
785
- if (command.repoRoot)
786
- return [path.resolve(command.repoRoot)];
787
- const roots = normalizeCollectionRoots(savedRoots);
788
- if (roots.length === 0) {
789
- throw new Error("No saved collection roots. Run `cockpit onboard --workspace <path>` or pass `cockpit backfill --workspace <path>`.");
790
- }
791
- return roots;
792
- }
793
- async function discoverBackfillWorktrees(roots, command) {
794
- return discoverGitWorktreesInRootsWithStatus(roots, {
795
- maxDepth: command.maxDepth ?? DEFAULT_DISCOVERY_MAX_DEPTH,
796
- maxWorktrees: command.maxRepos ?? DEFAULT_DISCOVERY_MAX_REPOS,
797
- });
798
- }
799
- function selectedSources(source) {
800
- if (source === "codex")
801
- return ["codex"];
802
- if (source === "claude")
803
- return ["claude_code"];
804
- return ["codex", "claude_code"];
805
- }
806
- function backfillWindow(command, now, pairedAt) {
807
- if (command.all) {
808
- return {
809
- mode: "all",
810
- since_days: null,
811
- started_at: new Date(now.getTime() - ALL_BACKFILL_SINCE_MINUTES * 60_000)
812
- .toISOString(),
813
- paired_at: pairedAt.toISOString(),
814
- since_minutes: ALL_BACKFILL_SINCE_MINUTES,
815
- };
816
- }
817
- const requestedMs = now.getTime() - (command.sinceDays ?? 1) * 24 * 60 * 60_000;
818
- const startedAtMs = Math.max(requestedMs, pairedAt.getTime());
819
- const sinceMinutes = Math.max(1, Math.ceil((now.getTime() - startedAtMs) / 60_000));
820
- return {
821
- mode: "since_days",
822
- since_days: command.sinceDays ?? null,
823
- started_at: new Date(startedAtMs).toISOString(),
824
- paired_at: pairedAt.toISOString(),
825
- since_minutes: sinceMinutes,
826
- };
827
- }
828
- async function validateBackfillReachability(options) {
829
- if (!options.dryRun)
830
- return;
831
- const response = await options.fetchImpl(options.dashboardUrl, {
832
- method: "HEAD",
833
- });
834
- if (response.status >= 500) {
835
- throw new Error(`Dashboard reachability failed with HTTP ${response.status}.`);
836
- }
837
- }
838
- /**
839
- * Reads both archived session stores and turns them into the candidates this
840
- * run will consider: scan each selected store, keep what the cursor has not
841
- * already resolved, order it, apply the operator's `--max-files` cap, then
842
- * census what could not be accounted for.
843
- */
844
- async function scanBackfillSessions(options) {
845
- const scanLimit = options.command.maxFiles ?? 10_000;
846
- const codexSessionDirs = defaultCodexSessionDirs(options.homeDir);
847
- const claudeProjectsDir = path.join(options.homeDir, ".claude", "projects");
848
- const codexAttribution = options.sources.includes("codex")
849
- ? await scanCodexHistory(options, codexSessionDirs, scanLimit)
850
- : null;
851
- const claudeAttribution = options.sources.includes("claude_code")
852
- ? await scanClaudeHistory(options, claudeProjectsDir, scanLimit)
853
- : null;
854
- const selection = selectBackfillCandidates({
855
- codexAttribution,
856
- claudeAttribution,
857
- cursor: options.cursor,
858
- pointerlessTerminalSessions: options.pointerlessTerminalSessions,
859
- maxFiles: options.command.maxFiles,
860
- });
861
- const issues = await backfillScanIssues({
862
- codexAttribution,
863
- claudeAttribution,
864
- codexSessionDirs,
865
- claudeProjectsDir,
866
- candidates: selection.candidates,
867
- omittedCandidateCount: selection.omittedCandidateCount,
868
- });
869
- return {
870
- candidates: selection.candidates,
871
- codexAttribution,
872
- claudeAttribution,
873
- issues,
874
- retryable_candidate_keys: retryableCandidateKeys(selection.candidates),
875
- omitted_candidate_count: selection.omittedCandidateCount,
876
- };
877
- }
878
- /**
879
- * The adapter's own limit is a memory guard, not permission to silently leave
880
- * history undiscovered. When it fires, re-scan the exact discovered population
881
- * so a user-facing `--max-files` cap can resume from a truthful ordering.
882
- */
883
- async function scanCodexHistory(request, sessionsDirs, scanLimit) {
884
- const scan = (limit) => scanAndAttributeCodexSessions({
885
- sessionsDirs,
886
- worktrees: request.worktrees,
887
- now: request.now,
888
- sinceMinutes: request.window.since_minutes,
889
- limit,
890
- collectionRoots: request.collectionRoots,
891
- });
892
- const firstPass = await scan(scanLimit);
893
- if (!firstPass.session_limit_applied)
894
- return firstPass;
895
- return scan(Math.max(scanLimit + 1, firstPass.discovered_file_count));
896
- }
897
- /** Same rule for the Claude store: a capped first pass is rescanned in full. */
898
- async function scanClaudeHistory(request, projectsDir, scanLimit) {
899
- const scan = (limit) => scanAndAttributeClaudeSessions({
900
- projectsDir,
901
- worktrees: request.worktrees,
902
- now: request.now,
903
- sinceMinutes: request.window.since_minutes,
904
- limit,
905
- collectionRoots: request.collectionRoots,
906
- });
907
- const firstPass = await scan(scanLimit);
908
- if (!firstPass.session_limit_applied)
909
- return firstPass;
910
- return scan(Math.max(scanLimit + 1, firstPass.discovered_session_count));
911
- }
912
- /**
913
- * Which scanned sessions this run will actually work on. A session is in play
914
- * if the cursor has not passed it, or if it is a terminal session that never
915
- * earned a pointer and so deserves another attempt; those retries sort first
916
- * so a `--max-files` cap spends its budget on them before newer history.
917
- */
918
- function selectBackfillCandidates(options) {
919
- const allCandidates = [
920
- ...(options.codexAttribution?.results.map(normalizeCodexCandidate) ?? []),
921
- ...(options.claudeAttribution?.results.map(normalizeClaudeCandidate) ?? []),
922
- ]
923
- .filter((candidate) => isAfterCursor(candidate, options.cursor) ||
924
- isPointerlessTerminalRetry(candidate, options.pointerlessTerminalSessions))
925
- .sort((a, b) => compareBackfillCandidatesForRetry(a, b, options.pointerlessTerminalSessions));
926
- const candidates = allCandidates.slice(0, options.maxFiles ?? Number.MAX_SAFE_INTEGER);
927
- return {
928
- candidates,
929
- omittedCandidateCount: allCandidates.length - candidates.length,
930
- };
931
- }
932
- async function loadPointerlessTerminalSessions(paths, sources) {
933
- const [codexCursor, claudeCursor] = await Promise.all([
934
- sources.includes("codex")
935
- ? readRawEvidenceCursor(paths)
936
- : Promise.resolve(null),
937
- sources.includes("claude_code")
938
- ? readRawEvidenceCursor(paths, { filename: CLAUDE_CURSOR_FILENAME })
939
- : Promise.resolve(null),
940
- ]);
941
- const terminalIds = (sessions) => new Set(Object.entries(sessions ?? {})
942
- .filter(([, entry]) => isPointerlessTerminalCursorEntry(entry))
943
- .map(([sessionId]) => sessionId));
944
- const result = {
945
- codex: terminalIds(codexCursor?.sessions),
946
- claude_code: terminalIds(claudeCursor?.sessions),
947
- };
948
- const count = result.codex.size + result.claude_code.size;
949
- if (count > 0) {
950
- console.error("[backfill] pointer-less terminal sessions reopened", JSON.stringify({
951
- count,
952
- codex_count: result.codex.size,
953
- claude_count: result.claude_code.size,
954
- }));
955
- }
956
- return result;
957
- }
958
- function isPointerlessTerminalCursorEntry(entry) {
959
- return (!entry.uploaded_object_key &&
960
- (entry.state === "ambiguous" ||
961
- entry.state === "unattributed" ||
962
- entry.state === "skipped"));
963
- }
964
- function isPointerlessTerminalRetry(candidate, sessions) {
965
- return sessions[candidate.source].has(candidate.session_id);
966
- }
967
- function compareBackfillCandidatesForRetry(a, b, sessions) {
968
- const aRetry = isPointerlessTerminalRetry(a, sessions);
969
- const bRetry = isPointerlessTerminalRetry(b, sessions);
970
- if (aRetry !== bRetry)
971
- return aRetry ? -1 : 1;
972
- if (aRetry && bRetry) {
973
- return (a.session_file_mtime_ms - b.session_file_mtime_ms ||
974
- candidateCursorKey(a).localeCompare(candidateCursorKey(b)));
975
- }
976
- return compareBackfillCandidates(a, b);
977
- }
978
- function normalizeCodexCandidate(result) {
979
- return {
980
- source: "codex",
981
- session_id: result.codex_session_id,
982
- file_path: result.file_path,
983
- state: result.state,
984
- reason: result.reason,
985
- signals: result.signals,
986
- attribution_score: result.attribution_score,
987
- path_score: result.path_score,
988
- content_hash_sha256: result.content_hash_sha256,
989
- byte_size: result.byte_size,
990
- session_file_mtime: result.session_file_mtime,
991
- session_file_mtime_ms: result.session_file_mtime_ms,
992
- worktree: result.worktree,
993
- cwd_basename: result.cwd_basename,
994
- cwd_hash: result.cwd_hash,
995
- };
996
- }
997
- function normalizeClaudeCandidate(result) {
998
- return {
999
- source: "claude_code",
1000
- session_id: result.claude_session_id,
1001
- file_path: result.file_path,
1002
- state: result.state,
1003
- reason: result.reason,
1004
- signals: result.signals,
1005
- attribution_score: result.attribution_score,
1006
- path_score: result.path_score,
1007
- content_hash_sha256: result.content_hash_sha256,
1008
- byte_size: result.byte_size,
1009
- session_file_mtime: result.session_file_mtime,
1010
- session_file_mtime_ms: result.session_file_mtime_ms,
1011
- worktree: result.worktree,
1012
- cwd_basename: result.cwd_basename,
1013
- cwd_hash: result.cwd_hash,
1014
- claude: result,
1015
- };
1016
- }
1017
- function isAfterCursor(candidate, cursor) {
1018
- const source = cursor.sources[candidate.source];
1019
- const oldest = source?.oldest_mtime_ms_processed;
1020
- if (oldest === null || oldest === undefined)
1021
- return true;
1022
- if (candidate.session_file_mtime_ms < oldest)
1023
- return true;
1024
- const key = candidateCursorKey(candidate);
1025
- if (candidate.session_file_mtime_ms === oldest) {
1026
- return !source.processed_keys_at_oldest_mtime.includes(key);
1027
- }
1028
- const newest = source.newest_mtime_ms_covered;
1029
- // A cursor written before upper-edge coverage existed gets one safe
1030
- // migration pass across the previously processed interval. Once that pass is
1031
- // acknowledged, subsequent --all runs only inspect genuinely newer files.
1032
- if (newest === null)
1033
- return true;
1034
- if (candidate.session_file_mtime_ms > newest)
1035
- return true;
1036
- if (candidate.session_file_mtime_ms < newest)
1037
- return false;
1038
- // A legacy cursor has no boundary identities. Rechecking the equal-time
1039
- // boundary is safe because durable evidence is content-addressed.
1040
- return !source.processed_keys_at_newest_mtime.includes(key);
1041
- }
1042
- function compareBackfillCandidates(a, b) {
1043
- return (b.session_file_mtime_ms - a.session_file_mtime_ms ||
1044
- candidateCursorKey(a).localeCompare(candidateCursorKey(b)));
1045
- }
1046
- function candidateCursorKey(candidate) {
1047
- return crypto
1048
- .createHash("sha256")
1049
- .update(JSON.stringify([
1050
- candidate.source,
1051
- candidate.session_id,
1052
- normalizedCursorPath(candidate.file_path),
1053
- ]))
1054
- .digest("hex");
1055
- }
1056
- function normalizedCursorPath(filePath) {
1057
- const windowsStyle = path.win32.isAbsolute(filePath) && !path.posix.isAbsolute(filePath);
1058
- if (windowsStyle)
1059
- return path.win32.normalize(filePath).toLowerCase();
1060
- const normalized = path.resolve(filePath);
1061
- return process.platform === "win32" ? normalized.toLowerCase() : normalized;
1062
- }
1063
- /**
1064
- * Everything the scan could not account for cleanly, in one ledger. Read in
1065
- * three passes — what the Codex store hid, what the Claude store hid, then
1066
- * what is wrong with the candidates that survived — because the first two are
1067
- * global (discovery may have missed a session at any mtime, so no watermark is
1068
- * safe to advance) and the third only stops the contiguous cursor prefix.
1069
- */
1070
- async function backfillScanIssues(options) {
1071
- const issues = [];
1072
- if (options.codexAttribution) {
1073
- await addCodexStoreScanIssues(issues, options.codexAttribution, options.codexSessionDirs);
1074
- }
1075
- if (options.claudeAttribution) {
1076
- await addClaudeStoreScanIssues(issues, options.claudeAttribution, options.claudeProjectsDir, options.candidates);
1077
- }
1078
- addCandidateScanIssues(issues, options.candidates, options.omittedCandidateCount);
1079
- sortScanIssuesByPriority(issues);
1080
- return issues;
1081
- }
1082
- /** What the archived Codex store could not tell us — every one of these hides history. */
1083
- async function addCodexStoreScanIssues(issues, codexAttribution, codexSessionDirs) {
1084
- countScanIssue(issues, "codex_session_limit_applied", codexAttribution.session_limit_applied
1085
- ? Math.max(1, codexAttribution.discovered_file_count -
1086
- codexAttribution.scanned_file_count)
1087
- : 0, "global");
1088
- // A session directory that simply does not exist on this machine is not a
1089
- // read failure; subtract those before reporting one.
1090
- const missingTopLevelDirs = (await Promise.all(codexSessionDirs.map(isMissingPath))).filter(Boolean).length;
1091
- countScanIssue(issues, "codex_directory_read_failed", Math.max(0, codexAttribution.directory_read_failed_count - missingTopLevelDirs), "global");
1092
- countScanIssue(issues, "codex_session_stat_failed", codexAttribution.stat_failed_count, "global");
1093
- }
1094
- /** The same census for the Claude store, which also has sidecars to account for. */
1095
- async function addClaudeStoreScanIssues(issues, claudeAttribution, claudeProjectsDir, candidates) {
1096
- countScanIssue(issues, "claude_session_limit_applied", claudeAttribution.session_limit_applied
1097
- ? Math.max(1, claudeAttribution.discovered_session_count -
1098
- claudeAttribution.scanned_session_count)
1099
- : 0, "global");
1100
- const projectsDirMissing = await isMissingPath(claudeProjectsDir);
1101
- countScanIssue(issues, "claude_project_dir_read_failed", Math.max(0, claudeAttribution.project_dir_read_failed_count -
1102
- (projectsDirMissing ? 1 : 0)), "global");
1103
- countScanIssue(issues, "claude_session_stat_failed", claudeAttribution.session_stat_failed_count, "global");
1104
- countScanIssue(issues, "claude_sidecar_stat_failed", claudeAttribution.sidecar_stat_failed_count, "global");
1105
- countScanIssue(issues, "claude_sidecar_dir_read_failed", await countUnreadableClaudeSidecarDirs(candidates), "global");
1106
- }
1107
- /** What is wrong with the sessions that did survive discovery, plus the one we chose to drop. */
1108
- function addCandidateScanIssues(issues, candidates, omittedCandidateCount) {
1109
- countScanIssue(issues, "backfill_max_files_applied", omittedCandidateCount, "selection");
1110
- countScanIssue(issues, "candidate_file_read_failed", candidates.filter((candidate) => candidate.reason === "file_read_failed")
1111
- .length, "candidate");
1112
- countScanIssue(issues, "candidate_worktree_unavailable", candidates.filter((candidate) => isRawEvidenceUploadableAttributionState(candidate.state, candidate.worktree !== null) && candidate.worktree === null).length, "candidate");
1113
- countScanIssue(issues, "claude_main_file_too_large", candidates.filter((candidate) => candidate.source === "claude_code" &&
1114
- candidate.claude?.main_file_oversized).length, "candidate");
1115
- countScanIssue(issues, "claude_sidecar_limit_applied", candidates.reduce((total, candidate) => total + (candidate.claude?.sidecars_capped ?? 0), 0), "candidate");
1116
- countScanIssue(issues, "claude_sidecar_file_unreadable", candidates.reduce((total, candidate) => total +
1117
- (candidate.claude?.sidecar_files.filter((sidecar) => sidecar.skipped_reason === "file_read_failed" ||
1118
- sidecar.skipped_reason === "file_too_large").length ?? 0), 0), "candidate");
1119
- }
1120
- function scanIssuePriority(scope) {
1121
- if (scope === "global")
1122
- return 0;
1123
- if (scope === "candidate")
1124
- return 1;
1125
- return 2;
1126
- }
1127
- function retryableCandidateKeys(candidates) {
1128
- return new Set(candidates
1129
- .filter((candidate) => candidate.reason === "file_read_failed" ||
1130
- candidate.reason === "repo_not_on_disk" ||
1131
- (isRawEvidenceUploadableAttributionState(candidate.state, candidate.worktree !== null) &&
1132
- !candidate.worktree) ||
1133
- Boolean(candidate.claude?.main_file_oversized) ||
1134
- (candidate.claude?.sidecars_capped ?? 0) > 0 ||
1135
- Boolean(candidate.claude?.sidecar_files.some((sidecar) => sidecar.skipped_reason === "file_read_failed" ||
1136
- sidecar.skipped_reason === "file_too_large")))
1137
- .map(candidateCursorKey));
1138
- }
1139
- /**
1140
- * A main session file whose only story is "too large to upload under the
1141
- * current cap" (BLI-2727). This mirrors exactly the two branches in
1142
- * `countReadOnlyGuards` that emit the `file_too_large` reason, so a candidate
1143
- * is in this set if and only if it contributed to that scan issue's count —
1144
- * one predicate, no drift between "why the issue fired" and "which candidate
1145
- * caused it". Deterministic and non-retryable: rerunning backfill cannot
1146
- * resolve it (only a larger cap or a smaller file can), so unlike a transient
1147
- * read failure it must never poison completion or a batch's success.
1148
- */
1149
- function oversizedBackfillCandidateKeys(candidates) {
1150
- const keys = new Set();
1151
- for (const candidate of candidates) {
1152
- if ((candidate.source === "claude_code" &&
1153
- candidate.claude?.main_file_oversized) ||
1154
- candidate.byte_size > RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES) {
1155
- keys.add(candidateCursorKey(candidate));
1156
- }
1157
- }
1158
- return keys;
1159
- }
1160
- // --- Scan issue ledger --------------------------------------------------
1161
- //
1162
- // `scan.issues` is the running ledger of everything the scan could not
1163
- // account for cleanly. Every write to it goes through one of the named
1164
- // operations below instead of a bare `.push`/`.sort`, so the ledger's shape
1165
- // (global vs candidate vs selection scope, priority order) has one owner.
1166
- /** Record one issue on the ledger. */
1167
- function addScanIssue(issues, issue) {
1168
- issues.push(issue);
1169
- }
1170
- /** Record one issue only if it actually happened; a zero count is not a finding. */
1171
- function countScanIssue(issues, reason, count, scope) {
1172
- if (count > 0)
1173
- addScanIssue(issues, { reason, count, scope });
1174
- }
1175
- /** Repo discovery could not fully enumerate a root: one global issue per reason. */
1176
- function addRepoDiscoveryIssues(issues, incompleteReasons) {
1177
- for (const reason of incompleteReasons) {
1178
- addScanIssue(issues, {
1179
- reason: `repo_discovery_${reason}`,
1180
- count: 1,
1181
- scope: "global",
1182
- });
1183
- }
1184
- }
1185
- /** Stable read order: global issues first, then candidate, then selection; alphabetical within a scope. */
1186
- function sortScanIssuesByPriority(issues) {
1187
- issues.sort((a, b) => scanIssuePriority(a.scope) - scanIssuePriority(b.scope) ||
1188
- a.reason.localeCompare(b.reason));
1189
- }
1190
- /** The read-only guard pass's two aggregate outcomes, each recorded once if it fired at all. */
1191
- function addReadOnlyGuardIssues(issues, guardCounts) {
1192
- const readFailed = guardCounts.get("file_read_failed");
1193
- if (readFailed) {
1194
- addScanIssue(issues, {
1195
- reason: "candidate_file_read_failed",
1196
- count: readFailed,
1197
- scope: "candidate",
1198
- });
1199
- }
1200
- const tooLarge = guardCounts.get("file_too_large");
1201
- if (tooLarge) {
1202
- addScanIssue(issues, {
1203
- reason: "file_too_large",
1204
- count: tooLarge,
1205
- scope: "candidate",
1206
- });
1207
- }
1208
- }
1209
- // BLI-2727: a deterministic, labeled oversized skip must never poison
1210
- // completion — it is a permanent, non-retryable fact about the file, not an
1211
- // in-flight problem a rerun can fix. `scan.issues`/`retryable_candidate_keys`
1212
- // still carry it (so it's never silently dropped from reporting); every
1213
- // completion-gating computation excludes it explicitly instead, by filtering
1214
- // through `blockingScanIssues` below.
1215
- //
1216
- // Two scan issues describe the exact same oversized-main candidates:
1217
- // `backfillScanIssues` pushes the Claude-specific `claude_main_file_too_large`
1218
- // (from `claude?.main_file_oversized`) and `countReadOnlyGuards` (via
1219
- // `addReadOnlyGuardIssues` above) separately pushes the source-agnostic
1220
- // `file_too_large` (same predicate as `oversizedBackfillCandidateKeys`, so
1221
- // this list can never drift from it). Both must be excluded from
1222
- // completion-gating together.
1223
- const OVERSIZED_SCAN_ISSUE_REASONS = new Set([
1224
- "file_too_large",
1225
- "claude_main_file_too_large",
1226
- ]);
1227
- function blockingScanIssues(issues) {
1228
- return issues.filter((issue) => !OVERSIZED_SCAN_ISSUE_REASONS.has(issue.reason));
1229
- }
1230
- async function countUnreadableClaudeSidecarDirs(candidates) {
1231
- let unreadable = 0;
1232
- for (const candidate of candidates) {
1233
- if (candidate.source !== "claude_code")
1234
- continue;
1235
- const subagentsDir = path.join(path.dirname(candidate.file_path), path.basename(candidate.file_path).replace(/\.jsonl$/i, ""), "subagents");
1236
- try {
1237
- await fs.readdir(subagentsDir);
1238
- }
1239
- catch (error) {
1240
- if (!isMissingFsError(error))
1241
- unreadable += 1;
1242
- }
1243
- }
1244
- return unreadable;
1245
- }
1246
- async function isMissingPath(filePath) {
1247
- try {
1248
- await fs.stat(filePath);
1249
- return false;
1250
- }
1251
- catch (error) {
1252
- return isMissingFsError(error);
1253
- }
1254
- }
1255
- function isMissingFsError(error) {
1256
- if (!error || typeof error !== "object")
1257
- return false;
1258
- const code = error.code;
1259
- return code === "ENOENT" || code === "ENOTDIR";
1260
- }
1261
- async function countReadOnlyGuards(candidates) {
1262
- const counts = new Map();
1263
- const retryableCandidateKeys = new Set();
1264
- // Aggregated: this runs over the whole archived history, so a per-candidate
1265
- // line could be thousands. The count already travels; the reason did not
1266
- // (BLI-3238).
1267
- let firstReadFailure = null;
1268
- for (const candidate of candidates) {
1269
- if (candidate.reason === "repo_not_on_disk") {
1270
- increment(counts, "repo_not_on_disk");
1271
- retryableCandidateKeys.add(candidateCursorKey(candidate));
1272
- }
1273
- if (candidate.source === "claude_code" && candidate.claude?.main_file_oversized) {
1274
- increment(counts, "file_too_large");
1275
- retryableCandidateKeys.add(candidateCursorKey(candidate));
1276
- continue;
1277
- }
1278
- if (candidate.byte_size > RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES) {
1279
- increment(counts, "file_too_large");
1280
- retryableCandidateKeys.add(candidateCursorKey(candidate));
1281
- continue;
1282
- }
1283
- if (!isRawEvidenceUploadableAttributionState(candidate.state, candidate.worktree !== null))
1284
- continue;
1285
- try {
1286
- await fs.readFile(candidate.file_path);
1287
- }
1288
- catch (error) {
1289
- increment(counts, "file_read_failed");
1290
- firstReadFailure ??= describeError(error);
1291
- retryableCandidateKeys.add(candidateCursorKey(candidate));
1292
- }
1293
- }
1294
- const readFailedCount = counts.get("file_read_failed") ?? 0;
1295
- if (readFailedCount > 0) {
1296
- console.error("[cockpit-backfill] archived sessions could not be read", JSON.stringify({
1297
- reason: "file_read_failed",
1298
- read_failed_count: readFailedCount,
1299
- candidate_count: candidates.length,
1300
- ...firstReadFailure,
1301
- }));
1302
- }
1303
- return {
1304
- counts,
1305
- retryable_candidate_keys: retryableCandidateKeys,
1306
- };
1307
- }
1308
- function reasonCountsFor(candidates, guardCounts, scanIssues) {
1309
- const counts = new Map();
1310
- for (const candidate of candidates)
1311
- increment(counts, candidate.reason);
1312
- for (const [reason, count] of guardCounts) {
1313
- counts.set(reason, Math.max(counts.get(reason) ?? 0, count));
1314
- }
1315
- for (const issue of scanIssues) {
1316
- counts.set(issue.reason, Math.max(counts.get(issue.reason) ?? 0, issue.count));
1317
- }
1318
- for (const required of ["file_too_large", "repo_not_on_disk"]) {
1319
- counts.set(required, counts.get(required) ?? 0);
1320
- }
1321
- return [...counts.entries()]
1322
- .sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))
1323
- .map(([reason, count]) => ({
1324
- reason,
1325
- count,
1326
- ...reasonClassification(reason),
1327
- }));
1328
- }
1329
- /**
1330
- * The skip-reason table printed under every backfill run. These strings are
1331
- * what a person reads when coverage is short, so each note says what would
1332
- * have to change, not merely that something went wrong. Reason labels here
1333
- * must match the ones the scan and the guards emit verbatim.
1334
- */
1335
- const REASON_VERDICTS = new Map([
1336
- [
1337
- "secret_like_content_guard",
1338
- {
1339
- classification: "retryable",
1340
- note: "historical guard result; collector now masks and retries",
1341
- },
1342
- ],
1343
- [
1344
- "secret_redaction_failed",
1345
- {
1346
- classification: "retryable",
1347
- note: "historical guard result; collector now masks and retries",
1348
- },
1349
- ],
1350
- [
1351
- "file_too_large",
1352
- {
1353
- classification: "retryable",
1354
- note: "until the evidence file cap is raised",
1355
- },
1356
- ],
1357
- [
1358
- "repo_not_on_disk",
1359
- { classification: "retryable", note: "repo must exist on disk" },
1360
- ],
1361
- [
1362
- "cwd_not_a_repo",
1363
- {
1364
- classification: "permanent",
1365
- note: "cwd exists but is not a repo or folder workspace",
1366
- },
1367
- ],
1368
- [
1369
- "multiple_transcript_origins",
1370
- {
1371
- classification: "permanent",
1372
- note: "multiple transcript origins; attribution is ambiguous",
1373
- },
1374
- ],
1375
- [
1376
- "single_repo_folder_fallback",
1377
- {
1378
- classification: "permanent",
1379
- note: "uploadable single-repo folder workspace fallback",
1380
- },
1381
- ],
1382
- [
1383
- "multi_repo_folder_workspace",
1384
- {
1385
- classification: "permanent",
1386
- note: "uploadable multi-repo folder workspace fallback",
1387
- },
1388
- ],
1389
- ]);
1390
- const DEFERRED_REASON_VERDICT = {
1391
- classification: "retryable",
1392
- note: "rerun cockpit backfill to continue",
1393
- };
1394
- const UNKNOWN_REASON_VERDICT = {
1395
- classification: "retryable",
1396
- note: "rerun after fixing source or collector state",
1397
- };
1398
- function reasonClassification(reason) {
1399
- const known = REASON_VERDICTS.get(reason);
1400
- if (known)
1401
- return known;
1402
- // A budget deferral is the one family, rather than one label: whichever
1403
- // budget ran out, the remedy is the same rerun.
1404
- if (reason.startsWith("deferred_"))
1405
- return DEFERRED_REASON_VERDICT;
1406
- return UNKNOWN_REASON_VERDICT;
1407
- }
1408
- function uploadableCandidates(candidates) {
1409
- return candidates.filter((candidate) => isRawEvidenceUploadableAttributionState(candidate.state, candidate.worktree !== null) &&
1410
- candidate.worktree);
1411
- }
1412
- function buildBackfillBatches(candidates) {
1413
- const byWorktree = new Map();
1414
- for (const candidate of candidates) {
1415
- if (!candidate.worktree)
1416
- continue;
1417
- const key = candidate.worktree.worktree_fingerprint;
1418
- byWorktree.set(key, [...(byWorktree.get(key) ?? []), candidate]);
1419
- }
1420
- const batches = [];
1421
- for (const group of byWorktree.values()) {
1422
- const worktree = group[0]?.worktree;
1423
- if (!worktree)
1424
- continue;
1425
- for (let offset = 0; offset < group.length; offset += BACKFILL_UPLOAD_BATCH_SESSIONS) {
1426
- batches.push({
1427
- worktree,
1428
- candidates: group.slice(offset, offset + BACKFILL_UPLOAD_BATCH_SESSIONS),
1429
- });
1430
- }
1431
- }
1432
- return batches;
1433
- }
1434
- async function ensureBackfillReportContext(options) {
1435
- const candidateWorktree = options.candidates.find((candidate) => candidate.worktree)?.worktree;
1436
- const representative = candidateWorktree ?? options.worktrees[0] ?? null;
1437
- // A deleted repo or an approved folder workspace can legitimately produce
1438
- // report-only rows with no current git worktree. Use the exact approved root
1439
- // as a synthetic local context instead of falling through to process.cwd().
1440
- const repoRoot = representative?.repo_root ??
1441
- options.collectionRoots[0] ??
1442
- process.cwd();
1443
- try {
1444
- await readLocalWorkContextForRepo(options.paths, repoRoot);
1445
- }
1446
- catch {
1447
- // Reading it can legitimately fail — there is no context yet, which is
1448
- // precisely why the next line creates one. That read is a probe and stays
1449
- // silent; the CREATE is the branch that has to speak (BLI-3238).
1450
- await startLocalWorkContext({
1451
- homeDir: options.homeDir,
1452
- repoRoot,
1453
- branch: representative?.branch,
1454
- }).catch((error) => {
1455
- // Context creation is best-effort here: postCodexSessionReport converts
1456
- // a remaining local-context failure into a retryable report reason. But
1457
- // that reason is `collector_not_ready`, which points the operator at
1458
- // setup rather than at whatever actually failed here.
1459
- console.error("[cockpit-backfill] could not start a local work context for the batch", JSON.stringify({
1460
- reason: "work_context_start_failed",
1461
- ...describeError(error),
1462
- }));
1463
- });
1464
- }
1465
- return { repoRoot };
1466
- }
1467
- /**
1468
- * Hands one batch to the ordinary sync path. A batch whose repo has no local
1469
- * work context yet is not a failure — backfill routinely visits repos this
1470
- * machine has never synced — so that one blocker is answered by creating the
1471
- * context and retrying; every other error propagates.
1472
- */
1473
- async function syncBackfillBatch(options) {
1474
- const syncOptions = backfillBatchSyncOptions(options);
1475
- try {
1476
- return await syncLocalAmbientEnvelope(syncOptions);
1477
- }
1478
- catch (error) {
1479
- if (error instanceof LocalUploadBlockedError &&
1480
- error.blocker === "missing_context") {
1481
- await startLocalWorkContext({
1482
- homeDir: options.command.homeDir,
1483
- repoRoot: options.batch.worktree.repo_root,
1484
- branch: options.batch.worktree.branch,
1485
- });
1486
- return await syncLocalAmbientEnvelope(syncOptions);
1487
- }
1488
- throw error;
1489
- }
1490
- }
1491
- /** The batch translated into the live sync path's envelope, session files and all. */
1492
- function backfillBatchSyncOptions(options) {
1493
- return {
1494
- homeDir: options.command.homeDir,
1495
- repoRoot: options.batch.worktree.repo_root,
1496
- worktreeInventory: worktreeInventoryForRepo(options.batch.worktree, options.worktrees),
1497
- codexSessionFiles: options.batch.candidates
1498
- .filter((candidate) => candidate.source === "codex")
1499
- .map((candidate) => ({
1500
- local_path: candidate.file_path,
1501
- codex_session_id: candidate.session_id,
1502
- })),
1503
- codexAttributionScan: options.codexAttribution ?? undefined,
1504
- claudeSessionFiles: options.batch.candidates
1505
- .filter((candidate) => candidate.source === "claude_code")
1506
- .map((candidate) => ({
1507
- local_path: candidate.file_path,
1508
- claude_session_id: candidate.session_id,
1509
- main_file_oversized: Boolean(candidate.claude?.main_file_oversized),
1510
- skip_main: false,
1511
- sidecar_files: candidate.claude?.sidecar_files
1512
- .filter((sidecar) => !sidecar.skipped_reason)
1513
- .map((sidecar) => ({ local_path: sidecar.local_path })) ?? [],
1514
- })),
1515
- claudeAttributionScan: options.claudeAttribution ?? undefined,
1516
- rawEvidenceBudget: options.rawEvidenceBudget,
1517
- // `cockpit backfill` is only ever an operator asking — by hand, through
1518
- // onboarding, through doctor, or by running the retry command Cockpit
1519
- // printed. The delivery-backoff window is the scheduler's cadence, so it
1520
- // does not gate this pass: honouring it here made the retry that follows a
1521
- // failed commit a 15-minute no-op that reported
1522
- // `durable_session_pointer_missing` and never named the hold (BLI-3118).
1523
- evidenceDeliveryMode: "operator_retry",
1524
- fetch: options.fetchImpl,
1525
- };
1526
- }
1527
- /**
1528
- * One attribution row per session this run saw, uploaded or not — the only
1529
- * record the server ever gets of a session the collector could not upload.
1530
- * Exported for the BLI-3272 regression test; not part of the CLI surface.
1531
- */
1532
- export function buildBackfillSessionReport(options) {
1533
- const uploads = indexMainTranscriptUploads(options.syncResults);
1534
- const bestBySession = bestCandidatePerSession(options.candidates);
1535
- return [...bestBySession.values()].map((candidate) => sessionAttributionRow(candidate, uploads, options.now));
1536
- }
1537
- function indexMainTranscriptUploads(syncResults) {
1538
- const bySourceAndSession = new Map();
1539
- const noUploadReasonBySessionId = new Map();
1540
- for (const sync of syncResults) {
1541
- if (sync.status !== "uploaded")
1542
- continue;
1543
- for (const outcome of sync.raw_evidence_outcomes) {
1544
- if (!outcome.codex_session_id)
1545
- continue;
1546
- const source = outcome.kind === "claude_jsonl"
1547
- ? "claude_code"
1548
- : outcome.kind === "codex_jsonl"
1549
- ? "codex"
1550
- : null;
1551
- if (!source)
1552
- continue;
1553
- if (!outcome.raw_evidence_pointer_id) {
1554
- if (outcome.reason) {
1555
- noUploadReasonBySessionId.set(outcome.codex_session_id, outcome.reason);
1556
- }
1557
- continue;
1558
- }
1559
- bySourceAndSession.set(`${source}:${outcome.codex_session_id}`, {
1560
- upload_state: outcome.upload_state,
1561
- raw_evidence_pointer_id: outcome.raw_evidence_pointer_id,
1562
- reason: outcome.reason,
1563
- });
1564
- }
1565
- }
1566
- return { bySourceAndSession, noUploadReasonBySessionId };
1567
- }
1568
- /**
1569
- * One row per session, not per file: the same session can be scanned from
1570
- * several paths, so the best-attributed sighting wins and the most recent one
1571
- * breaks a tie.
1572
- */
1573
- function bestCandidatePerSession(candidates) {
1574
- const bestByKey = new Map();
1575
- for (const candidate of candidates) {
1576
- const key = `${candidate.source}:${candidate.session_id}`;
1577
- const existing = bestByKey.get(key);
1578
- if (!existing || rank(candidate.state) > rank(existing.state)) {
1579
- bestByKey.set(key, candidate);
1580
- }
1581
- else if (existing &&
1582
- rank(candidate.state) === rank(existing.state) &&
1583
- candidate.session_file_mtime_ms > existing.session_file_mtime_ms) {
1584
- bestByKey.set(key, candidate);
1585
- }
1586
- }
1587
- return bestByKey;
1588
- }
1589
- function sessionAttributionRow(candidate, uploads, now) {
1590
- return {
1591
- codex_session_id: candidate.session_id,
1592
- source: candidate.source,
1593
- observed_at: now.toISOString(),
1594
- attribution_state: candidate.state,
1595
- attribution_reason: candidate.reason,
1596
- attribution_score: candidate.attribution_score,
1597
- path_score: candidate.path_score,
1598
- signals: candidate.signals,
1599
- ...(candidate.content_hash_sha256
1600
- ? { session_file_hash_sha256: candidate.content_hash_sha256 }
1601
- : {}),
1602
- session_file_byte_size: candidate.byte_size,
1603
- session_file_mtime: candidate.session_file_mtime,
1604
- ...(candidate.worktree
1605
- ? {
1606
- repo_fingerprint: candidate.worktree.repo_fingerprint,
1607
- worktree_fingerprint: candidate.worktree.worktree_fingerprint,
1608
- repo_label: candidate.worktree.repo_label,
1609
- branch: candidate.worktree.branch,
1610
- }
1611
- : {}),
1612
- ...(candidate.cwd_basename ? { cwd_basename: candidate.cwd_basename } : {}),
1613
- ...(candidate.cwd_hash ? { cwd_hash: candidate.cwd_hash } : {}),
1614
- ...sessionUploadFields(candidate, uploads),
1615
- };
1616
- }
1617
- /**
1618
- * The upload half of a report row. Every branch names an outcome: a pointer
1619
- * when one exists, and otherwise WHY there is none — BLI-2107 for an
1620
- * attributed session and BLI-3272 for a refused one. Backfill is the path that
1621
- * revisits old sessions, so a silent branch here would keep rewriting the very
1622
- * NULL/NULL rows those tickets found.
1623
- */
1624
- function sessionUploadFields(candidate, uploads) {
1625
- const upload = uploads.bySourceAndSession.get(`${candidate.source}:${candidate.session_id}`);
1626
- if (upload) {
1627
- return {
1628
- raw_evidence_pointer_id: upload.raw_evidence_pointer_id,
1629
- upload_state: upload.upload_state,
1630
- ...(upload.upload_state === "upload_failed"
1631
- ? { upload_reason: upload.reason ?? NO_UPLOAD_ATTEMPT_RECORDED }
1632
- : {}),
1633
- };
1634
- }
1635
- const observedReason = uploads.noUploadReasonBySessionId.get(candidate.session_id);
1636
- const wasUploadable = isRawEvidenceUploadableAttributionState(candidate.state, candidate.worktree !== null);
1637
- return {
1638
- upload_state: "not_uploaded",
1639
- upload_reason: observedReason ??
1640
- (wasUploadable
1641
- ? NO_UPLOAD_ATTEMPT_RECORDED
1642
- : notUploadableAttributionStateReason(candidate.reason)),
1643
- };
1644
- }
1645
- function rank(state) {
1646
- switch (state) {
1647
- case "attributed":
1648
- return 5;
1649
- case "attributed_fallback":
1650
- return 4;
1651
- case "ambiguous":
1652
- return 3;
1653
- case "unattributed":
1654
- return 2;
1655
- case "skipped":
1656
- return 1;
1657
- default:
1658
- return 0;
1659
- }
1660
- }
1661
- /**
1662
- * Which candidates in this batch are now safe to leave behind forever: a
1663
- * session counts as durable only when its main transcript AND every sidecar
1664
- * we expected earned a pointer, and nothing in it failed. This set is what the
1665
- * cursor advances through, so an over-generous answer here loses history
1666
- * permanently.
1667
- */
1668
- function durableBackfillCandidateKeys(batch, sync) {
1669
- const durable = new Set();
1670
- if (sync.status !== "uploaded")
1671
- return durable;
1672
- if (deferredEvidenceCount(sync) > 0) {
1673
- // Deferred outcomes have no per-file identity. Advancing any candidate in
1674
- // this batch could therefore strand the deferred main transcript.
1675
- return durable;
1676
- }
1677
- const outcomesBySession = summarizeEvidenceOutcomesBySession(sync);
1678
- for (const candidate of batch.candidates) {
1679
- const summary = outcomesBySession.get(`${candidate.source}:${candidate.session_id}`);
1680
- if (summary &&
1681
- !summary.failed &&
1682
- summary.durableMainCount > 0 &&
1683
- summary.durableSidecarCount >= expectedSidecarCount(candidate)) {
1684
- durable.add(candidateCursorKey(candidate));
1685
- }
1686
- }
1687
- return durable;
1688
- }
1689
- function summarizeEvidenceOutcomesBySession(sync) {
1690
- const outcomesBySession = new Map();
1691
- for (const outcome of sync.raw_evidence_outcomes) {
1692
- const source = backfillSourceForEvidenceKind(outcome.kind);
1693
- if (!source || !outcome.codex_session_id)
1694
- continue;
1695
- const key = `${source}:${outcome.codex_session_id}`;
1696
- const summary = outcomesBySession.get(key) ?? {
1697
- durableMainCount: 0,
1698
- durableSidecarCount: 0,
1699
- failed: false,
1700
- };
1701
- if (outcome.upload_state === "upload_failed") {
1702
- summary.failed = true;
1703
- }
1704
- else if (outcome.raw_evidence_pointer_id &&
1705
- (outcome.upload_state === "uploaded" ||
1706
- outcome.upload_state === "reused_existing")) {
1707
- if (outcome.kind === "codex_jsonl" || outcome.kind === "claude_jsonl") {
1708
- summary.durableMainCount += 1;
1709
- }
1710
- else if (outcome.kind === "claude_jsonl_sidecar") {
1711
- summary.durableSidecarCount += 1;
1712
- }
1713
- }
1714
- outcomesBySession.set(key, summary);
1715
- }
1716
- return outcomesBySession;
1717
- }
1718
- /** Sidecars this session owes a pointer for; one already skipped at scan time is not owed. */
1719
- function expectedSidecarCount(candidate) {
1720
- if (candidate.source !== "claude_code")
1721
- return 0;
1722
- return (candidate.claude?.sidecar_files.filter((sidecar) => !sidecar.skipped_reason)
1723
- .length ?? 0);
1724
- }
1725
- /**
1726
- * Writes the pointer a session earned back into the live raw-evidence cursor,
1727
- * so the next scheduled sync knows it is already delivered. Backfill and sync
1728
- * share these cursors; the caller holds the collection lock for exactly this.
1729
- */
1730
- async function recordBackfillDurableSessionPointers(options) {
1731
- const durableObjectBySession = indexDurableMainObjectKeys(options.syncResults);
1732
- let recordedCount = 0;
1733
- for (const source of ["codex", "claude_code"]) {
1734
- const filename = source === "claude_code" ? CLAUDE_CURSOR_FILENAME : undefined;
1735
- const cursor = await readRawEvidenceCursor(options.paths, { filename });
1736
- let changed = false;
1737
- for (const candidate of options.candidates) {
1738
- if (candidate.source !== source)
1739
- continue;
1740
- const objectKey = durableObjectBySession.get(`${source}:${candidate.session_id}`);
1741
- const prior = cursor.sessions[candidate.session_id];
1742
- // No pointer, no prior entry, or already pointed: nothing this pass owes.
1743
- if (!objectKey || !prior || prior.uploaded_object_key)
1744
- continue;
1745
- cursor.sessions[candidate.session_id] = deliveredCursorEntry(prior, candidate, objectKey, options.now);
1746
- changed = true;
1747
- recordedCount += 1;
1748
- }
1749
- if (changed) {
1750
- await writeRawEvidenceCursor(options.paths, cursor, {
1751
- filename,
1752
- sessionsOnly: source === "claude_code",
1753
- });
1754
- }
1755
- }
1756
- if (recordedCount > 0) {
1757
- console.error("[backfill] terminal session pointers recorded", JSON.stringify({ count: recordedCount }));
1758
- }
1759
- }
1760
- /** Storage keys for main transcripts that actually landed, keyed by source and session. */
1761
- function indexDurableMainObjectKeys(syncResults) {
1762
- const durableObjectBySession = new Map();
1763
- for (const sync of syncResults) {
1764
- for (const outcome of sync.raw_evidence_outcomes) {
1765
- if (!outcome.codex_session_id ||
1766
- (outcome.kind !== "codex_jsonl" && outcome.kind !== "claude_jsonl") ||
1767
- (outcome.upload_state !== "uploaded" &&
1768
- outcome.upload_state !== "reused_existing") ||
1769
- !outcome.object_key) {
1770
- continue;
1771
- }
1772
- const source = outcome.kind === "codex_jsonl" ? "codex" : "claude_code";
1773
- durableObjectBySession.set(`${source}:${outcome.codex_session_id}`, outcome.object_key);
1774
- }
1775
- }
1776
- return durableObjectBySession;
1777
- }
1778
- /** The cursor entry for a session backfill has just delivered in full. */
1779
- function deliveredCursorEntry(prior, candidate, objectKey, now) {
1780
- return {
1781
- ...prior,
1782
- file_hash_sha256: candidate.content_hash_sha256,
1783
- file_mtime_ms: candidate.session_file_mtime_ms,
1784
- byte_size: candidate.byte_size,
1785
- // The whole file was delivered, so the incremental reader starts at its end.
1786
- byte_offset: candidate.byte_size,
1787
- state: candidate.state,
1788
- reason: candidate.reason,
1789
- worktree_fingerprint: candidate.worktree?.worktree_fingerprint ?? null,
1790
- uploaded_object_key: objectKey,
1791
- uploaded_at: now.toISOString(),
1792
- uploaded_byte_size: candidate.byte_size,
1793
- last_seen_at: now.toISOString(),
1794
- };
1795
- }
1796
- function countSessionUploadFailures(sync) {
1797
- return sync.raw_evidence_outcomes.filter((outcome) => Boolean(outcome.codex_session_id) &&
1798
- outcome.upload_state === "upload_failed" &&
1799
- Boolean(backfillSourceForEvidenceKind(outcome.kind))).length;
1800
- }
1801
- function backfillSourceForEvidenceKind(kind) {
1802
- if (kind === "codex_jsonl" || kind === "codex_image_attachment") {
1803
- return "codex";
1804
- }
1805
- if (kind === "claude_jsonl" ||
1806
- kind === "claude_jsonl_sidecar" ||
1807
- kind === "claude_image_attachment") {
1808
- return "claude_code";
1809
- }
1810
- return null;
1811
- }
1812
- function advanceBackfillCursorThroughResolvedPrefix(options) {
1813
- if (!options.discoveryComplete)
1814
- return false;
1815
- const observations = [];
1816
- for (const source of ["codex", "claude_code"]) {
1817
- const remaining = options.candidates
1818
- .filter((candidate) => candidate.source === source &&
1819
- isAfterCursor(candidate, options.cursor))
1820
- .sort(compareBackfillCandidates);
1821
- for (const candidate of remaining) {
1822
- const key = candidateCursorKey(candidate);
1823
- if (options.retryableCandidateKeys.has(key))
1824
- break;
1825
- const resolved = isRawEvidenceUploadableAttributionState(candidate.state, candidate.worktree !== null)
1826
- ? options.durableCandidateKeys.has(key)
1827
- : true;
1828
- if (!resolved)
1829
- break;
1830
- observations.push({
1831
- source: candidate.source,
1832
- cursor_key: key,
1833
- state: candidate.state,
1834
- reason: candidate.reason,
1835
- session_file_mtime_ms: candidate.session_file_mtime_ms,
1836
- session_file_mtime: candidate.session_file_mtime,
1837
- });
1838
- }
1839
- }
1840
- if (observations.length === 0)
1841
- return false;
1842
- recordBackfillCursorObservations(options.cursor, observations, options.now);
1843
- return true;
1844
- }
1845
- function worktreeInventoryForRepo(current, worktrees) {
1846
- return worktrees
1847
- .filter((worktree) => worktree.repo_fingerprint
1848
- ? worktree.repo_fingerprint === current.repo_fingerprint
1849
- : worktree.repo_label === current.repo_label)
1850
- .map((worktree) => ({
1851
- repo: worktree.repo_root,
1852
- repo_label: worktree.repo_label,
1853
- repo_fingerprint: worktree.repo_fingerprint,
1854
- repo_origin_url: worktree.repo_origin_url ?? undefined,
1855
- head_sha: worktree.head_sha ?? undefined,
1856
- worktree_label: worktree.worktree_label,
1857
- worktree_fingerprint: worktree.worktree_fingerprint,
1858
- worktree_is_primary: worktree.worktree_is_primary,
1859
- branch: worktree.branch,
1860
- }));
1861
- }
1862
- function baseBackfillResult(command, options) {
1863
- const states = countBy(options.scan.candidates, (candidate) => candidate.state);
1864
- const uploadable = uploadableCandidates(options.scan.candidates);
1865
- const perSource = {
1866
- codex: sourceCounts("codex", options.scan),
1867
- claude_code: sourceCounts("claude_code", options.scan),
1868
- };
1869
- return {
1870
- status: "complete",
1871
- dry_run: command.dryRun,
1872
- dashboard_url: options.dashboardUrl,
1873
- verify_url: `${options.dashboardUrl}/my-work`,
1874
- sources: options.sources,
1875
- window: options.window,
1876
- counts: {
1877
- total: options.scan.candidates.length,
1878
- uploadable: uploadable.length,
1879
- backfilled: 0,
1880
- skipped: options.scan.candidates.length - uploadable.length,
1881
- failed: 0,
1882
- deferred: 0,
1883
- remaining: 0,
1884
- states,
1885
- reasons: options.reasonCounts,
1886
- per_source: perSource,
1887
- },
1888
- batches: {
1889
- total: Math.ceil(uploadable.length / BACKFILL_UPLOAD_BATCH_SESSIONS),
1890
- completed: 0,
1891
- failed: 0,
1892
- },
1893
- resume_cursor: options.cursor,
1894
- retry_command: backfillRetryCommand(command),
1895
- report: emptyReport("not_posted"),
1896
- server_acknowledged: {
1897
- codex_session_report_recorded_count: 0,
1898
- raw_evidence_uploaded_object_count: 0,
1899
- raw_evidence_uploaded_chunk_count: 0,
1900
- },
1901
- };
1902
- }
1903
- function blockedBackfillResult(command, options) {
1904
- return {
1905
- status: "blocked",
1906
- dry_run: command.dryRun,
1907
- dashboard_url: options.dashboardUrl,
1908
- verify_url: `${options.dashboardUrl}/my-work`,
1909
- sources: selectedSources(command.source),
1910
- window: {
1911
- mode: command.all ? "all" : "since_days",
1912
- since_days: command.sinceDays ?? null,
1913
- started_at: options.now.toISOString(),
1914
- paired_at: options.now.toISOString(),
1915
- since_minutes: 0,
1916
- },
1917
- counts: {
1918
- total: 0,
1919
- uploadable: 0,
1920
- backfilled: 0,
1921
- skipped: 0,
1922
- failed: 0,
1923
- deferred: 0,
1924
- remaining: 0,
1925
- states: {},
1926
- reasons: [],
1927
- per_source: {
1928
- codex: emptySourceCounts(),
1929
- claude_code: emptySourceCounts(),
1930
- },
1931
- },
1932
- batches: { total: 0, completed: 0, failed: 0 },
1933
- resume_cursor: options.cursor,
1934
- retry_command: backfillRetryCommand(command),
1935
- report: emptyReport("not_posted"),
1936
- server_acknowledged: {
1937
- codex_session_report_recorded_count: 0,
1938
- raw_evidence_uploaded_object_count: 0,
1939
- raw_evidence_uploaded_chunk_count: 0,
1940
- },
1941
- failure_reason: options.reason,
1942
- };
1943
- }
1944
- function sourceCounts(source, scan) {
1945
- const candidates = scan.candidates.filter((candidate) => candidate.source === source);
1946
- const scanned = source === "codex"
1947
- ? (scan.codexAttribution?.scanned_file_count ?? 0)
1948
- : (scan.claudeAttribution?.scanned_session_count ?? 0);
1949
- return {
1950
- scanned,
1951
- selected: candidates.length,
1952
- uploadable: uploadableCandidates(candidates).length,
1953
- states: countBy(candidates, (candidate) => candidate.state),
1954
- };
1955
- }
1956
- function emptySourceCounts() {
1957
- return { scanned: 0, selected: 0, uploadable: 0, states: {} };
1958
- }
1959
- function emptyReport(reason) {
1960
- return {
1961
- posted: false,
1962
- reason,
1963
- chunk_count: 0,
1964
- recorded_count: 0,
1965
- failed_count: 0,
1966
- chunks: [],
1967
- };
1968
- }
1969
- function countBy(items, keyOf) {
1970
- const counts = {};
1971
- for (const item of items) {
1972
- const key = keyOf(item);
1973
- counts[key] = (counts[key] ?? 0) + 1;
1974
- }
1975
- return counts;
1976
- }
1977
- function increment(counts, key) {
1978
- counts.set(key, (counts.get(key) ?? 0) + 1);
1979
- }
1980
- function writeDryRunSummary(io, options) {
1981
- const uploadable = uploadableCandidates(options.candidates);
1982
- writeLine(io.stdout, `${options.dryRunOnly ? "DRY-RUN" : "Review"}: ${options.candidates.length} session(s), ${uploadable.length} uploadable, ${options.candidates.length - uploadable.length} skipped.`);
1983
- writeReasonTable(io, options.reasonCounts);
1984
- if (options.dryRunOnly) {
1985
- writeLine(io.stdout, "DRY-RUN: wrote nothing (no cursor, marker, report, or upload).");
1986
- }
1987
- writeLine(io.stdout, `Verify after upload: ${options.dashboardUrl}/my-work`);
1988
- }
1989
- function writeHumanBackfillResult(result, io) {
1990
- if (result.status === "complete") {
1991
- writeLine(io.stdout, `PASS: ${result.counts.backfilled} backfilled, ${result.counts.skipped} skipped (table). Verify: ${result.verify_url}`);
1992
- writeReasonTable(io, result.counts.reasons);
1993
- return;
1994
- }
1995
- if (result.status === "partial") {
1996
- const at = result.blocked_at;
1997
- if (at) {
1998
- writeLine(io.stderr, `BLOCKED: ${at.what} at batch ${at.batch_index}/${at.batch_total}, ${at.done}/${at.total} done`);
1999
- }
2000
- writeLine(io.stderr, `Failure: ${result.failure_reason ?? "partial_backfill"}`);
2001
- writeLine(io.stderr, `Retry: ${result.retry_command}`);
2002
- writeLine(io.stderr, `Stopped: ${result.counts.remaining} remaining — rerun cockpit backfill to continue`);
2003
- writeReasonTable(io, result.counts.reasons);
2004
- return;
2005
- }
2006
- const at = result.blocked_at;
2007
- if (at) {
2008
- writeLine(io.stderr, `BLOCKED: ${at.what} at batch ${at.batch_index}/${at.batch_total}, ${at.done}/${at.total} done`);
2009
- }
2010
- else {
2011
- writeLine(io.stderr, "BLOCKED: backfill could not run.");
2012
- }
2013
- writeLine(io.stderr, `Failure: ${result.failure_reason ?? "no_sessions"}`);
2014
- writeLine(io.stderr, `Retry: ${result.retry_command}`);
2015
- }
2016
- function writeReasonTable(io, reasons) {
2017
- if (reasons.length === 0)
2018
- return;
2019
- writeLine(io.stdout, "Skip/reason table:");
2020
- for (const reason of reasons) {
2021
- writeLine(io.stdout, `- ${reason.reason}: ${reason.count} (${reason.classification}; ${reason.note})`);
2022
- }
2023
- }
2024
- async function confirmAllBackfill(io) {
2025
- const answer = await readLine(io, "Proceed with --all backfill upload? [y/N] ");
2026
- return answer.trim().toLowerCase() === "y" || answer.trim().toLowerCase() === "yes";
2027
- }
2028
- async function readLine(io, prompt) {
2029
- io.stdout.write(prompt);
2030
- io.stdin.setEncoding("utf8");
2031
- return new Promise((resolve) => {
2032
- const onData = (chunk) => {
2033
- io.stdin.removeListener("data", onData);
2034
- io.stdin.pause();
2035
- resolve(chunk);
2036
- };
2037
- io.stdin.resume();
2038
- io.stdin.on("data", onData);
2039
- });
2040
- }
2041
- function isInteractiveStdin(io) {
2042
- return Boolean(io.stdin.isTTY);
2043
- }
2044
- function parseRequiredDate(value, label) {
2045
- const date = new Date(value);
2046
- if (!Number.isFinite(date.getTime())) {
2047
- throw new Error(`Collector session ${label} is invalid.`);
2048
- }
2049
- return date;
2050
- }
2051
- function normalizeDashboardUrl(value) {
2052
- return value.trim().replace(/\/+$/, "");
2053
- }
2054
- function writeLine(stream, text) {
2055
- stream.write(`${text}\n`);
2056
- }
2057
- function yieldToEventLoop() {
2058
- return new Promise((resolve) => setTimeout(resolve, 0));
2059
139
  }