@bli-cockpit/cli 0.2.157 → 0.2.159

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -167,6 +167,7 @@ async function attributeOneSession(file, worktrees, collectionRoots) {
167
167
  file_path: file.file,
168
168
  file_name: fileName,
169
169
  codex_session_id: sessionIdFromFileName(fileName) ?? shortHash(file.file),
170
+ session_id_source: sessionIdFromFileName(fileName) ? "file_name" : "path_hash",
170
171
  cwd_basename: null,
171
172
  cwd_hash: null,
172
173
  session_file_mtime: new Date(file.mtimeMs).toISOString(),
@@ -197,6 +198,7 @@ async function attributeOneSession(file, worktrees, collectionRoots) {
197
198
  const metaSessionId = sanitizeSessionId(signals.session_ids[0]);
198
199
  if (metaSessionId) {
199
200
  base.codex_session_id = metaSessionId;
201
+ base.session_id_source = "session_meta";
200
202
  }
201
203
  const primaryCwd = signals.cwds[0] ?? null;
202
204
  if (primaryCwd) {
@@ -10,6 +10,7 @@
10
10
  */
11
11
  import { recordBackfillCursorObservations, } from "../cursors/backfill-cursor.js";
12
12
  import { CLAUDE_CURSOR_FILENAME, readRawEvidenceCursor, writeRawEvidenceCursor, } from "../cursors/raw-evidence-cursor.js";
13
+ import { isRedactionStubObjectKey, markCursorEntryStubbed } from "../cursors/raw-evidence-stub-entry.js";
13
14
  import { isRawEvidenceUploadableAttributionState } from "../raw-evidence-attribution-policy.js";
14
15
  import { candidateCursorKey, compareBackfillCandidates, isAfterCursor, } from "./backfill-candidates.js";
15
16
  import { indexDurableMainObjectKeys } from "./backfill-evidence-outcomes.js";
@@ -34,10 +35,11 @@ export async function recordBackfillDurableSessionPointers(options) {
34
35
  // No pointer, no prior entry, or already pointed: nothing this pass owes.
35
36
  // BLI-11102: a pointer the server named a redaction stub is not a
36
37
  // delivery, so a real pointer replaces it and the mark clears.
37
- const replacesStub = Boolean(prior?.stub_reoffer) && prior?.uploaded_object_key !== objectKey;
38
+ // BLI-11119: an entry the cursor itself knows is `stubbed` counts too.
39
+ const replacesStub = Boolean(prior?.stub_reoffer || prior?.stubbed) && prior?.uploaded_object_key !== objectKey;
38
40
  if (!objectKey || !prior || (prior.uploaded_object_key && !replacesStub))
39
41
  continue;
40
- if (replacesStub)
42
+ if (replacesStub && !isRedactionStubObjectKey(objectKey))
41
43
  stubsReplaced += 1;
42
44
  cursor.sessions[candidate.session_id] = deliveredCursorEntry(prior, candidate, objectKey, options.now);
43
45
  changed = true;
@@ -71,8 +73,15 @@ function deliveredCursorEntry(prior, candidate, objectKey, now) {
71
73
  uploaded_byte_size: candidate.byte_size,
72
74
  last_seen_at: now.toISOString(),
73
75
  };
76
+ // BLI-11119: a re-offer that stubbed again earned another marker, not the
77
+ // transcript. The entry says so and keeps its mark for the next pass.
78
+ if (isRedactionStubObjectKey(objectKey)) {
79
+ markCursorEntryStubbed(entry);
80
+ return entry;
81
+ }
74
82
  // The mark says "this pointer is a stub"; the pointer written above is not.
75
83
  delete entry.stub_reoffer;
84
+ delete entry.stubbed;
76
85
  return entry;
77
86
  }
78
87
  export function advanceBackfillCursorThroughResolvedPrefix(options) {
@@ -21,6 +21,7 @@ import { labelOutsideRootCandidates, outsideRootsWorkspace, } from "./backfill-o
21
21
  import { reasonCountsFor } from "./backfill-reasons.js";
22
22
  import { blockedBackfillResult } from "./backfill-result.js";
23
23
  import { emptyStubReofferPlan, loadStubReofferPlan, reopenStubbedSessions, } from "./backfill-stub-reoffer.js";
24
+ import { loadStubCursorHashes, locateStubbedSessions } from "./backfill-stub-locate.js";
24
25
  import { resolveBackfillScope } from "./backfill-scope.js";
25
26
  import { stubSessionKey } from "../evidence-stubs-client.js";
26
27
  /**
@@ -64,6 +65,7 @@ export async function scanBackfillRun(command, io, now) {
64
65
  fetchImpl: io.fetch,
65
66
  });
66
67
  reopenStubbedSessions(pointerlessTerminalSessions, stubReoffer);
68
+ const stubCursorHashes = await loadStubCursorHashes(paths, stubReoffer);
67
69
  const scan = await scanBackfillSessions({
68
70
  command,
69
71
  homeDir: command.homeDir ?? os.homedir(),
@@ -73,6 +75,8 @@ export async function scanBackfillRun(command, io, now) {
73
75
  window: scope.window,
74
76
  cursor,
75
77
  pointerlessTerminalSessions,
78
+ stubReoffer,
79
+ stubCursorHashes,
76
80
  now,
77
81
  });
78
82
  const reasonCounts = await auditScannedCandidates(scan, worktreeDiscovery.incomplete_reasons);
@@ -129,9 +133,19 @@ async function scanBackfillSessions(options) {
129
133
  const claudeAttribution = options.sources.includes("claude_code")
130
134
  ? await scanClaudeHistory(options, claudeProjectsDir, scanLimit)
131
135
  : null;
136
+ // BLI-11119: a stubbed session is found by the id in its content or by the
137
+ // file hash its cursor entry recorded, across every root the scan walked.
138
+ const scanned = locateStubbedSessions({
139
+ plan: options.stubReoffer,
140
+ candidates: [
141
+ ...(codexAttribution?.results.map(normalizeCodexCandidate) ?? []),
142
+ ...(claudeAttribution?.results.map(normalizeClaudeCandidate) ?? []),
143
+ ],
144
+ cursorHashes: options.stubCursorHashes,
145
+ sinceMinutes: options.window.since_minutes,
146
+ });
132
147
  const selection = selectBackfillCandidates({
133
- codexAttribution,
134
- claudeAttribution,
148
+ scanned,
135
149
  cursor: options.cursor,
136
150
  pointerlessTerminalSessions: options.pointerlessTerminalSessions,
137
151
  maxFiles: options.command.maxFiles,
@@ -201,10 +215,7 @@ async function scanClaudeHistory(request, projectsDir, scanLimit) {
201
215
  * so a `--max-files` cap spends its budget on them before newer history.
202
216
  */
203
217
  function selectBackfillCandidates(options) {
204
- const allCandidates = [
205
- ...(options.codexAttribution?.results.map(normalizeCodexCandidate) ?? []),
206
- ...(options.claudeAttribution?.results.map(normalizeClaudeCandidate) ?? []),
207
- ]
218
+ const allCandidates = options.scanned
208
219
  .filter((candidate) => !options.onlySessions ||
209
220
  options.onlySessions.has(stubSessionKey(candidate.source, candidate.session_id)))
210
221
  .filter((candidate) => isAfterCursor(candidate, options.cursor) ||
@@ -266,6 +277,7 @@ function normalizeCodexCandidate(result) {
266
277
  return {
267
278
  source: "codex",
268
279
  session_id: result.codex_session_id,
280
+ session_id_source: result.session_id_source,
269
281
  file_path: result.file_path,
270
282
  state: result.state,
271
283
  reason: result.reason,
@@ -0,0 +1,104 @@
1
+ /**
2
+ * Where on this machine is the transcript behind a stub? (BLI-11119)
3
+ *
4
+ * The stubs door names a session by the id inside its object key, and the
5
+ * collector wrote that key with the id it read from the transcript's CONTENT
6
+ * (`session_meta.id` for Codex). A rollout's file name does not always carry
7
+ * that id: a resumed or forked session is filed as
8
+ * `rollout-<time>-<parent>_<child>.jsonl` while its first line names another
9
+ * id, and Codex moves files from `sessions/` into `archived_sessions/`. So a
10
+ * stubbed session is located the way it was keyed, across every root the scan
11
+ * walked, never by file name alone:
12
+ *
13
+ * 1. a scanned session whose id matches (`found_by: session_meta`, or
14
+ * `file_name` when the file's first line gave no id and the scan fell back
15
+ * to the name, which is then the id the key used too);
16
+ * 2. otherwise a scanned file whose sha256 equals the cursor entry's
17
+ * `file_hash_sha256` for that session (`found_by: file_hash`); it is offered
18
+ * under the STUBBED session's id, because the server retires a stub only for
19
+ * a successor that names the same session;
20
+ * 3. otherwise `not_found`, recorded on the cursor so the scheduled sync stops
21
+ * rescanning history for it every tick (see `sync-stub-heal.ts`).
22
+ */
23
+ import { CLAUDE_CURSOR_FILENAME, readRawEvidenceCursor, } from "../cursors/raw-evidence-cursor.js";
24
+ import { stubSessionKey } from "../evidence-stubs-client.js";
25
+ /** How many per-session lines the locate log carries; the counts cover the rest. */
26
+ const LOGGED_SESSION_LIMIT = 25;
27
+ /** `codex:<id>` -> the file hash the cursor last recorded for that stubbed session. */
28
+ export async function loadStubCursorHashes(paths, plan) {
29
+ const hashes = new Map();
30
+ if (plan.sessions.size === 0)
31
+ return hashes;
32
+ for (const source of ["codex", "claude_code"]) {
33
+ const filename = source === "claude_code" ? CLAUDE_CURSOR_FILENAME : undefined;
34
+ const cursor = await readRawEvidenceCursor(paths, { filename }).catch(() => null);
35
+ if (!cursor)
36
+ continue;
37
+ for (const key of plan.sessions.keys()) {
38
+ if (!key.startsWith(`${source}:`))
39
+ continue;
40
+ const hash = cursor.sessions[key.slice(source.length + 1)]?.file_hash_sha256;
41
+ if (hash)
42
+ hashes.set(key, hash);
43
+ }
44
+ }
45
+ return hashes;
46
+ }
47
+ /**
48
+ * Match every stubbed session to a scanned file, record how on `plan.found_by`,
49
+ * and return the scanned candidates plus one candidate per hash match, carrying
50
+ * the stubbed session's id. Logs the outcome, found or not.
51
+ */
52
+ export function locateStubbedSessions(options) {
53
+ const { plan } = options;
54
+ if (plan.sessions.size === 0)
55
+ return options.candidates;
56
+ const byKey = new Map();
57
+ const byHash = new Map();
58
+ for (const candidate of options.candidates) {
59
+ byKey.set(stubSessionKey(candidate.source, candidate.session_id), candidate);
60
+ if (candidate.content_hash_sha256) {
61
+ const hashKey = `${candidate.source}:${candidate.content_hash_sha256}`;
62
+ if (!byHash.has(hashKey))
63
+ byHash.set(hashKey, candidate);
64
+ }
65
+ }
66
+ const extra = [];
67
+ const sessions = [];
68
+ for (const key of plan.sessions.keys()) {
69
+ const found = locateOne(key, byKey, byHash, options.cursorHashes);
70
+ plan.found_by.set(key, found.foundBy);
71
+ if (found.candidate)
72
+ extra.push(found.candidate);
73
+ sessions.push({ session: key, found_by: found.foundBy });
74
+ }
75
+ const counts = { session_meta: 0, file_name: 0, file_hash: 0, not_found: 0 };
76
+ for (const session of sessions)
77
+ counts[session.found_by] += 1;
78
+ console.error("[backfill] stubbed sessions located", JSON.stringify({
79
+ reason: counts.not_found === sessions.length ? "stub_sessions_not_found" : "stub_sessions_located",
80
+ sessions: sessions.length,
81
+ ...counts,
82
+ scanned_candidates: options.candidates.length,
83
+ since_minutes: options.sinceMinutes,
84
+ found_by: sessions.slice(0, LOGGED_SESSION_LIMIT),
85
+ }));
86
+ return extra.length > 0 ? [...options.candidates, ...extra] : options.candidates;
87
+ }
88
+ function locateOne(key, byKey, byHash, cursorHashes) {
89
+ const byId = byKey.get(key);
90
+ if (byId) {
91
+ const fromContent = byId.source === "codex" && byId.session_id_source === "session_meta";
92
+ return { foundBy: fromContent ? "session_meta" : "file_name" };
93
+ }
94
+ const at = key.indexOf(":");
95
+ const source = key.slice(0, at);
96
+ const hash = cursorHashes.get(key);
97
+ const byContent = hash ? byHash.get(`${source}:${hash}`) : undefined;
98
+ if (!byContent)
99
+ return { foundBy: "not_found" };
100
+ return {
101
+ foundBy: "file_hash",
102
+ candidate: { ...byContent, session_id: key.slice(at + 1), session_id_source: undefined },
103
+ };
104
+ }
@@ -1,10 +1,18 @@
1
1
  import { CLAUDE_CURSOR_FILENAME, readRawEvidenceCursor, writeRawEvidenceCursor, } from "../cursors/raw-evidence-cursor.js";
2
+ import { markCursorEntryStubbed } from "../cursors/raw-evidence-stub-entry.js";
2
3
  import { fetchStubbedSessions, stubSessionKey } from "../evidence-stubs-client.js";
3
4
  import { describeError } from "../health-detail.js";
4
5
  import { readLocalCollectorSessionFile } from "../local-state.js";
5
6
  import { candidateCursorKey } from "./backfill-candidates.js";
6
7
  export function emptyStubReofferPlan(reason) {
7
- return { status: "skipped", reason, sessions: new Map(), stub_count: 0, marked_local_sessions: 0 };
8
+ return {
9
+ status: "skipped",
10
+ reason,
11
+ sessions: new Map(),
12
+ stub_count: 0,
13
+ marked_local_sessions: 0,
14
+ found_by: new Map(),
15
+ };
8
16
  }
9
17
  /** Ask the server which of this operator's pointers are stubs. Never throws; writes nothing. */
10
18
  export async function loadStubReofferPlan(options) {
@@ -25,6 +33,7 @@ export async function loadStubReofferPlan(options) {
25
33
  sessions,
26
34
  stub_count: answer.unique_stub_count,
27
35
  marked_local_sessions: 0,
36
+ found_by: new Map(),
28
37
  };
29
38
  }
30
39
  /**
@@ -34,7 +43,7 @@ export async function loadStubReofferPlan(options) {
34
43
  export async function markStubReoffers(paths, plan, now) {
35
44
  if (plan.sessions.size === 0)
36
45
  return;
37
- plan.marked_local_sessions = await markStubbedSessions(paths, plan.sessions, now);
46
+ plan.marked_local_sessions = await markStubbedSessions(paths, plan, now);
38
47
  }
39
48
  /** Reopen stubbed sessions exactly like pointer-less ones: selected past the cursor, sorted first. */
40
49
  export function reopenStubbedSessions(reopen, plan) {
@@ -67,6 +76,7 @@ export function summarizeStubReoffer(plan, candidates, durableCandidateKeys = ne
67
76
  reoffered_sessions: reoffered,
68
77
  reoffered_stubs: stubsOf(offered.filter((candidate) => attempted.includes(candidate))),
69
78
  replaced_stubs: stubsOf(offered.filter((candidate) => durableCandidateKeys.has(candidateCursorKey(candidate)))),
79
+ found_by: countFoundBy(plan),
70
80
  };
71
81
  if (plan.sessions.size > 0) {
72
82
  console.error("[backfill] redaction-stubbed sessions re-offered", JSON.stringify({
@@ -79,20 +89,43 @@ export function summarizeStubReoffer(plan, candidates, durableCandidateKeys = ne
79
89
  }
80
90
  return summary;
81
91
  }
82
- async function markStubbedSessions(paths, sessions, now) {
92
+ function countFoundBy(plan) {
93
+ const counts = { session_meta: 0, file_name: 0, file_hash: 0, not_found: 0 };
94
+ for (const method of plan.found_by.values())
95
+ counts[method] += 1;
96
+ return counts;
97
+ }
98
+ /**
99
+ * Mark every door-named session this cursor tracks, and make its entry tell
100
+ * the truth (BLI-11119): an entry written before the stub rule existed claimed
101
+ * the full file as uploaded. The door is the source of truth, so each such
102
+ * entry is reconciled to `stubbed: true` with the marker's bytes, counted and
103
+ * logged as `stub_cursor_reconciled`. A session the scan could not find in any
104
+ * root keeps `not_found_at`, so the scheduled sync waits before looking again.
105
+ */
106
+ async function markStubbedSessions(paths, plan, now) {
83
107
  let marked = 0;
108
+ let reconciled = 0;
84
109
  for (const source of ["codex", "claude_code"]) {
85
110
  const filename = source === "claude_code" ? CLAUDE_CURSOR_FILENAME : undefined;
86
111
  try {
87
112
  const cursor = await readRawEvidenceCursor(paths, { filename });
88
113
  let changed = false;
89
- for (const [key, pointerIds] of sessions) {
114
+ for (const [key, pointerIds] of plan.sessions) {
90
115
  if (!key.startsWith(`${source}:`))
91
116
  continue;
92
117
  const entry = cursor.sessions[key.slice(source.length + 1)];
93
118
  if (!entry)
94
119
  continue;
95
- entry.stub_reoffer = { reason: "stub_reoffered", stub_pointer_ids: pointerIds, marked_at: now.toISOString() };
120
+ const notFound = plan.found_by.get(key) === "not_found";
121
+ entry.stub_reoffer = {
122
+ reason: "stub_reoffered",
123
+ stub_pointer_ids: pointerIds,
124
+ marked_at: now.toISOString(),
125
+ ...(notFound ? { not_found_at: now.toISOString() } : {}),
126
+ };
127
+ if (markCursorEntryStubbed(entry))
128
+ reconciled += 1;
96
129
  changed = true;
97
130
  marked += 1;
98
131
  }
@@ -105,5 +138,6 @@ async function markStubbedSessions(paths, sessions, now) {
105
138
  console.error("[backfill] stub mark not written to the cursor", JSON.stringify({ reason: "stub_mark_write_failed", source, ...describeError(error) }));
106
139
  }
107
140
  }
141
+ console.error("[backfill] stubbed sessions' cursor entries reconciled with the stubs door", JSON.stringify({ reason: "stub_cursor_reconciled", count: reconciled, marked_local_sessions: marked }));
108
142
  return marked;
109
143
  }
@@ -0,0 +1,27 @@
1
+ /**
2
+ * `cockpit sources check [--older-than <days>] [--json]` (BLI-11122).
3
+ *
4
+ * One verb, read-only. There is deliberately no `sources clean` and no
5
+ * `--apply`: Tower never deletes a person's raw transcripts; deleting one is
6
+ * the person's own act, outside Tower. A path is never accepted: the roots are
7
+ * the ones the collector reads (`sources-discovery.ts`).
8
+ */
9
+ import { optionalNonEmpty, optionalPositiveInteger, optionalUrl, parseNamedArgs, } from "./local-arg-values.js";
10
+ export function parseSourcesArgs(args) {
11
+ const values = parseNamedArgs(args, {
12
+ allowedFlags: ["--older-than", "--home", "--dashboard-url", "--json"],
13
+ valueFlags: ["--older-than", "--home", "--dashboard-url"],
14
+ });
15
+ const action = values.positionals[0] ?? "check";
16
+ if (action !== "check" || values.positionals.length > 1) {
17
+ throw new Error("sources has one verb: `cockpit sources check [--older-than <days>] [--json]`.");
18
+ }
19
+ return {
20
+ kind: "sources",
21
+ action: "check",
22
+ olderThanDays: optionalPositiveInteger(values.flags.get("--older-than"), "--older-than") ?? null,
23
+ homeDir: optionalNonEmpty(values.flags.get("--home")),
24
+ dashboardUrl: optionalUrl(values.flags.get("--dashboard-url")),
25
+ json: values.booleans.has("--json"),
26
+ };
27
+ }
@@ -26,4 +26,5 @@
26
26
  */
27
27
  export { parseOnboardArgs, parseUpdateArgs, parseDoctorArgs, parseInstallArgs, parseReleaseArgs, parseLoginArgs, parseLogoutArgs, } from "./local-args-collector-setup.js";
28
28
  export { parseStartArgs, parseSyncArgs, parseAnalyzeArgs, parseBackfillArgs, } from "./local-args-collector-work.js";
29
- export { parseStatusArgs, parseSessionsArgs, parseServeArgs, parseAutostartArgs, parseAgentRulesArgs, parseMemoryArgs, parseCleanArgs, } from "./local-args-collector-status.js";
29
+ export { parseStatusArgs, parseSessionsArgs, parseServeArgs, parseAutostartArgs, parseAgentRulesArgs, parseMemoryArgs, parseCleanArgs, } from "./local-args-collector-status.js";
30
+ export { parseSourcesArgs } from "./local-args-collector-sources.js";
@@ -17,7 +17,7 @@
17
17
  */
18
18
  import { parseRootsArgs } from "./roots.js";
19
19
  import { SURFACE_ALIASES } from "./local-help.js";
20
- import { parseAgentRulesArgs, parseAnalyzeArgs, parseAutostartArgs, parseBackfillArgs, parseCleanArgs, parseDoctorArgs, parseInstallArgs, parseLoginArgs, parseMemoryArgs, parseLogoutArgs, parseOnboardArgs, parseReleaseArgs, parseServeArgs, parseSessionsArgs, parseStartArgs, parseStatusArgs, parseSyncArgs, parseUpdateArgs, } from "./local-args-collector.js";
20
+ import { parseAgentRulesArgs, parseAnalyzeArgs, parseAutostartArgs, parseBackfillArgs, parseCleanArgs, parseDoctorArgs, parseInstallArgs, parseLoginArgs, parseMemoryArgs, parseLogoutArgs, parseOnboardArgs, parseReleaseArgs, parseServeArgs, parseSessionsArgs, parseSourcesArgs, parseStartArgs, parseStatusArgs, parseSyncArgs, parseUpdateArgs, } from "./local-args-collector.js";
21
21
  import { parseBriefArgs, parseCorrectArgs, parseDocsArgs, parseIssueArgs, parseJarvisArgs, parseMailArgs, parseCalArgs, parseModelArgs, parseMsgArgs, parseNotesArgs, parseOpsArgs, parseProjectArgs, parseModelsArgs, parseScoutArgs, parseSearchArgs, parseSettingsArgs, parseSlackArgs, parseTeamArgs, parseWorkbookArgs, parseUsageArgs, parseCareersArgs, } from "./local-args-tower.js";
22
22
  // `normalizeUrl` has always been part of this module's surface — `local.ts` and
23
23
  // `local-auth.ts` import it from here — so it stays exported from this address
@@ -107,6 +107,8 @@ export function parseLocalArgs(argv) {
107
107
  return parseMemoryArgs(argv.slice(1));
108
108
  case "clean":
109
109
  return parseCleanArgs(argv.slice(1));
110
+ case "sources":
111
+ return parseSourcesArgs(argv.slice(1));
110
112
  case "docs":
111
113
  return parseDocsArgs(argv.slice(1));
112
114
  case "msg":
@@ -366,6 +366,20 @@ export function localSubcommandHelp(command) {
366
366
  "laptop that has been off.",
367
367
  ],
368
368
  ],
369
+ [
370
+ "sources",
371
+ [
372
+ "Usage: cockpit sources check [--older-than <days>] [--dashboard-url <url>] [--json]",
373
+ "",
374
+ "Answers, for every Codex and Claude transcript on this machine, whether Tower",
375
+ "holds it: stored in full, stored in part, or not stored, with the reason.",
376
+ "--older-than <days> checks only files not modified in that many days.",
377
+ "--json lists every file with its path, verdict and reason.",
378
+ "Read-only. Tower never deletes these files; deleting one is your own act.",
379
+ "Before deleting any transcript by hand, run this and delete only files it",
380
+ "answers \"stored in full\". See docs/runbooks/freeing-disk-on-a-collector-machine.md.",
381
+ ],
382
+ ],
369
383
  [
370
384
  "release",
371
385
  [
@@ -65,6 +65,7 @@ export const rootCommandNames = new Set([
65
65
  "agent-rules",
66
66
  "memory",
67
67
  "clean",
68
+ "sources",
68
69
  "docs",
69
70
  "msg",
70
71
  "issue",
@@ -117,6 +118,7 @@ export function localCommandHelp(command) {
117
118
  " cockpit agent-rules [install|uninstall|status] [--host codex|claude|all] [--workspace <path>] [--json]",
118
119
  " cockpit memory [search \"<question>\"|save|update <id>|forget <id>|profile|install|status|log <win|loss|noise> --store <bli|supermemory|both> \"<reason>\"] [--container <tag>] [--limit <n>] [--no-recent] [--custom-id <id>] [--extract] [--content-stdin] [--reason <label>] [--file <path>] [--dashboard-url <url>] [--dry-run] [--json]",
119
120
  " cockpit clean [--dry-run] [--all-committed] [--reconcile] [--dashboard-url <url>] [--json]",
121
+ " cockpit sources check [--older-than <days>] [--dashboard-url <url>] [--json]",
120
122
  " cockpit docs [list|tree|read <id|slug>|create --title <t>|update <id>|delete <id|slug>] [--parent <id>|root|--clear-parent] [--query <text> [--word]] [--limit <n>] [--visibility org|private] [--file <path>|--body-stdin] [--allow-empty] [--dashboard-url <url>] [--json]",
121
123
  " cockpit msg [channels|create <name>|dm <email>|read <channel>|send <channel>|thread <id> --channel <channel>] [--private] [--members a@x,b@y] [--description <text>] [--thread <id>] [--limit <n>] [--dashboard-url <url>] [--json]",
122
124
  " cockpit issue [list|show <BLI-id>|create --title <t>|update <BLI-id>|move <BLI-id> <state>|comment <BLI-id>|history <BLI-id>] [--state <s>] [--assignee me|unassigned|<uuid>] [--project <name|id>] [--limit <n>] [--priority 0-4] [--size S|M|L] [--parent <BLI-id>] [--file <path>|--body-stdin] [--dashboard-url <url>] [--json]",
@@ -37,6 +37,7 @@ import { runMemoryLog } from "./memory-log.js";
37
37
  import { runMemoryInstall } from "./memory-install.js";
38
38
  import { runMemoryStore, runMemoryVerbList } from "./memory-store.js";
39
39
  import { runClean } from "./clean.js";
40
+ import { runSourcesCheck } from "./sources-check.js";
40
41
  import { runDocs } from "./docs.js";
41
42
  import { runMsg } from "./msg.js";
42
43
  import { runIssue } from "./issue.js";
@@ -213,6 +214,8 @@ export async function runLocalCockpitCli(argv, io = defaultIo()) {
213
214
  return await runMemoryStore(command, io);
214
215
  case "clean":
215
216
  return await runClean(command, io);
217
+ case "sources":
218
+ return await runSourcesCheck(command, io);
216
219
  case "docs":
217
220
  return await runDocs(command, io);
218
221
  case "msg":
@@ -21,7 +21,7 @@ export async function runCockpitCli(argv, io) {
21
21
  }
22
22
 
23
23
  if (command === "--version" || command === "-V" || command === "version") {
24
- writeLine(io?.stdout ?? process.stdout, "0.2.157");
24
+ writeLine(io?.stdout ?? process.stdout, "0.2.159");
25
25
  return 0;
26
26
  }
27
27
 
@@ -11,6 +11,7 @@ import { DEFAULT_DASHBOARD_URL, getCollectorRuntimePaths, readLocalCollectorConf
11
11
  import { normalizeDashboardUrl } from "../upload-http.js";
12
12
  import { flushPendingCodexSessionReports, queueCodexSessionReport, } from "../upload.js";
13
13
  import { CLAUDE_CURSOR_FILENAME, countStaleSessions, readRawEvidenceCursor, recordSessionObservation, writeRawEvidenceCursor, } from "../cursors/raw-evidence-cursor.js";
14
+ import { isRedactionStubObjectKey, markCursorEntryStubbed } from "../cursors/raw-evidence-stub-entry.js";
14
15
  import { claudeAttributionReadFailureCount, codexAttributionReadFailureCount, } from "./agent-session-report.js";
15
16
  import { liveSyncCursorEntryRequiresRetry } from "./session-sync-attribution.js";
16
17
  /**
@@ -331,5 +332,16 @@ function recordSourceObservation(options, result) {
331
332
  uploaded_byte_size: uploadedByteSize,
332
333
  last_seen_at: options.now.toISOString(),
333
334
  };
335
+ // BLI-11119: a pointer that is the redaction-crash marker is not a delivery.
336
+ // The entry records the marker's bytes, and keeps the re-offer mark while
337
+ // the stub still stands.
338
+ if (isRedactionStubObjectKey(uploadedObjectKey)) {
339
+ markCursorEntryStubbed(entry);
340
+ if (prior?.stub_reoffer)
341
+ entry.stub_reoffer = prior.stub_reoffer;
342
+ if (uploadedThisSync) {
343
+ console.error("[collector sync] session delivered only as a redaction-crash stub", JSON.stringify({ reason: "stub_cursor_recorded", source: options.source, byte_size: result.byte_size }));
344
+ }
345
+ }
334
346
  recordSessionObservation(options.cursor, sessionId, entry);
335
347
  }
@@ -0,0 +1,135 @@
1
+ import { writeLine } from "./cli-io.js";
2
+ import { CLAUDE_CURSOR_FILENAME, readRawEvidenceCursor, } from "../cursors/raw-evidence-cursor.js";
3
+ import { fetchTranscriptObjects } from "../evidence-transcripts-client.js";
4
+ import { DEFAULT_DASHBOARD_URL, getCollectorRuntimePaths, readLocalCollectorConfig, readLocalCollectorSessionFile, } from "../local-state.js";
5
+ import { discoverLocalTranscripts } from "./sources-discovery.js";
6
+ import { fileFacts, judgeSource } from "./sources-verdict.js";
7
+ const VERDICTS = ["stored_full", "stored_partial", "not_stored"];
8
+ const DAY_MS = 24 * 60 * 60 * 1000;
9
+ export async function runSourcesCheck(command, io, now = new Date()) {
10
+ const startedAt = Date.now();
11
+ const paths = getCollectorRuntimePaths(command.homeDir);
12
+ const config = await readLocalCollectorConfig(paths).catch(() => null);
13
+ const dashboardUrl = command.dashboardUrl ?? config?.dashboard_url ?? DEFAULT_DASHBOARD_URL;
14
+ const session = await readLocalCollectorSessionFile(paths).catch(() => null);
15
+ if (!session || session.session_state !== "valid" || typeof session.device_token !== "string" || !session.device_token) {
16
+ console.error("[collector sources] check not run", JSON.stringify({ reason: "collector_not_paired" }));
17
+ return fail(io, command, "collector_not_paired", "This machine is not signed in to Tower, so Tower cannot be asked. Run `cockpit doctor`.");
18
+ }
19
+ const remote = await fetchTranscriptObjects({ dashboardUrl, deviceToken: session.device_token, fetchImpl: io.fetch });
20
+ if (remote.status === "unavailable") {
21
+ console.error("[collector sources] check not answered", JSON.stringify({ reason: `transcript_door_unavailable:${remote.reason}` }));
22
+ return fail(io, command, `transcript_door_unavailable:${remote.reason}`, `Tower could not be asked which transcripts it holds (${remote.reason}). No file is answered; nothing here means "not stored".`);
23
+ }
24
+ const homeDir = command.homeDir ?? paths.home_dir;
25
+ const olderThanMs = command.olderThanDays === null ? null : now.getTime() - command.olderThanDays * DAY_MS;
26
+ const local = await discoverLocalTranscripts({ homeDir, olderThanMs });
27
+ const cursors = {
28
+ codex: await readRawEvidenceCursor(paths),
29
+ claude_code: await readRawEvidenceCursor(paths, { filename: CLAUDE_CURSOR_FILENAME, sessionsOnly: true }),
30
+ };
31
+ const report = emptyReport(command, remote, local.newer_count, local.file_read_failed + local.directory_read_failed);
32
+ for (const file of local.files) {
33
+ const judged = await judgeSource({
34
+ file,
35
+ remotes: remote.files.get(file.key) ?? [],
36
+ cursorEntry: file.kind === "claude_subagent" ? null : (cursors[file.source].sessions[file.session_id] ?? null),
37
+ facts: fileFacts(file),
38
+ }).catch((error) => {
39
+ report.local.unreadable += 1;
40
+ console.error("[collector sources] a transcript could not be judged; left out of the answer", JSON.stringify({ reason: "file_unreadable", kind: file.kind, byte_size: file.byte_size, error_name: error?.name ?? "Error" }));
41
+ return null;
42
+ });
43
+ if (!judged)
44
+ continue;
45
+ report.local.examined += 1;
46
+ report.counts[judged.verdict].files += 1;
47
+ report.counts[judged.verdict].bytes += file.byte_size;
48
+ report.reasons[judged.reason] = (report.reasons[judged.reason] ?? 0) + 1;
49
+ report.files.push({
50
+ path: file.file_path,
51
+ source: file.source,
52
+ kind: file.kind,
53
+ session_id: file.session_id,
54
+ byte_size: file.byte_size,
55
+ modified_at: new Date(file.mtime_ms).toISOString(),
56
+ verdict: judged.verdict,
57
+ reason: judged.reason,
58
+ remote_byte_size: judged.remote_byte_size,
59
+ });
60
+ }
61
+ report.files.sort((a, b) => VERDICTS.indexOf(a.verdict) - VERDICTS.indexOf(b.verdict) || a.path.localeCompare(b.path));
62
+ // Counts and bytes only: never a path, a session id or a byte of content.
63
+ console.error("[collector sources] check", JSON.stringify({
64
+ reason: `sources_check_${report.status}`,
65
+ older_than_days: report.older_than_days,
66
+ remote_status: remote.status,
67
+ remote_unique: remote.unique,
68
+ examined: report.local.examined,
69
+ newer_skipped: report.local.newer_skipped,
70
+ unreadable: report.local.unreadable,
71
+ stored_full: report.counts.stored_full,
72
+ stored_partial: report.counts.stored_partial,
73
+ not_stored: report.counts.not_stored,
74
+ reasons: report.reasons,
75
+ elapsed_ms: Date.now() - startedAt,
76
+ }));
77
+ if (command.json)
78
+ writeLine(io.stdout, JSON.stringify(report, null, 2));
79
+ else
80
+ for (const line of sourcesCheckLines(report))
81
+ writeLine(io.stdout, line);
82
+ return 0;
83
+ }
84
+ function emptyReport(command, remote, newerSkipped, unreadable) {
85
+ const { files, ...remoteCounts } = remote;
86
+ return {
87
+ status: remote.status === "exhausted" ? "complete" : "incomplete",
88
+ reason: remote.status === "exhausted" ? "ok" : `transcript_list_${remote.status}${remote.reason === "ok" ? "" : `:${remote.reason}`}`,
89
+ older_than_days: command.olderThanDays,
90
+ remote: { ...remoteCounts, file_count: files.size },
91
+ local: { examined: 0, newer_skipped: newerSkipped, unreadable },
92
+ counts: {
93
+ stored_full: { files: 0, bytes: 0 },
94
+ stored_partial: { files: 0, bytes: 0 },
95
+ not_stored: { files: 0, bytes: 0 },
96
+ },
97
+ reasons: {},
98
+ files: [],
99
+ };
100
+ }
101
+ function fail(io, command, reason, sentence) {
102
+ if (command.json) {
103
+ writeLine(io.stdout, JSON.stringify({ status: "unavailable", reason, older_than_days: command.olderThanDays }, null, 2));
104
+ }
105
+ else {
106
+ writeLine(io.stderr, sentence);
107
+ }
108
+ return 1;
109
+ }
110
+ /** What a person reads: counts and bytes per verdict, then why. Paths only with --json. */
111
+ export function sourcesCheckLines(report) {
112
+ const scope = report.older_than_days === null ? "every transcript" : `transcripts not modified in ${report.older_than_days} days`;
113
+ const lines = [
114
+ `Tower sources check: ${scope} on this machine`,
115
+ ` stored in full ${report.counts.stored_full.files} file(s), ${mb(report.counts.stored_full.bytes)} MB`,
116
+ ` stored in part ${report.counts.stored_partial.files} file(s), ${mb(report.counts.stored_partial.bytes)} MB`,
117
+ ` not stored ${report.counts.not_stored.files} file(s), ${mb(report.counts.not_stored.bytes)} MB`,
118
+ ];
119
+ const reasons = Object.entries(report.reasons).sort((a, b) => b[1] - a[1]);
120
+ if (reasons.length > 0)
121
+ lines.push(` why ${reasons.map(([reason, count]) => `${reason} ${count}`).join(", ")}`);
122
+ if (report.local.newer_skipped > 0)
123
+ lines.push(` newer ${report.local.newer_skipped} file(s) modified inside the window, not checked`);
124
+ if (report.local.unreadable > 0)
125
+ lines.push(` unreadable ${report.local.unreadable} file(s) could not be read and are not answered`);
126
+ lines.push(` Tower's list ${report.remote.status}: ${report.remote.unique} current transcript object(s) over ${report.remote.pages} page(s)`);
127
+ if (report.status === "incomplete") {
128
+ lines.push(` INCOMPLETE Tower's list was cut short (${report.reason}); "not stored" and "stored in part" may be understated. Run it again.`);
129
+ }
130
+ lines.push("Tower never deletes these files. `--json` lists every file with its path and verdict. Only a \"stored in full\" file is safe to delete by your own hand; see docs/runbooks/freeing-disk-on-a-collector-machine.md.");
131
+ return lines;
132
+ }
133
+ function mb(bytes) {
134
+ return (bytes / (1024 * 1024)).toFixed(1);
135
+ }
@@ -0,0 +1,232 @@
1
+ /**
2
+ * Every agent transcript on this machine, named the way the collector names
3
+ * its upload (BLI-11122).
4
+ *
5
+ * `cockpit sources check` has to pair each local file with the object Tower
6
+ * holds for it, and an object key carries the session id the collector read
7
+ * from the transcript's CONTENT, not its file name (a resumed or forked Codex
8
+ * rollout is filed under one id and names another; see
9
+ * `backfill-stub-locate.ts`). So each file is named by the same rules the
10
+ * readers use, kept here as pure functions so both path layouts (macOS and
11
+ * Windows) are testable without either file system:
12
+ *
13
+ * - Codex (`~/.codex/sessions/**`, `~/.codex/archived_sessions/**`): the first
14
+ * `session_meta` record's `payload.id`, else the uuid in the file name, else
15
+ * a hash of the path (`codex-attribution.ts`).
16
+ * - Claude main (`~/.claude/projects/<project>/<uuid>.jsonl`): the first
17
+ * record's `sessionId`, else the file name's uuid (`claude-attribution-score.ts`).
18
+ * - Claude subagent (`<project>/<uuid>/subagents/agent-*.jsonl`): the main's
19
+ * session id plus the file stem (`raw-evidence-claude-reader.ts`).
20
+ *
21
+ * This module reads files and never writes, moves or deletes one.
22
+ */
23
+ import fs from "node:fs/promises";
24
+ import path from "node:path";
25
+ import { createReadStream } from "node:fs";
26
+ import readline from "node:readline";
27
+ import { transcriptFileKey } from "@bli-cockpit/telemetry-core";
28
+ import { SESSION_FILE_UUID_PATTERN, sanitizeSessionId, sessionIdFromFileName, shortHash, } from "../adapters/attribution-core.js";
29
+ import { isSecretLikePath, safeKeySegment } from "../adapters/raw-evidence-keys.js";
30
+ import { describeError } from "../health-detail.js";
31
+ export function sourcesRoots(homeDir, pathApi = path) {
32
+ return {
33
+ codex: [
34
+ pathApi.join(homeDir, ".codex", "sessions"),
35
+ pathApi.join(homeDir, ".codex", "archived_sessions"),
36
+ ],
37
+ claudeProjects: pathApi.join(homeDir, ".claude", "projects"),
38
+ };
39
+ }
40
+ /** Which kind of transcript a path is, or null for anything the collector does not upload as one. */
41
+ export function classifyTranscriptPath(filePath, roots, pathApi = path) {
42
+ if (!filePath.toLowerCase().endsWith(".jsonl"))
43
+ return null;
44
+ for (const root of roots.codex) {
45
+ const rel = pathApi.relative(root, filePath);
46
+ if (rel && !rel.startsWith("..") && !pathApi.isAbsolute(rel))
47
+ return { kind: "codex_rollout" };
48
+ }
49
+ const rel = pathApi.relative(roots.claudeProjects, filePath);
50
+ if (!rel || rel.startsWith("..") || pathApi.isAbsolute(rel))
51
+ return null;
52
+ const parts = rel.split(pathApi.sep);
53
+ if (parts.length === 2 && SESSION_FILE_UUID_PATTERN.test(parts[1]))
54
+ return { kind: "claude_main" };
55
+ if (parts.length === 4 && parts[2] === "subagents" && parts[3].startsWith("agent-")) {
56
+ return {
57
+ kind: "claude_subagent",
58
+ main_session_dir: parts[1],
59
+ sidecar_stem: parts[3].replace(/\.jsonl$/i, ""),
60
+ };
61
+ }
62
+ return null;
63
+ }
64
+ /** The session id a reader would key this file's session by, from its content id when it has one. */
65
+ export function sessionIdFor(filePath, contentId, pathApi = path) {
66
+ const fromContent = sanitizeSessionId(contentId ?? undefined);
67
+ if (fromContent)
68
+ return { session_id: fromContent, session_id_source: "content" };
69
+ const fromName = sessionIdFromFileName(pathApi.basename(filePath));
70
+ if (fromName)
71
+ return { session_id: fromName, session_id_source: "file_name" };
72
+ return { session_id: shortHash(filePath), session_id_source: "path_hash" };
73
+ }
74
+ /** The key of the object a file would be uploaded as; see `raw-evidence-*-reader.ts`. */
75
+ export function transcriptKeyFor(source, sessionId, sidecarStem) {
76
+ const identity = {
77
+ source,
78
+ session_id: safeKeySegment(sessionId),
79
+ sidecar: sidecarStem === null ? null : isSecretLikePath(sidecarStem) ? "redacted-file-name" : safeKeySegment(sidecarStem),
80
+ };
81
+ return transcriptFileKey(identity);
82
+ }
83
+ /**
84
+ * The first content id in a transcript: Codex's first `session_meta`
85
+ * `payload.id`, Claude's first `sessionId`. Streams line by line and stops at
86
+ * the first hit; a line that does not parse is skipped, as the readers do.
87
+ */
88
+ export async function readContentSessionId(filePath, source) {
89
+ const stream = createReadStream(filePath, { encoding: "utf8" });
90
+ const lines = readline.createInterface({ input: stream, crlfDelay: Infinity });
91
+ try {
92
+ for await (const line of lines) {
93
+ const id = contentIdFromLine(line, source);
94
+ if (id)
95
+ return id;
96
+ }
97
+ return null;
98
+ }
99
+ finally {
100
+ lines.close();
101
+ stream.destroy();
102
+ }
103
+ }
104
+ export function contentIdFromLine(line, source) {
105
+ if (!line.trim())
106
+ return null;
107
+ let record;
108
+ try {
109
+ record = JSON.parse(line);
110
+ }
111
+ catch {
112
+ return null;
113
+ }
114
+ if (!record || typeof record !== "object" || Array.isArray(record))
115
+ return null;
116
+ const entry = record;
117
+ if (source === "claude_code")
118
+ return trimmed(entry["sessionId"]);
119
+ if (entry["type"] !== "session_meta")
120
+ return null;
121
+ const payload = entry["payload"];
122
+ if (!payload || typeof payload !== "object")
123
+ return null;
124
+ return trimmed(payload["id"]);
125
+ }
126
+ function trimmed(value) {
127
+ return typeof value === "string" && value.trim() ? value.trim() : null;
128
+ }
129
+ /**
130
+ * Walk every root, name every transcript. `olderThanMs` keeps only files whose
131
+ * mtime is before it; the rest are counted, never read.
132
+ */
133
+ export async function discoverLocalTranscripts(options) {
134
+ const roots = sourcesRoots(options.homeDir);
135
+ const result = { files: [], newer_count: 0, directory_read_failed: 0, file_read_failed: 0 };
136
+ const found = [];
137
+ let firstFailure = null;
138
+ const stack = [...roots.codex, roots.claudeProjects];
139
+ const seen = new Set();
140
+ while (stack.length > 0) {
141
+ const dir = stack.pop();
142
+ let entries;
143
+ try {
144
+ entries = await fs.readdir(dir, { withFileTypes: true });
145
+ }
146
+ catch (error) {
147
+ if (error?.code !== "ENOENT") {
148
+ result.directory_read_failed += 1;
149
+ firstFailure ??= describeError(error);
150
+ }
151
+ continue;
152
+ }
153
+ for (const entry of entries) {
154
+ const full = path.join(dir, entry.name);
155
+ if (entry.isDirectory()) {
156
+ stack.push(full);
157
+ continue;
158
+ }
159
+ if (!entry.isFile())
160
+ continue;
161
+ const shape = classifyTranscriptPath(full, roots);
162
+ if (!shape || seen.has(full))
163
+ continue;
164
+ seen.add(full);
165
+ const stat = await fs.stat(full).catch((error) => {
166
+ result.file_read_failed += 1;
167
+ firstFailure ??= describeError(error);
168
+ return null;
169
+ });
170
+ if (!stat)
171
+ continue;
172
+ if (options.olderThanMs !== null && stat.mtimeMs >= options.olderThanMs) {
173
+ result.newer_count += 1;
174
+ continue;
175
+ }
176
+ found.push({ file: full, size: stat.size, mtimeMs: stat.mtimeMs, shape });
177
+ }
178
+ }
179
+ // Claude mains first, so a subagent file can take its main's content id.
180
+ const claudeMainIds = new Map();
181
+ found.sort((a, b) => shapeOrder(a.shape) - shapeOrder(b.shape) || a.file.localeCompare(b.file));
182
+ for (const item of found) {
183
+ const source = item.shape.kind === "codex_rollout" ? "codex" : "claude_code";
184
+ let contentId = null;
185
+ if (item.shape.kind !== "claude_subagent") {
186
+ const read = await readContentSessionId(item.file, source).then((id) => ({ ok: true, id }), (error) => {
187
+ result.file_read_failed += 1;
188
+ firstFailure ??= describeError(error);
189
+ return { ok: false, id: null };
190
+ });
191
+ if (!read.ok)
192
+ continue;
193
+ contentId = read.id;
194
+ }
195
+ let named;
196
+ if (item.shape.kind === "claude_subagent") {
197
+ // The reader keys a subagent file under its MAIN session's id.
198
+ const mainPath = path.join(path.dirname(path.dirname(path.dirname(item.file))), `${item.shape.main_session_dir}.jsonl`);
199
+ const known = claudeMainIds.get(mainPath);
200
+ named = known
201
+ ? { session_id: known, session_id_source: "content" }
202
+ : sessionIdFor(mainPath, await readContentSessionId(mainPath, "claude_code").catch(() => null));
203
+ }
204
+ else {
205
+ named = sessionIdFor(item.file, contentId);
206
+ if (item.shape.kind === "claude_main")
207
+ claudeMainIds.set(item.file, named.session_id);
208
+ }
209
+ result.files.push({
210
+ source,
211
+ kind: item.shape.kind,
212
+ file_path: item.file,
213
+ byte_size: item.size,
214
+ mtime_ms: item.mtimeMs,
215
+ ...named,
216
+ key: transcriptKeyFor(source, named.session_id, item.shape.kind === "claude_subagent" ? item.shape.sidecar_stem : null),
217
+ });
218
+ }
219
+ if (result.directory_read_failed > 0 || result.file_read_failed > 0) {
220
+ console.error("[collector sources] some transcripts could not be read; they are left out of the answer", JSON.stringify({
221
+ reason: "sources_discovery_incomplete",
222
+ directory_read_failed: result.directory_read_failed,
223
+ file_read_failed: result.file_read_failed,
224
+ found_count: result.files.length,
225
+ ...(firstFailure ? { first_failure: firstFailure } : {}),
226
+ }));
227
+ }
228
+ return result;
229
+ }
230
+ function shapeOrder(shape) {
231
+ return shape.kind === "claude_main" ? 0 : shape.kind === "codex_rollout" ? 1 : 2;
232
+ }
@@ -0,0 +1,128 @@
1
+ /**
2
+ * Does Tower hold this transcript, in full, in part, or not at all?
3
+ * (BLI-11122)
4
+ *
5
+ * One file, the current objects Tower lists for it, and this machine's cursor
6
+ * entry for its session. The rules, cheapest first; every answer names why:
7
+ *
8
+ * not_stored no current object for the file (`no_object`), or only the
9
+ * redaction-crash stub (`stub_only`): a stub holds no byte.
10
+ * stored_full the cursor's uploaded object is still current, the file
11
+ * is the size it was when uploaded and its hash is the one the
12
+ * cursor last recorded (`unchanged_since_upload`); or the
13
+ * file, redacted exactly as the upload redacts it, has the
14
+ * hash of a current object (`content_hash_match`) or is no
15
+ * larger than one (`remote_at_least_sanitized_size`).
16
+ * stored_partial a current object exists but is smaller than the redacted
17
+ * file (`changed_since_upload` when the cursor saw the file
18
+ * grow after its upload, else `remote_smaller`), or its size
19
+ * is not recorded (`remote_size_unknown`).
20
+ *
21
+ * A file too large to redact in memory is compared by its raw size
22
+ * (`remote_at_least_file_size`). A file whose redaction would crash to the stub
23
+ * is compared the same way, never by the stub's 42 bytes.
24
+ *
25
+ * Read-only: this module reads the file and never writes, moves or deletes it.
26
+ */
27
+ import crypto from "node:crypto";
28
+ import fs from "node:fs/promises";
29
+ import path from "node:path";
30
+ import { RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES, REDACTION_STUB_SHA256, } from "@bli-cockpit/telemetry-core";
31
+ import { sanitizeTextEvidenceForUpload } from "../adapters/raw-evidence-sanitize.js";
32
+ import { isSecretLikePath } from "../adapters/raw-evidence-keys.js";
33
+ export async function judgeSource(input) {
34
+ const { file, remotes, cursorEntry } = input;
35
+ if (remotes.length === 0)
36
+ return { verdict: "not_stored", reason: "no_object", remote_byte_size: null };
37
+ const held = remotes.filter((remote) => !remote.stub);
38
+ if (held.length === 0)
39
+ return { verdict: "not_stored", reason: "stub_only", remote_byte_size: null };
40
+ const sizes = held.map((remote) => remote.byte_size).filter((size) => size !== null);
41
+ const best = sizes.length > 0 ? Math.max(...sizes) : null;
42
+ const full = (reason) => ({ verdict: "stored_full", reason, remote_byte_size: best });
43
+ const partial = (reason) => ({
44
+ verdict: "stored_partial",
45
+ reason,
46
+ remote_byte_size: best,
47
+ });
48
+ if (cursorEntry &&
49
+ !cursorEntry.stubbed &&
50
+ cursorEntry.uploaded_object_key &&
51
+ held.some((remote) => remote.object_key === cursorEntry.uploaded_object_key) &&
52
+ cursorEntry.uploaded_byte_size === file.byte_size &&
53
+ cursorEntry.file_hash_sha256 &&
54
+ cursorEntry.file_hash_sha256 === (await input.facts.rawSha256())) {
55
+ return full("unchanged_since_upload");
56
+ }
57
+ const sanitized = await input.facts.sanitized();
58
+ if (sanitized) {
59
+ if (held.some((remote) => remote.content_hash_sha256 === sanitized.sha256))
60
+ return full("content_hash_match");
61
+ if (best === null)
62
+ return partial("remote_size_unknown");
63
+ if (best >= sanitized.byte_size)
64
+ return full("remote_at_least_sanitized_size");
65
+ }
66
+ else {
67
+ if (best === null)
68
+ return partial("remote_size_unknown");
69
+ if (best >= file.byte_size)
70
+ return full("remote_at_least_file_size");
71
+ }
72
+ const grewAfterUpload = cursorEntry?.uploaded_object_key != null &&
73
+ cursorEntry.uploaded_byte_size != null &&
74
+ cursorEntry.uploaded_byte_size < file.byte_size;
75
+ return partial(grewAfterUpload ? "changed_since_upload" : "remote_smaller");
76
+ }
77
+ /**
78
+ * Reads the file at most once, on first ask, and redacts it exactly as the
79
+ * upload does (`raw-evidence-transcript-file.ts`): same sanitizer, same
80
+ * secret-like-name rule, same byte cap.
81
+ */
82
+ export function fileFacts(file, maxBytes = RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES) {
83
+ let bytes = null;
84
+ const read = () => {
85
+ bytes ??= file.byte_size > maxBytes ? Promise.resolve(null) : fs.readFile(file.file_path);
86
+ return bytes;
87
+ };
88
+ let rawHash = null;
89
+ let sanitized = null;
90
+ return {
91
+ rawSha256() {
92
+ rawHash ??= read().then(async (buffer) => (buffer ? sha256(buffer) : await streamSha256(file.file_path)));
93
+ return rawHash;
94
+ },
95
+ sanitized() {
96
+ sanitized ??= read().then((buffer) => {
97
+ if (!buffer)
98
+ return null;
99
+ const result = sanitizeTextEvidenceForUpload({
100
+ text: buffer.toString("utf8"),
101
+ originalBytes: buffer,
102
+ redactedFields: [`${file.kind === "codex_rollout" ? "codex_jsonl" : file.kind === "claude_main" ? "claude_jsonl" : "claude_jsonl_sidecar"}.body`],
103
+ secretLikeFileName: isSecretLikePath(path.basename(file.file_path)),
104
+ });
105
+ const hash = sha256(result.bytes);
106
+ if (hash === REDACTION_STUB_SHA256)
107
+ return null;
108
+ return { sha256: hash, byte_size: result.bytes.byteLength };
109
+ });
110
+ return sanitized;
111
+ },
112
+ };
113
+ }
114
+ function sha256(buffer) {
115
+ return crypto.createHash("sha256").update(buffer).digest("hex");
116
+ }
117
+ async function streamSha256(filePath) {
118
+ const hash = crypto.createHash("sha256");
119
+ const handle = await fs.open(filePath, "r");
120
+ try {
121
+ for await (const chunk of handle.createReadStream())
122
+ hash.update(chunk);
123
+ }
124
+ finally {
125
+ await handle.close().catch(() => undefined);
126
+ }
127
+ return hash.digest("hex");
128
+ }
@@ -8,6 +8,13 @@ import { DEFAULT_DASHBOARD_URL, getCollectorRuntimePaths, readLocalCollectorConf
8
8
  /** At most this many stubbed sessions are re-offered per tick (one backfill batch). */
9
9
  export const SYNC_STUB_SESSIONS_PER_TICK = 25;
10
10
  const DAY_MS = 24 * 60 * 60_000;
11
+ /**
12
+ * BLI-11119: a stubbed session a re-offer could not find in any root, by
13
+ * content id or file hash, is looked for again after this long. Without it
14
+ * every tick rescanned (and hashed) every transcript back to the session's
15
+ * date for a file that is gone.
16
+ */
17
+ export const STUB_NOT_FOUND_RETRY_MS = DAY_MS;
11
18
  const DEFAULT_DEPS = { runBackfill, loadPlan: loadStubReofferPlan };
12
19
  export async function healStubsDuringTick(command, io, deps = DEFAULT_DEPS, now = new Date()) {
13
20
  const summary = {
@@ -17,6 +24,7 @@ export async function healStubsDuringTick(command, io, deps = DEFAULT_DEPS, now
17
24
  stubs_remaining: 0,
18
25
  sessions_local: 0,
19
26
  sessions_selected: 0,
27
+ sessions_not_found_recently: 0,
20
28
  };
21
29
  try {
22
30
  summary.reason = await runStubPass(command, io, deps, now, summary);
@@ -54,13 +62,18 @@ async function runStubPass(command, io, deps, now, summary) {
54
62
  summary.sessions_local = local.length;
55
63
  if (local.length === 0)
56
64
  return "stubs_not_on_this_machine";
57
- const selected = local.slice(0, SYNC_STUB_SESSIONS_PER_TICK);
65
+ const lookable = local.filter((session) => !notFoundRecently(session.notFoundAt, now));
66
+ summary.sessions_not_found_recently = local.length - lookable.length;
67
+ if (lookable.length === 0)
68
+ return "stubs_not_found_recently";
69
+ const selected = lookable.slice(0, SYNC_STUB_SESSIONS_PER_TICK);
58
70
  summary.sessions_selected = selected.length;
59
71
  const trimmed = {
60
72
  ...plan,
61
73
  sessions: new Map(selected.map((session) => [session.key, session.pointerIds])),
62
74
  stub_count: selected.reduce((total, session) => total + session.pointerIds.length, 0),
63
75
  marked_local_sessions: 0,
76
+ found_by: new Map(),
64
77
  };
65
78
  const window = stubWindow(selected, sessionFile.paired_at, now);
66
79
  const result = await deps.runBackfill({
@@ -90,11 +103,18 @@ async function locallyTrackedStubSessions(paths, plan) {
90
103
  const at = key.indexOf(":");
91
104
  const source = key.slice(0, at);
92
105
  const entry = cursors[source]?.sessions[key.slice(at + 1)];
93
- if (entry)
94
- local.push({ key, pointerIds, mtimeMs: entry.file_mtime_ms });
106
+ if (entry) {
107
+ local.push({ key, pointerIds, mtimeMs: entry.file_mtime_ms, notFoundAt: entry.stub_reoffer?.not_found_at ?? null });
108
+ }
95
109
  }
96
110
  return local;
97
111
  }
112
+ function notFoundRecently(notFoundAt, now) {
113
+ if (!notFoundAt)
114
+ return false;
115
+ const at = Date.parse(notFoundAt);
116
+ return Number.isFinite(at) && now.getTime() - at < STUB_NOT_FOUND_RETRY_MS;
117
+ }
98
118
  /**
99
119
  * The narrowest backfill window that still reaches every selected session:
100
120
  * back to the oldest one's file time plus a day. A backfill window never
@@ -196,6 +196,7 @@ function parseSessionEntry(value) {
196
196
  uploaded_byte_size: optionalNumberOrNull(record["uploaded_byte_size"]),
197
197
  last_seen_at: lastSeenAt,
198
198
  ...parseStubReoffer(record["stub_reoffer"]),
199
+ ...(record["stubbed"] === true ? { stubbed: true } : {}),
199
200
  };
200
201
  }
201
202
  function parseStubReoffer(value) {
@@ -208,7 +209,15 @@ function parseStubReoffer(value) {
208
209
  : [];
209
210
  if (record["reason"] !== "stub_reoffered" || !markedAt || ids.length === 0)
210
211
  return {};
211
- return { stub_reoffer: { reason: "stub_reoffered", stub_pointer_ids: ids, marked_at: markedAt } };
212
+ const notFoundAt = optionalString(record["not_found_at"]);
213
+ return {
214
+ stub_reoffer: {
215
+ reason: "stub_reoffered",
216
+ stub_pointer_ids: ids,
217
+ marked_at: markedAt,
218
+ ...(notFoundAt ? { not_found_at: notFoundAt } : {}),
219
+ },
220
+ };
212
221
  }
213
222
  function parseCheckpoints(value) {
214
223
  if (!value || typeof value !== "object")
@@ -0,0 +1,39 @@
1
+ /**
2
+ * A redaction-crash stub is a promise to come back, not a delivery (BLI-11119).
3
+ *
4
+ * When the redactor crashed, the collector uploaded the 42-byte marker
5
+ * `REDACTION_STUB_TEXT` in place of the transcript, and the cursor entry then
6
+ * recorded the ORIGINAL file size as uploaded. Nothing local ever considered
7
+ * the session undelivered. This module is the one place a cursor entry is told
8
+ * the truth about a stub: `stubbed: true`, `uploaded_byte_size` the marker's
9
+ * 42 bytes, and `byte_offset` 0 because no session byte is durable.
10
+ *
11
+ * Two ways a stub is recognised, both deterministic:
12
+ * - at write time, by the object key: every transcript key's last segment
13
+ * starts with the first 16 hex of its content hash (a re-keyed pointer keeps
14
+ * that prefix), and the marker always hashes to `REDACTION_STUB_SHA256`;
15
+ * - after the fact, by the stubs door's answer, which is the source of truth
16
+ * for every entry written before this rule existed.
17
+ */
18
+ import { REDACTION_STUB_BYTE_SIZE, REDACTION_STUB_SHA256, } from "@bli-cockpit/telemetry-core";
19
+ const STUB_KEY_PREFIX = REDACTION_STUB_SHA256.slice(0, 16);
20
+ /** True when this object key names the redaction-crash marker, not a transcript. */
21
+ export function isRedactionStubObjectKey(objectKey) {
22
+ if (!objectKey)
23
+ return false;
24
+ const segment = objectKey.slice(objectKey.lastIndexOf("/") + 1);
25
+ return segment.startsWith(STUB_KEY_PREFIX);
26
+ }
27
+ /**
28
+ * Make the entry say what the server holds: a stub. Returns whether anything
29
+ * changed, so a caller reconciling old entries can count what it corrected.
30
+ */
31
+ export function markCursorEntryStubbed(entry) {
32
+ const changed = entry.stubbed !== true ||
33
+ entry.uploaded_byte_size !== REDACTION_STUB_BYTE_SIZE ||
34
+ entry.byte_offset !== 0;
35
+ entry.stubbed = true;
36
+ entry.uploaded_byte_size = REDACTION_STUB_BYTE_SIZE;
37
+ entry.byte_offset = 0;
38
+ return changed;
39
+ }
@@ -0,0 +1,118 @@
1
+ /**
2
+ * Asks `POST /api/ambient/evidence/transcripts` for every current transcript
3
+ * object this operator owns, and groups the answer by local transcript file
4
+ * (BLI-11122). `cockpit sources check` compares each group to the file on disk.
5
+ *
6
+ * Never throws. A door that cannot be asked (an older dashboard answers 404, a
7
+ * network error, a malformed page) returns `unavailable` with the reason and
8
+ * the check says so instead of guessing.
9
+ *
10
+ * Bounded pages of up to 500, walked by the server's keyset cursor, at most
11
+ * `maxPages` of them. The result says whether the list was exhausted, capped by
12
+ * `maxPages`, or cut short by a failed page after some landed, and carries the
13
+ * requested, returned and unique counts so completeness is a number, not a claim.
14
+ */
15
+ import { EVIDENCE_TRANSCRIPTS_MAX_PAGE, EVIDENCE_TRANSCRIPTS_SCHEMA_VERSION, EvidenceTranscriptsResponseSchema, transcriptFileFromObjectKey, transcriptFileKey, } from "@bli-cockpit/telemetry-core";
16
+ import { describeError } from "./health-detail.js";
17
+ const TRANSCRIPTS_PAGE_TIMEOUT_MS = 30_000;
18
+ /** 400 pages of 500 is 200,000 current transcript objects: far past any operator today. */
19
+ export const TRANSCRIPTS_DEFAULT_MAX_PAGES = 400;
20
+ export async function fetchTranscriptObjects(options) {
21
+ const maxPages = options.maxPages ?? TRANSCRIPTS_DEFAULT_MAX_PAGES;
22
+ const pageSize = Math.min(options.pageSize ?? EVIDENCE_TRANSCRIPTS_MAX_PAGE, EVIDENCE_TRANSCRIPTS_MAX_PAGE);
23
+ const files = new Map();
24
+ const seen = new Set();
25
+ let returned = 0;
26
+ let unkeyed = 0;
27
+ let pages = 0;
28
+ let after = null;
29
+ for (;;) {
30
+ if (pages >= maxPages)
31
+ return finish("capped", "ok");
32
+ const page = await requestPage({ ...options, after, limit: pageSize });
33
+ if (!page.ok)
34
+ return finish(pages === 0 ? "unavailable" : "failed_partial", page.reason);
35
+ pages += 1;
36
+ for (const object of page.objects) {
37
+ returned += 1;
38
+ if (seen.has(object.ref_id))
39
+ continue;
40
+ seen.add(object.ref_id);
41
+ addObject(object);
42
+ }
43
+ // Advance by what the server returned; a page that names no cursor is the last.
44
+ if (!page.nextAfter)
45
+ return finish("exhausted", "ok");
46
+ after = page.nextAfter;
47
+ }
48
+ function addObject(object) {
49
+ const identity = transcriptFileFromObjectKey(object.object_key);
50
+ if (!identity) {
51
+ unkeyed += 1;
52
+ return;
53
+ }
54
+ const key = transcriptFileKey(identity);
55
+ const list = files.get(key) ?? [];
56
+ list.push({
57
+ object_key: object.object_key,
58
+ byte_size: object.byte_size,
59
+ content_hash_sha256: object.content_hash_sha256,
60
+ stub: object.stub,
61
+ });
62
+ files.set(key, list);
63
+ }
64
+ function finish(status, reason) {
65
+ const result = {
66
+ status,
67
+ reason,
68
+ pages,
69
+ requested: pages * pageSize,
70
+ returned,
71
+ unique: seen.size,
72
+ unkeyed,
73
+ files,
74
+ };
75
+ console.error("[collector sources] transcript objects listed", JSON.stringify({
76
+ reason: status === "unavailable" ? `transcript_door_unavailable:${reason}` : `transcripts_${status}`,
77
+ pages,
78
+ requested: result.requested,
79
+ returned,
80
+ unique: seen.size,
81
+ unkeyed,
82
+ file_count: files.size,
83
+ ...(status === "failed_partial" ? { failure: reason } : {}),
84
+ }));
85
+ return result;
86
+ }
87
+ }
88
+ async function requestPage(options) {
89
+ const controller = new AbortController();
90
+ const timeout = setTimeout(() => controller.abort(), TRANSCRIPTS_PAGE_TIMEOUT_MS);
91
+ try {
92
+ const response = await options.fetchImpl(`${options.dashboardUrl}/api/ambient/evidence/transcripts`, {
93
+ method: "POST",
94
+ headers: {
95
+ "Content-Type": "application/json",
96
+ Authorization: `Bearer ${options.deviceToken}`,
97
+ },
98
+ body: JSON.stringify({
99
+ schema_version: EVIDENCE_TRANSCRIPTS_SCHEMA_VERSION,
100
+ after: options.after,
101
+ limit: options.limit,
102
+ }),
103
+ signal: controller.signal,
104
+ });
105
+ if (!response.ok)
106
+ return { ok: false, reason: `http_${response.status}` };
107
+ const parsed = EvidenceTranscriptsResponseSchema.safeParse(await response.json().catch(() => null));
108
+ if (!parsed.success)
109
+ return { ok: false, reason: "invalid_response" };
110
+ return { ok: true, objects: parsed.data.objects, nextAfter: parsed.data.next_after };
111
+ }
112
+ catch (error) {
113
+ return { ok: false, reason: describeError(error).error_name };
114
+ }
115
+ finally {
116
+ clearTimeout(timeout);
117
+ }
118
+ }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@bli-cockpit/cli",
3
- "version": "0.2.157",
3
+ "version": "0.2.159",
4
4
  "private": false,
5
5
  "type": "module",
6
6
  "bin": {
@@ -27,8 +27,8 @@
27
27
  "test": "node dist/cli.js --help && node ../../scripts/assert-public-cli-routing.mjs && node ../../scripts/assert-public-cli-verb-help.mjs && node ../../scripts/assert-public-cli-surface-doors.mjs && node ../../scripts/assert-public-cli-runtime-files.mjs && node ../../scripts/assert-public-cli-exit-contract.mjs && node ../../scripts/assert-public-cli-no-fleet-posts.mjs && node ../../scripts/assert-public-cli-doctor-json.mjs && node ../../scripts/assert-public-package-pack.mjs --workspace=@bli-cockpit/cli"
28
28
  },
29
29
  "dependencies": {
30
- "@bli-cockpit/memory-mcp": "0.1.42",
31
- "@bli-cockpit/mcp": "0.1.76",
32
- "@bli-cockpit/telemetry-core": "0.1.54"
30
+ "@bli-cockpit/memory-mcp": "0.1.43",
31
+ "@bli-cockpit/mcp": "0.1.77",
32
+ "@bli-cockpit/telemetry-core": "0.1.55"
33
33
  }
34
34
  }