@bli-cockpit/cli 0.2.158 → 0.2.159

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,27 @@
1
+ /**
2
+ * `cockpit sources check [--older-than <days>] [--json]` (BLI-11122).
3
+ *
4
+ * One verb, read-only. There is deliberately no `sources clean` and no
5
+ * `--apply`: Tower never deletes a person's raw transcripts; deleting one is
6
+ * the person's own act, outside Tower. A path is never accepted: the roots are
7
+ * the ones the collector reads (`sources-discovery.ts`).
8
+ */
9
+ import { optionalNonEmpty, optionalPositiveInteger, optionalUrl, parseNamedArgs, } from "./local-arg-values.js";
10
+ export function parseSourcesArgs(args) {
11
+ const values = parseNamedArgs(args, {
12
+ allowedFlags: ["--older-than", "--home", "--dashboard-url", "--json"],
13
+ valueFlags: ["--older-than", "--home", "--dashboard-url"],
14
+ });
15
+ const action = values.positionals[0] ?? "check";
16
+ if (action !== "check" || values.positionals.length > 1) {
17
+ throw new Error("sources has one verb: `cockpit sources check [--older-than <days>] [--json]`.");
18
+ }
19
+ return {
20
+ kind: "sources",
21
+ action: "check",
22
+ olderThanDays: optionalPositiveInteger(values.flags.get("--older-than"), "--older-than") ?? null,
23
+ homeDir: optionalNonEmpty(values.flags.get("--home")),
24
+ dashboardUrl: optionalUrl(values.flags.get("--dashboard-url")),
25
+ json: values.booleans.has("--json"),
26
+ };
27
+ }
@@ -26,4 +26,5 @@
26
26
  */
27
27
  export { parseOnboardArgs, parseUpdateArgs, parseDoctorArgs, parseInstallArgs, parseReleaseArgs, parseLoginArgs, parseLogoutArgs, } from "./local-args-collector-setup.js";
28
28
  export { parseStartArgs, parseSyncArgs, parseAnalyzeArgs, parseBackfillArgs, } from "./local-args-collector-work.js";
29
- export { parseStatusArgs, parseSessionsArgs, parseServeArgs, parseAutostartArgs, parseAgentRulesArgs, parseMemoryArgs, parseCleanArgs, } from "./local-args-collector-status.js";
29
+ export { parseStatusArgs, parseSessionsArgs, parseServeArgs, parseAutostartArgs, parseAgentRulesArgs, parseMemoryArgs, parseCleanArgs, } from "./local-args-collector-status.js";
30
+ export { parseSourcesArgs } from "./local-args-collector-sources.js";
@@ -17,7 +17,7 @@
17
17
  */
18
18
  import { parseRootsArgs } from "./roots.js";
19
19
  import { SURFACE_ALIASES } from "./local-help.js";
20
- import { parseAgentRulesArgs, parseAnalyzeArgs, parseAutostartArgs, parseBackfillArgs, parseCleanArgs, parseDoctorArgs, parseInstallArgs, parseLoginArgs, parseMemoryArgs, parseLogoutArgs, parseOnboardArgs, parseReleaseArgs, parseServeArgs, parseSessionsArgs, parseStartArgs, parseStatusArgs, parseSyncArgs, parseUpdateArgs, } from "./local-args-collector.js";
20
+ import { parseAgentRulesArgs, parseAnalyzeArgs, parseAutostartArgs, parseBackfillArgs, parseCleanArgs, parseDoctorArgs, parseInstallArgs, parseLoginArgs, parseMemoryArgs, parseLogoutArgs, parseOnboardArgs, parseReleaseArgs, parseServeArgs, parseSessionsArgs, parseSourcesArgs, parseStartArgs, parseStatusArgs, parseSyncArgs, parseUpdateArgs, } from "./local-args-collector.js";
21
21
  import { parseBriefArgs, parseCorrectArgs, parseDocsArgs, parseIssueArgs, parseJarvisArgs, parseMailArgs, parseCalArgs, parseModelArgs, parseMsgArgs, parseNotesArgs, parseOpsArgs, parseProjectArgs, parseModelsArgs, parseScoutArgs, parseSearchArgs, parseSettingsArgs, parseSlackArgs, parseTeamArgs, parseWorkbookArgs, parseUsageArgs, parseCareersArgs, } from "./local-args-tower.js";
22
22
  // `normalizeUrl` has always been part of this module's surface — `local.ts` and
23
23
  // `local-auth.ts` import it from here — so it stays exported from this address
@@ -107,6 +107,8 @@ export function parseLocalArgs(argv) {
107
107
  return parseMemoryArgs(argv.slice(1));
108
108
  case "clean":
109
109
  return parseCleanArgs(argv.slice(1));
110
+ case "sources":
111
+ return parseSourcesArgs(argv.slice(1));
110
112
  case "docs":
111
113
  return parseDocsArgs(argv.slice(1));
112
114
  case "msg":
@@ -366,6 +366,20 @@ export function localSubcommandHelp(command) {
366
366
  "laptop that has been off.",
367
367
  ],
368
368
  ],
369
+ [
370
+ "sources",
371
+ [
372
+ "Usage: cockpit sources check [--older-than <days>] [--dashboard-url <url>] [--json]",
373
+ "",
374
+ "Answers, for every Codex and Claude transcript on this machine, whether Tower",
375
+ "holds it: stored in full, stored in part, or not stored, with the reason.",
376
+ "--older-than <days> checks only files not modified in that many days.",
377
+ "--json lists every file with its path, verdict and reason.",
378
+ "Read-only. Tower never deletes these files; deleting one is your own act.",
379
+ "Before deleting any transcript by hand, run this and delete only files it",
380
+ "answers \"stored in full\". See docs/runbooks/freeing-disk-on-a-collector-machine.md.",
381
+ ],
382
+ ],
369
383
  [
370
384
  "release",
371
385
  [
@@ -65,6 +65,7 @@ export const rootCommandNames = new Set([
65
65
  "agent-rules",
66
66
  "memory",
67
67
  "clean",
68
+ "sources",
68
69
  "docs",
69
70
  "msg",
70
71
  "issue",
@@ -117,6 +118,7 @@ export function localCommandHelp(command) {
117
118
  " cockpit agent-rules [install|uninstall|status] [--host codex|claude|all] [--workspace <path>] [--json]",
118
119
  " cockpit memory [search \"<question>\"|save|update <id>|forget <id>|profile|install|status|log <win|loss|noise> --store <bli|supermemory|both> \"<reason>\"] [--container <tag>] [--limit <n>] [--no-recent] [--custom-id <id>] [--extract] [--content-stdin] [--reason <label>] [--file <path>] [--dashboard-url <url>] [--dry-run] [--json]",
119
120
  " cockpit clean [--dry-run] [--all-committed] [--reconcile] [--dashboard-url <url>] [--json]",
121
+ " cockpit sources check [--older-than <days>] [--dashboard-url <url>] [--json]",
120
122
  " cockpit docs [list|tree|read <id|slug>|create --title <t>|update <id>|delete <id|slug>] [--parent <id>|root|--clear-parent] [--query <text> [--word]] [--limit <n>] [--visibility org|private] [--file <path>|--body-stdin] [--allow-empty] [--dashboard-url <url>] [--json]",
121
123
  " cockpit msg [channels|create <name>|dm <email>|read <channel>|send <channel>|thread <id> --channel <channel>] [--private] [--members a@x,b@y] [--description <text>] [--thread <id>] [--limit <n>] [--dashboard-url <url>] [--json]",
122
124
  " cockpit issue [list|show <BLI-id>|create --title <t>|update <BLI-id>|move <BLI-id> <state>|comment <BLI-id>|history <BLI-id>] [--state <s>] [--assignee me|unassigned|<uuid>] [--project <name|id>] [--limit <n>] [--priority 0-4] [--size S|M|L] [--parent <BLI-id>] [--file <path>|--body-stdin] [--dashboard-url <url>] [--json]",
@@ -37,6 +37,7 @@ import { runMemoryLog } from "./memory-log.js";
37
37
  import { runMemoryInstall } from "./memory-install.js";
38
38
  import { runMemoryStore, runMemoryVerbList } from "./memory-store.js";
39
39
  import { runClean } from "./clean.js";
40
+ import { runSourcesCheck } from "./sources-check.js";
40
41
  import { runDocs } from "./docs.js";
41
42
  import { runMsg } from "./msg.js";
42
43
  import { runIssue } from "./issue.js";
@@ -213,6 +214,8 @@ export async function runLocalCockpitCli(argv, io = defaultIo()) {
213
214
  return await runMemoryStore(command, io);
214
215
  case "clean":
215
216
  return await runClean(command, io);
217
+ case "sources":
218
+ return await runSourcesCheck(command, io);
216
219
  case "docs":
217
220
  return await runDocs(command, io);
218
221
  case "msg":
@@ -21,7 +21,7 @@ export async function runCockpitCli(argv, io) {
21
21
  }
22
22
 
23
23
  if (command === "--version" || command === "-V" || command === "version") {
24
- writeLine(io?.stdout ?? process.stdout, "0.2.158");
24
+ writeLine(io?.stdout ?? process.stdout, "0.2.159");
25
25
  return 0;
26
26
  }
27
27
 
@@ -0,0 +1,135 @@
1
+ import { writeLine } from "./cli-io.js";
2
+ import { CLAUDE_CURSOR_FILENAME, readRawEvidenceCursor, } from "../cursors/raw-evidence-cursor.js";
3
+ import { fetchTranscriptObjects } from "../evidence-transcripts-client.js";
4
+ import { DEFAULT_DASHBOARD_URL, getCollectorRuntimePaths, readLocalCollectorConfig, readLocalCollectorSessionFile, } from "../local-state.js";
5
+ import { discoverLocalTranscripts } from "./sources-discovery.js";
6
+ import { fileFacts, judgeSource } from "./sources-verdict.js";
7
+ const VERDICTS = ["stored_full", "stored_partial", "not_stored"];
8
+ const DAY_MS = 24 * 60 * 60 * 1000;
9
+ export async function runSourcesCheck(command, io, now = new Date()) {
10
+ const startedAt = Date.now();
11
+ const paths = getCollectorRuntimePaths(command.homeDir);
12
+ const config = await readLocalCollectorConfig(paths).catch(() => null);
13
+ const dashboardUrl = command.dashboardUrl ?? config?.dashboard_url ?? DEFAULT_DASHBOARD_URL;
14
+ const session = await readLocalCollectorSessionFile(paths).catch(() => null);
15
+ if (!session || session.session_state !== "valid" || typeof session.device_token !== "string" || !session.device_token) {
16
+ console.error("[collector sources] check not run", JSON.stringify({ reason: "collector_not_paired" }));
17
+ return fail(io, command, "collector_not_paired", "This machine is not signed in to Tower, so Tower cannot be asked. Run `cockpit doctor`.");
18
+ }
19
+ const remote = await fetchTranscriptObjects({ dashboardUrl, deviceToken: session.device_token, fetchImpl: io.fetch });
20
+ if (remote.status === "unavailable") {
21
+ console.error("[collector sources] check not answered", JSON.stringify({ reason: `transcript_door_unavailable:${remote.reason}` }));
22
+ return fail(io, command, `transcript_door_unavailable:${remote.reason}`, `Tower could not be asked which transcripts it holds (${remote.reason}). No file is answered; nothing here means "not stored".`);
23
+ }
24
+ const homeDir = command.homeDir ?? paths.home_dir;
25
+ const olderThanMs = command.olderThanDays === null ? null : now.getTime() - command.olderThanDays * DAY_MS;
26
+ const local = await discoverLocalTranscripts({ homeDir, olderThanMs });
27
+ const cursors = {
28
+ codex: await readRawEvidenceCursor(paths),
29
+ claude_code: await readRawEvidenceCursor(paths, { filename: CLAUDE_CURSOR_FILENAME, sessionsOnly: true }),
30
+ };
31
+ const report = emptyReport(command, remote, local.newer_count, local.file_read_failed + local.directory_read_failed);
32
+ for (const file of local.files) {
33
+ const judged = await judgeSource({
34
+ file,
35
+ remotes: remote.files.get(file.key) ?? [],
36
+ cursorEntry: file.kind === "claude_subagent" ? null : (cursors[file.source].sessions[file.session_id] ?? null),
37
+ facts: fileFacts(file),
38
+ }).catch((error) => {
39
+ report.local.unreadable += 1;
40
+ console.error("[collector sources] a transcript could not be judged; left out of the answer", JSON.stringify({ reason: "file_unreadable", kind: file.kind, byte_size: file.byte_size, error_name: error?.name ?? "Error" }));
41
+ return null;
42
+ });
43
+ if (!judged)
44
+ continue;
45
+ report.local.examined += 1;
46
+ report.counts[judged.verdict].files += 1;
47
+ report.counts[judged.verdict].bytes += file.byte_size;
48
+ report.reasons[judged.reason] = (report.reasons[judged.reason] ?? 0) + 1;
49
+ report.files.push({
50
+ path: file.file_path,
51
+ source: file.source,
52
+ kind: file.kind,
53
+ session_id: file.session_id,
54
+ byte_size: file.byte_size,
55
+ modified_at: new Date(file.mtime_ms).toISOString(),
56
+ verdict: judged.verdict,
57
+ reason: judged.reason,
58
+ remote_byte_size: judged.remote_byte_size,
59
+ });
60
+ }
61
+ report.files.sort((a, b) => VERDICTS.indexOf(a.verdict) - VERDICTS.indexOf(b.verdict) || a.path.localeCompare(b.path));
62
+ // Counts and bytes only: never a path, a session id or a byte of content.
63
+ console.error("[collector sources] check", JSON.stringify({
64
+ reason: `sources_check_${report.status}`,
65
+ older_than_days: report.older_than_days,
66
+ remote_status: remote.status,
67
+ remote_unique: remote.unique,
68
+ examined: report.local.examined,
69
+ newer_skipped: report.local.newer_skipped,
70
+ unreadable: report.local.unreadable,
71
+ stored_full: report.counts.stored_full,
72
+ stored_partial: report.counts.stored_partial,
73
+ not_stored: report.counts.not_stored,
74
+ reasons: report.reasons,
75
+ elapsed_ms: Date.now() - startedAt,
76
+ }));
77
+ if (command.json)
78
+ writeLine(io.stdout, JSON.stringify(report, null, 2));
79
+ else
80
+ for (const line of sourcesCheckLines(report))
81
+ writeLine(io.stdout, line);
82
+ return 0;
83
+ }
84
+ function emptyReport(command, remote, newerSkipped, unreadable) {
85
+ const { files, ...remoteCounts } = remote;
86
+ return {
87
+ status: remote.status === "exhausted" ? "complete" : "incomplete",
88
+ reason: remote.status === "exhausted" ? "ok" : `transcript_list_${remote.status}${remote.reason === "ok" ? "" : `:${remote.reason}`}`,
89
+ older_than_days: command.olderThanDays,
90
+ remote: { ...remoteCounts, file_count: files.size },
91
+ local: { examined: 0, newer_skipped: newerSkipped, unreadable },
92
+ counts: {
93
+ stored_full: { files: 0, bytes: 0 },
94
+ stored_partial: { files: 0, bytes: 0 },
95
+ not_stored: { files: 0, bytes: 0 },
96
+ },
97
+ reasons: {},
98
+ files: [],
99
+ };
100
+ }
101
+ function fail(io, command, reason, sentence) {
102
+ if (command.json) {
103
+ writeLine(io.stdout, JSON.stringify({ status: "unavailable", reason, older_than_days: command.olderThanDays }, null, 2));
104
+ }
105
+ else {
106
+ writeLine(io.stderr, sentence);
107
+ }
108
+ return 1;
109
+ }
110
+ /** What a person reads: counts and bytes per verdict, then why. Paths only with --json. */
111
+ export function sourcesCheckLines(report) {
112
+ const scope = report.older_than_days === null ? "every transcript" : `transcripts not modified in ${report.older_than_days} days`;
113
+ const lines = [
114
+ `Tower sources check: ${scope} on this machine`,
115
+ ` stored in full ${report.counts.stored_full.files} file(s), ${mb(report.counts.stored_full.bytes)} MB`,
116
+ ` stored in part ${report.counts.stored_partial.files} file(s), ${mb(report.counts.stored_partial.bytes)} MB`,
117
+ ` not stored ${report.counts.not_stored.files} file(s), ${mb(report.counts.not_stored.bytes)} MB`,
118
+ ];
119
+ const reasons = Object.entries(report.reasons).sort((a, b) => b[1] - a[1]);
120
+ if (reasons.length > 0)
121
+ lines.push(` why ${reasons.map(([reason, count]) => `${reason} ${count}`).join(", ")}`);
122
+ if (report.local.newer_skipped > 0)
123
+ lines.push(` newer ${report.local.newer_skipped} file(s) modified inside the window, not checked`);
124
+ if (report.local.unreadable > 0)
125
+ lines.push(` unreadable ${report.local.unreadable} file(s) could not be read and are not answered`);
126
+ lines.push(` Tower's list ${report.remote.status}: ${report.remote.unique} current transcript object(s) over ${report.remote.pages} page(s)`);
127
+ if (report.status === "incomplete") {
128
+ lines.push(` INCOMPLETE Tower's list was cut short (${report.reason}); "not stored" and "stored in part" may be understated. Run it again.`);
129
+ }
130
+ lines.push("Tower never deletes these files. `--json` lists every file with its path and verdict. Only a \"stored in full\" file is safe to delete by your own hand; see docs/runbooks/freeing-disk-on-a-collector-machine.md.");
131
+ return lines;
132
+ }
133
+ function mb(bytes) {
134
+ return (bytes / (1024 * 1024)).toFixed(1);
135
+ }
@@ -0,0 +1,232 @@
1
+ /**
2
+ * Every agent transcript on this machine, named the way the collector names
3
+ * its upload (BLI-11122).
4
+ *
5
+ * `cockpit sources check` has to pair each local file with the object Tower
6
+ * holds for it, and an object key carries the session id the collector read
7
+ * from the transcript's CONTENT, not its file name (a resumed or forked Codex
8
+ * rollout is filed under one id and names another; see
9
+ * `backfill-stub-locate.ts`). So each file is named by the same rules the
10
+ * readers use, kept here as pure functions so both path layouts (macOS and
11
+ * Windows) are testable without either file system:
12
+ *
13
+ * - Codex (`~/.codex/sessions/**`, `~/.codex/archived_sessions/**`): the first
14
+ * `session_meta` record's `payload.id`, else the uuid in the file name, else
15
+ * a hash of the path (`codex-attribution.ts`).
16
+ * - Claude main (`~/.claude/projects/<project>/<uuid>.jsonl`): the first
17
+ * record's `sessionId`, else the file name's uuid (`claude-attribution-score.ts`).
18
+ * - Claude subagent (`<project>/<uuid>/subagents/agent-*.jsonl`): the main's
19
+ * session id plus the file stem (`raw-evidence-claude-reader.ts`).
20
+ *
21
+ * This module reads files and never writes, moves or deletes one.
22
+ */
23
+ import fs from "node:fs/promises";
24
+ import path from "node:path";
25
+ import { createReadStream } from "node:fs";
26
+ import readline from "node:readline";
27
+ import { transcriptFileKey } from "@bli-cockpit/telemetry-core";
28
+ import { SESSION_FILE_UUID_PATTERN, sanitizeSessionId, sessionIdFromFileName, shortHash, } from "../adapters/attribution-core.js";
29
+ import { isSecretLikePath, safeKeySegment } from "../adapters/raw-evidence-keys.js";
30
+ import { describeError } from "../health-detail.js";
31
+ export function sourcesRoots(homeDir, pathApi = path) {
32
+ return {
33
+ codex: [
34
+ pathApi.join(homeDir, ".codex", "sessions"),
35
+ pathApi.join(homeDir, ".codex", "archived_sessions"),
36
+ ],
37
+ claudeProjects: pathApi.join(homeDir, ".claude", "projects"),
38
+ };
39
+ }
40
+ /** Which kind of transcript a path is, or null for anything the collector does not upload as one. */
41
+ export function classifyTranscriptPath(filePath, roots, pathApi = path) {
42
+ if (!filePath.toLowerCase().endsWith(".jsonl"))
43
+ return null;
44
+ for (const root of roots.codex) {
45
+ const rel = pathApi.relative(root, filePath);
46
+ if (rel && !rel.startsWith("..") && !pathApi.isAbsolute(rel))
47
+ return { kind: "codex_rollout" };
48
+ }
49
+ const rel = pathApi.relative(roots.claudeProjects, filePath);
50
+ if (!rel || rel.startsWith("..") || pathApi.isAbsolute(rel))
51
+ return null;
52
+ const parts = rel.split(pathApi.sep);
53
+ if (parts.length === 2 && SESSION_FILE_UUID_PATTERN.test(parts[1]))
54
+ return { kind: "claude_main" };
55
+ if (parts.length === 4 && parts[2] === "subagents" && parts[3].startsWith("agent-")) {
56
+ return {
57
+ kind: "claude_subagent",
58
+ main_session_dir: parts[1],
59
+ sidecar_stem: parts[3].replace(/\.jsonl$/i, ""),
60
+ };
61
+ }
62
+ return null;
63
+ }
64
+ /** The session id a reader would key this file's session by, from its content id when it has one. */
65
+ export function sessionIdFor(filePath, contentId, pathApi = path) {
66
+ const fromContent = sanitizeSessionId(contentId ?? undefined);
67
+ if (fromContent)
68
+ return { session_id: fromContent, session_id_source: "content" };
69
+ const fromName = sessionIdFromFileName(pathApi.basename(filePath));
70
+ if (fromName)
71
+ return { session_id: fromName, session_id_source: "file_name" };
72
+ return { session_id: shortHash(filePath), session_id_source: "path_hash" };
73
+ }
74
+ /** The key of the object a file would be uploaded as; see `raw-evidence-*-reader.ts`. */
75
+ export function transcriptKeyFor(source, sessionId, sidecarStem) {
76
+ const identity = {
77
+ source,
78
+ session_id: safeKeySegment(sessionId),
79
+ sidecar: sidecarStem === null ? null : isSecretLikePath(sidecarStem) ? "redacted-file-name" : safeKeySegment(sidecarStem),
80
+ };
81
+ return transcriptFileKey(identity);
82
+ }
83
+ /**
84
+ * The first content id in a transcript: Codex's first `session_meta`
85
+ * `payload.id`, Claude's first `sessionId`. Streams line by line and stops at
86
+ * the first hit; a line that does not parse is skipped, as the readers do.
87
+ */
88
+ export async function readContentSessionId(filePath, source) {
89
+ const stream = createReadStream(filePath, { encoding: "utf8" });
90
+ const lines = readline.createInterface({ input: stream, crlfDelay: Infinity });
91
+ try {
92
+ for await (const line of lines) {
93
+ const id = contentIdFromLine(line, source);
94
+ if (id)
95
+ return id;
96
+ }
97
+ return null;
98
+ }
99
+ finally {
100
+ lines.close();
101
+ stream.destroy();
102
+ }
103
+ }
104
+ export function contentIdFromLine(line, source) {
105
+ if (!line.trim())
106
+ return null;
107
+ let record;
108
+ try {
109
+ record = JSON.parse(line);
110
+ }
111
+ catch {
112
+ return null;
113
+ }
114
+ if (!record || typeof record !== "object" || Array.isArray(record))
115
+ return null;
116
+ const entry = record;
117
+ if (source === "claude_code")
118
+ return trimmed(entry["sessionId"]);
119
+ if (entry["type"] !== "session_meta")
120
+ return null;
121
+ const payload = entry["payload"];
122
+ if (!payload || typeof payload !== "object")
123
+ return null;
124
+ return trimmed(payload["id"]);
125
+ }
126
+ function trimmed(value) {
127
+ return typeof value === "string" && value.trim() ? value.trim() : null;
128
+ }
129
+ /**
130
+ * Walk every root, name every transcript. `olderThanMs` keeps only files whose
131
+ * mtime is before it; the rest are counted, never read.
132
+ */
133
+ export async function discoverLocalTranscripts(options) {
134
+ const roots = sourcesRoots(options.homeDir);
135
+ const result = { files: [], newer_count: 0, directory_read_failed: 0, file_read_failed: 0 };
136
+ const found = [];
137
+ let firstFailure = null;
138
+ const stack = [...roots.codex, roots.claudeProjects];
139
+ const seen = new Set();
140
+ while (stack.length > 0) {
141
+ const dir = stack.pop();
142
+ let entries;
143
+ try {
144
+ entries = await fs.readdir(dir, { withFileTypes: true });
145
+ }
146
+ catch (error) {
147
+ if (error?.code !== "ENOENT") {
148
+ result.directory_read_failed += 1;
149
+ firstFailure ??= describeError(error);
150
+ }
151
+ continue;
152
+ }
153
+ for (const entry of entries) {
154
+ const full = path.join(dir, entry.name);
155
+ if (entry.isDirectory()) {
156
+ stack.push(full);
157
+ continue;
158
+ }
159
+ if (!entry.isFile())
160
+ continue;
161
+ const shape = classifyTranscriptPath(full, roots);
162
+ if (!shape || seen.has(full))
163
+ continue;
164
+ seen.add(full);
165
+ const stat = await fs.stat(full).catch((error) => {
166
+ result.file_read_failed += 1;
167
+ firstFailure ??= describeError(error);
168
+ return null;
169
+ });
170
+ if (!stat)
171
+ continue;
172
+ if (options.olderThanMs !== null && stat.mtimeMs >= options.olderThanMs) {
173
+ result.newer_count += 1;
174
+ continue;
175
+ }
176
+ found.push({ file: full, size: stat.size, mtimeMs: stat.mtimeMs, shape });
177
+ }
178
+ }
179
+ // Claude mains first, so a subagent file can take its main's content id.
180
+ const claudeMainIds = new Map();
181
+ found.sort((a, b) => shapeOrder(a.shape) - shapeOrder(b.shape) || a.file.localeCompare(b.file));
182
+ for (const item of found) {
183
+ const source = item.shape.kind === "codex_rollout" ? "codex" : "claude_code";
184
+ let contentId = null;
185
+ if (item.shape.kind !== "claude_subagent") {
186
+ const read = await readContentSessionId(item.file, source).then((id) => ({ ok: true, id }), (error) => {
187
+ result.file_read_failed += 1;
188
+ firstFailure ??= describeError(error);
189
+ return { ok: false, id: null };
190
+ });
191
+ if (!read.ok)
192
+ continue;
193
+ contentId = read.id;
194
+ }
195
+ let named;
196
+ if (item.shape.kind === "claude_subagent") {
197
+ // The reader keys a subagent file under its MAIN session's id.
198
+ const mainPath = path.join(path.dirname(path.dirname(path.dirname(item.file))), `${item.shape.main_session_dir}.jsonl`);
199
+ const known = claudeMainIds.get(mainPath);
200
+ named = known
201
+ ? { session_id: known, session_id_source: "content" }
202
+ : sessionIdFor(mainPath, await readContentSessionId(mainPath, "claude_code").catch(() => null));
203
+ }
204
+ else {
205
+ named = sessionIdFor(item.file, contentId);
206
+ if (item.shape.kind === "claude_main")
207
+ claudeMainIds.set(item.file, named.session_id);
208
+ }
209
+ result.files.push({
210
+ source,
211
+ kind: item.shape.kind,
212
+ file_path: item.file,
213
+ byte_size: item.size,
214
+ mtime_ms: item.mtimeMs,
215
+ ...named,
216
+ key: transcriptKeyFor(source, named.session_id, item.shape.kind === "claude_subagent" ? item.shape.sidecar_stem : null),
217
+ });
218
+ }
219
+ if (result.directory_read_failed > 0 || result.file_read_failed > 0) {
220
+ console.error("[collector sources] some transcripts could not be read; they are left out of the answer", JSON.stringify({
221
+ reason: "sources_discovery_incomplete",
222
+ directory_read_failed: result.directory_read_failed,
223
+ file_read_failed: result.file_read_failed,
224
+ found_count: result.files.length,
225
+ ...(firstFailure ? { first_failure: firstFailure } : {}),
226
+ }));
227
+ }
228
+ return result;
229
+ }
230
+ function shapeOrder(shape) {
231
+ return shape.kind === "claude_main" ? 0 : shape.kind === "codex_rollout" ? 1 : 2;
232
+ }
@@ -0,0 +1,128 @@
1
+ /**
2
+ * Does Tower hold this transcript, in full, in part, or not at all?
3
+ * (BLI-11122)
4
+ *
5
+ * One file, the current objects Tower lists for it, and this machine's cursor
6
+ * entry for its session. The rules, cheapest first; every answer names why:
7
+ *
8
+ * not_stored no current object for the file (`no_object`), or only the
9
+ * redaction-crash stub (`stub_only`): a stub holds no byte.
10
+ * stored_full the cursor's uploaded object is still current, the file
11
+ * is the size it was when uploaded and its hash is the one the
12
+ * cursor last recorded (`unchanged_since_upload`); or the
13
+ * file, redacted exactly as the upload redacts it, has the
14
+ * hash of a current object (`content_hash_match`) or is no
15
+ * larger than one (`remote_at_least_sanitized_size`).
16
+ * stored_partial a current object exists but is smaller than the redacted
17
+ * file (`changed_since_upload` when the cursor saw the file
18
+ * grow after its upload, else `remote_smaller`), or its size
19
+ * is not recorded (`remote_size_unknown`).
20
+ *
21
+ * A file too large to redact in memory is compared by its raw size
22
+ * (`remote_at_least_file_size`). A file whose redaction would crash to the stub
23
+ * is compared the same way, never by the stub's 42 bytes.
24
+ *
25
+ * Read-only: this module reads the file and never writes, moves or deletes it.
26
+ */
27
+ import crypto from "node:crypto";
28
+ import fs from "node:fs/promises";
29
+ import path from "node:path";
30
+ import { RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES, REDACTION_STUB_SHA256, } from "@bli-cockpit/telemetry-core";
31
+ import { sanitizeTextEvidenceForUpload } from "../adapters/raw-evidence-sanitize.js";
32
+ import { isSecretLikePath } from "../adapters/raw-evidence-keys.js";
33
+ export async function judgeSource(input) {
34
+ const { file, remotes, cursorEntry } = input;
35
+ if (remotes.length === 0)
36
+ return { verdict: "not_stored", reason: "no_object", remote_byte_size: null };
37
+ const held = remotes.filter((remote) => !remote.stub);
38
+ if (held.length === 0)
39
+ return { verdict: "not_stored", reason: "stub_only", remote_byte_size: null };
40
+ const sizes = held.map((remote) => remote.byte_size).filter((size) => size !== null);
41
+ const best = sizes.length > 0 ? Math.max(...sizes) : null;
42
+ const full = (reason) => ({ verdict: "stored_full", reason, remote_byte_size: best });
43
+ const partial = (reason) => ({
44
+ verdict: "stored_partial",
45
+ reason,
46
+ remote_byte_size: best,
47
+ });
48
+ if (cursorEntry &&
49
+ !cursorEntry.stubbed &&
50
+ cursorEntry.uploaded_object_key &&
51
+ held.some((remote) => remote.object_key === cursorEntry.uploaded_object_key) &&
52
+ cursorEntry.uploaded_byte_size === file.byte_size &&
53
+ cursorEntry.file_hash_sha256 &&
54
+ cursorEntry.file_hash_sha256 === (await input.facts.rawSha256())) {
55
+ return full("unchanged_since_upload");
56
+ }
57
+ const sanitized = await input.facts.sanitized();
58
+ if (sanitized) {
59
+ if (held.some((remote) => remote.content_hash_sha256 === sanitized.sha256))
60
+ return full("content_hash_match");
61
+ if (best === null)
62
+ return partial("remote_size_unknown");
63
+ if (best >= sanitized.byte_size)
64
+ return full("remote_at_least_sanitized_size");
65
+ }
66
+ else {
67
+ if (best === null)
68
+ return partial("remote_size_unknown");
69
+ if (best >= file.byte_size)
70
+ return full("remote_at_least_file_size");
71
+ }
72
+ const grewAfterUpload = cursorEntry?.uploaded_object_key != null &&
73
+ cursorEntry.uploaded_byte_size != null &&
74
+ cursorEntry.uploaded_byte_size < file.byte_size;
75
+ return partial(grewAfterUpload ? "changed_since_upload" : "remote_smaller");
76
+ }
77
+ /**
78
+ * Reads the file at most once, on first ask, and redacts it exactly as the
79
+ * upload does (`raw-evidence-transcript-file.ts`): same sanitizer, same
80
+ * secret-like-name rule, same byte cap.
81
+ */
82
+ export function fileFacts(file, maxBytes = RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES) {
83
+ let bytes = null;
84
+ const read = () => {
85
+ bytes ??= file.byte_size > maxBytes ? Promise.resolve(null) : fs.readFile(file.file_path);
86
+ return bytes;
87
+ };
88
+ let rawHash = null;
89
+ let sanitized = null;
90
+ return {
91
+ rawSha256() {
92
+ rawHash ??= read().then(async (buffer) => (buffer ? sha256(buffer) : await streamSha256(file.file_path)));
93
+ return rawHash;
94
+ },
95
+ sanitized() {
96
+ sanitized ??= read().then((buffer) => {
97
+ if (!buffer)
98
+ return null;
99
+ const result = sanitizeTextEvidenceForUpload({
100
+ text: buffer.toString("utf8"),
101
+ originalBytes: buffer,
102
+ redactedFields: [`${file.kind === "codex_rollout" ? "codex_jsonl" : file.kind === "claude_main" ? "claude_jsonl" : "claude_jsonl_sidecar"}.body`],
103
+ secretLikeFileName: isSecretLikePath(path.basename(file.file_path)),
104
+ });
105
+ const hash = sha256(result.bytes);
106
+ if (hash === REDACTION_STUB_SHA256)
107
+ return null;
108
+ return { sha256: hash, byte_size: result.bytes.byteLength };
109
+ });
110
+ return sanitized;
111
+ },
112
+ };
113
+ }
114
+ function sha256(buffer) {
115
+ return crypto.createHash("sha256").update(buffer).digest("hex");
116
+ }
117
+ async function streamSha256(filePath) {
118
+ const hash = crypto.createHash("sha256");
119
+ const handle = await fs.open(filePath, "r");
120
+ try {
121
+ for await (const chunk of handle.createReadStream())
122
+ hash.update(chunk);
123
+ }
124
+ finally {
125
+ await handle.close().catch(() => undefined);
126
+ }
127
+ return hash.digest("hex");
128
+ }
@@ -0,0 +1,118 @@
1
+ /**
2
+ * Asks `POST /api/ambient/evidence/transcripts` for every current transcript
3
+ * object this operator owns, and groups the answer by local transcript file
4
+ * (BLI-11122). `cockpit sources check` compares each group to the file on disk.
5
+ *
6
+ * Never throws. A door that cannot be asked (an older dashboard answers 404, a
7
+ * network error, a malformed page) returns `unavailable` with the reason and
8
+ * the check says so instead of guessing.
9
+ *
10
+ * Bounded pages of up to 500, walked by the server's keyset cursor, at most
11
+ * `maxPages` of them. The result says whether the list was exhausted, capped by
12
+ * `maxPages`, or cut short by a failed page after some landed, and carries the
13
+ * requested, returned and unique counts so completeness is a number, not a claim.
14
+ */
15
+ import { EVIDENCE_TRANSCRIPTS_MAX_PAGE, EVIDENCE_TRANSCRIPTS_SCHEMA_VERSION, EvidenceTranscriptsResponseSchema, transcriptFileFromObjectKey, transcriptFileKey, } from "@bli-cockpit/telemetry-core";
16
+ import { describeError } from "./health-detail.js";
17
+ const TRANSCRIPTS_PAGE_TIMEOUT_MS = 30_000;
18
+ /** 400 pages of 500 is 200,000 current transcript objects: far past any operator today. */
19
+ export const TRANSCRIPTS_DEFAULT_MAX_PAGES = 400;
20
+ export async function fetchTranscriptObjects(options) {
21
+ const maxPages = options.maxPages ?? TRANSCRIPTS_DEFAULT_MAX_PAGES;
22
+ const pageSize = Math.min(options.pageSize ?? EVIDENCE_TRANSCRIPTS_MAX_PAGE, EVIDENCE_TRANSCRIPTS_MAX_PAGE);
23
+ const files = new Map();
24
+ const seen = new Set();
25
+ let returned = 0;
26
+ let unkeyed = 0;
27
+ let pages = 0;
28
+ let after = null;
29
+ for (;;) {
30
+ if (pages >= maxPages)
31
+ return finish("capped", "ok");
32
+ const page = await requestPage({ ...options, after, limit: pageSize });
33
+ if (!page.ok)
34
+ return finish(pages === 0 ? "unavailable" : "failed_partial", page.reason);
35
+ pages += 1;
36
+ for (const object of page.objects) {
37
+ returned += 1;
38
+ if (seen.has(object.ref_id))
39
+ continue;
40
+ seen.add(object.ref_id);
41
+ addObject(object);
42
+ }
43
+ // Advance by what the server returned; a page that names no cursor is the last.
44
+ if (!page.nextAfter)
45
+ return finish("exhausted", "ok");
46
+ after = page.nextAfter;
47
+ }
48
+ function addObject(object) {
49
+ const identity = transcriptFileFromObjectKey(object.object_key);
50
+ if (!identity) {
51
+ unkeyed += 1;
52
+ return;
53
+ }
54
+ const key = transcriptFileKey(identity);
55
+ const list = files.get(key) ?? [];
56
+ list.push({
57
+ object_key: object.object_key,
58
+ byte_size: object.byte_size,
59
+ content_hash_sha256: object.content_hash_sha256,
60
+ stub: object.stub,
61
+ });
62
+ files.set(key, list);
63
+ }
64
+ function finish(status, reason) {
65
+ const result = {
66
+ status,
67
+ reason,
68
+ pages,
69
+ requested: pages * pageSize,
70
+ returned,
71
+ unique: seen.size,
72
+ unkeyed,
73
+ files,
74
+ };
75
+ console.error("[collector sources] transcript objects listed", JSON.stringify({
76
+ reason: status === "unavailable" ? `transcript_door_unavailable:${reason}` : `transcripts_${status}`,
77
+ pages,
78
+ requested: result.requested,
79
+ returned,
80
+ unique: seen.size,
81
+ unkeyed,
82
+ file_count: files.size,
83
+ ...(status === "failed_partial" ? { failure: reason } : {}),
84
+ }));
85
+ return result;
86
+ }
87
+ }
88
+ async function requestPage(options) {
89
+ const controller = new AbortController();
90
+ const timeout = setTimeout(() => controller.abort(), TRANSCRIPTS_PAGE_TIMEOUT_MS);
91
+ try {
92
+ const response = await options.fetchImpl(`${options.dashboardUrl}/api/ambient/evidence/transcripts`, {
93
+ method: "POST",
94
+ headers: {
95
+ "Content-Type": "application/json",
96
+ Authorization: `Bearer ${options.deviceToken}`,
97
+ },
98
+ body: JSON.stringify({
99
+ schema_version: EVIDENCE_TRANSCRIPTS_SCHEMA_VERSION,
100
+ after: options.after,
101
+ limit: options.limit,
102
+ }),
103
+ signal: controller.signal,
104
+ });
105
+ if (!response.ok)
106
+ return { ok: false, reason: `http_${response.status}` };
107
+ const parsed = EvidenceTranscriptsResponseSchema.safeParse(await response.json().catch(() => null));
108
+ if (!parsed.success)
109
+ return { ok: false, reason: "invalid_response" };
110
+ return { ok: true, objects: parsed.data.objects, nextAfter: parsed.data.next_after };
111
+ }
112
+ catch (error) {
113
+ return { ok: false, reason: describeError(error).error_name };
114
+ }
115
+ finally {
116
+ clearTimeout(timeout);
117
+ }
118
+ }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@bli-cockpit/cli",
3
- "version": "0.2.158",
3
+ "version": "0.2.159",
4
4
  "private": false,
5
5
  "type": "module",
6
6
  "bin": {
@@ -27,8 +27,8 @@
27
27
  "test": "node dist/cli.js --help && node ../../scripts/assert-public-cli-routing.mjs && node ../../scripts/assert-public-cli-verb-help.mjs && node ../../scripts/assert-public-cli-surface-doors.mjs && node ../../scripts/assert-public-cli-runtime-files.mjs && node ../../scripts/assert-public-cli-exit-contract.mjs && node ../../scripts/assert-public-cli-no-fleet-posts.mjs && node ../../scripts/assert-public-cli-doctor-json.mjs && node ../../scripts/assert-public-package-pack.mjs --workspace=@bli-cockpit/cli"
28
28
  },
29
29
  "dependencies": {
30
- "@bli-cockpit/memory-mcp": "0.1.42",
31
- "@bli-cockpit/mcp": "0.1.76",
32
- "@bli-cockpit/telemetry-core": "0.1.54"
30
+ "@bli-cockpit/memory-mcp": "0.1.43",
31
+ "@bli-cockpit/mcp": "0.1.77",
32
+ "@bli-cockpit/telemetry-core": "0.1.55"
33
33
  }
34
34
  }