@bli-cockpit/cli 0.2.157 → 0.2.159
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/codex-attribution.js +2 -0
- package/dist/commands/backfill-checkpoint.js +11 -2
- package/dist/commands/backfill-scan.js +18 -6
- package/dist/commands/backfill-stub-locate.js +104 -0
- package/dist/commands/backfill-stub-reoffer.js +39 -5
- package/dist/commands/local-args-collector-sources.js +27 -0
- package/dist/commands/local-args-collector.js +2 -1
- package/dist/commands/local-args.js +3 -1
- package/dist/commands/local-help-commands.js +14 -0
- package/dist/commands/local-help.js +2 -0
- package/dist/commands/local.js +3 -0
- package/dist/commands/public-root.js +1 -1
- package/dist/commands/session-sync-record.js +12 -0
- package/dist/commands/sources-check.js +135 -0
- package/dist/commands/sources-discovery.js +232 -0
- package/dist/commands/sources-verdict.js +128 -0
- package/dist/commands/sync-stub-heal.js +23 -3
- package/dist/cursors/raw-evidence-cursor.js +10 -1
- package/dist/cursors/raw-evidence-stub-entry.js +39 -0
- package/dist/evidence-transcripts-client.js +118 -0
- package/package.json +4 -4
|
@@ -167,6 +167,7 @@ async function attributeOneSession(file, worktrees, collectionRoots) {
|
|
|
167
167
|
file_path: file.file,
|
|
168
168
|
file_name: fileName,
|
|
169
169
|
codex_session_id: sessionIdFromFileName(fileName) ?? shortHash(file.file),
|
|
170
|
+
session_id_source: sessionIdFromFileName(fileName) ? "file_name" : "path_hash",
|
|
170
171
|
cwd_basename: null,
|
|
171
172
|
cwd_hash: null,
|
|
172
173
|
session_file_mtime: new Date(file.mtimeMs).toISOString(),
|
|
@@ -197,6 +198,7 @@ async function attributeOneSession(file, worktrees, collectionRoots) {
|
|
|
197
198
|
const metaSessionId = sanitizeSessionId(signals.session_ids[0]);
|
|
198
199
|
if (metaSessionId) {
|
|
199
200
|
base.codex_session_id = metaSessionId;
|
|
201
|
+
base.session_id_source = "session_meta";
|
|
200
202
|
}
|
|
201
203
|
const primaryCwd = signals.cwds[0] ?? null;
|
|
202
204
|
if (primaryCwd) {
|
|
@@ -10,6 +10,7 @@
|
|
|
10
10
|
*/
|
|
11
11
|
import { recordBackfillCursorObservations, } from "../cursors/backfill-cursor.js";
|
|
12
12
|
import { CLAUDE_CURSOR_FILENAME, readRawEvidenceCursor, writeRawEvidenceCursor, } from "../cursors/raw-evidence-cursor.js";
|
|
13
|
+
import { isRedactionStubObjectKey, markCursorEntryStubbed } from "../cursors/raw-evidence-stub-entry.js";
|
|
13
14
|
import { isRawEvidenceUploadableAttributionState } from "../raw-evidence-attribution-policy.js";
|
|
14
15
|
import { candidateCursorKey, compareBackfillCandidates, isAfterCursor, } from "./backfill-candidates.js";
|
|
15
16
|
import { indexDurableMainObjectKeys } from "./backfill-evidence-outcomes.js";
|
|
@@ -34,10 +35,11 @@ export async function recordBackfillDurableSessionPointers(options) {
|
|
|
34
35
|
// No pointer, no prior entry, or already pointed: nothing this pass owes.
|
|
35
36
|
// BLI-11102: a pointer the server named a redaction stub is not a
|
|
36
37
|
// delivery, so a real pointer replaces it and the mark clears.
|
|
37
|
-
|
|
38
|
+
// BLI-11119: an entry the cursor itself knows is `stubbed` counts too.
|
|
39
|
+
const replacesStub = Boolean(prior?.stub_reoffer || prior?.stubbed) && prior?.uploaded_object_key !== objectKey;
|
|
38
40
|
if (!objectKey || !prior || (prior.uploaded_object_key && !replacesStub))
|
|
39
41
|
continue;
|
|
40
|
-
if (replacesStub)
|
|
42
|
+
if (replacesStub && !isRedactionStubObjectKey(objectKey))
|
|
41
43
|
stubsReplaced += 1;
|
|
42
44
|
cursor.sessions[candidate.session_id] = deliveredCursorEntry(prior, candidate, objectKey, options.now);
|
|
43
45
|
changed = true;
|
|
@@ -71,8 +73,15 @@ function deliveredCursorEntry(prior, candidate, objectKey, now) {
|
|
|
71
73
|
uploaded_byte_size: candidate.byte_size,
|
|
72
74
|
last_seen_at: now.toISOString(),
|
|
73
75
|
};
|
|
76
|
+
// BLI-11119: a re-offer that stubbed again earned another marker, not the
|
|
77
|
+
// transcript. The entry says so and keeps its mark for the next pass.
|
|
78
|
+
if (isRedactionStubObjectKey(objectKey)) {
|
|
79
|
+
markCursorEntryStubbed(entry);
|
|
80
|
+
return entry;
|
|
81
|
+
}
|
|
74
82
|
// The mark says "this pointer is a stub"; the pointer written above is not.
|
|
75
83
|
delete entry.stub_reoffer;
|
|
84
|
+
delete entry.stubbed;
|
|
76
85
|
return entry;
|
|
77
86
|
}
|
|
78
87
|
export function advanceBackfillCursorThroughResolvedPrefix(options) {
|
|
@@ -21,6 +21,7 @@ import { labelOutsideRootCandidates, outsideRootsWorkspace, } from "./backfill-o
|
|
|
21
21
|
import { reasonCountsFor } from "./backfill-reasons.js";
|
|
22
22
|
import { blockedBackfillResult } from "./backfill-result.js";
|
|
23
23
|
import { emptyStubReofferPlan, loadStubReofferPlan, reopenStubbedSessions, } from "./backfill-stub-reoffer.js";
|
|
24
|
+
import { loadStubCursorHashes, locateStubbedSessions } from "./backfill-stub-locate.js";
|
|
24
25
|
import { resolveBackfillScope } from "./backfill-scope.js";
|
|
25
26
|
import { stubSessionKey } from "../evidence-stubs-client.js";
|
|
26
27
|
/**
|
|
@@ -64,6 +65,7 @@ export async function scanBackfillRun(command, io, now) {
|
|
|
64
65
|
fetchImpl: io.fetch,
|
|
65
66
|
});
|
|
66
67
|
reopenStubbedSessions(pointerlessTerminalSessions, stubReoffer);
|
|
68
|
+
const stubCursorHashes = await loadStubCursorHashes(paths, stubReoffer);
|
|
67
69
|
const scan = await scanBackfillSessions({
|
|
68
70
|
command,
|
|
69
71
|
homeDir: command.homeDir ?? os.homedir(),
|
|
@@ -73,6 +75,8 @@ export async function scanBackfillRun(command, io, now) {
|
|
|
73
75
|
window: scope.window,
|
|
74
76
|
cursor,
|
|
75
77
|
pointerlessTerminalSessions,
|
|
78
|
+
stubReoffer,
|
|
79
|
+
stubCursorHashes,
|
|
76
80
|
now,
|
|
77
81
|
});
|
|
78
82
|
const reasonCounts = await auditScannedCandidates(scan, worktreeDiscovery.incomplete_reasons);
|
|
@@ -129,9 +133,19 @@ async function scanBackfillSessions(options) {
|
|
|
129
133
|
const claudeAttribution = options.sources.includes("claude_code")
|
|
130
134
|
? await scanClaudeHistory(options, claudeProjectsDir, scanLimit)
|
|
131
135
|
: null;
|
|
136
|
+
// BLI-11119: a stubbed session is found by the id in its content or by the
|
|
137
|
+
// file hash its cursor entry recorded, across every root the scan walked.
|
|
138
|
+
const scanned = locateStubbedSessions({
|
|
139
|
+
plan: options.stubReoffer,
|
|
140
|
+
candidates: [
|
|
141
|
+
...(codexAttribution?.results.map(normalizeCodexCandidate) ?? []),
|
|
142
|
+
...(claudeAttribution?.results.map(normalizeClaudeCandidate) ?? []),
|
|
143
|
+
],
|
|
144
|
+
cursorHashes: options.stubCursorHashes,
|
|
145
|
+
sinceMinutes: options.window.since_minutes,
|
|
146
|
+
});
|
|
132
147
|
const selection = selectBackfillCandidates({
|
|
133
|
-
|
|
134
|
-
claudeAttribution,
|
|
148
|
+
scanned,
|
|
135
149
|
cursor: options.cursor,
|
|
136
150
|
pointerlessTerminalSessions: options.pointerlessTerminalSessions,
|
|
137
151
|
maxFiles: options.command.maxFiles,
|
|
@@ -201,10 +215,7 @@ async function scanClaudeHistory(request, projectsDir, scanLimit) {
|
|
|
201
215
|
* so a `--max-files` cap spends its budget on them before newer history.
|
|
202
216
|
*/
|
|
203
217
|
function selectBackfillCandidates(options) {
|
|
204
|
-
const allCandidates =
|
|
205
|
-
...(options.codexAttribution?.results.map(normalizeCodexCandidate) ?? []),
|
|
206
|
-
...(options.claudeAttribution?.results.map(normalizeClaudeCandidate) ?? []),
|
|
207
|
-
]
|
|
218
|
+
const allCandidates = options.scanned
|
|
208
219
|
.filter((candidate) => !options.onlySessions ||
|
|
209
220
|
options.onlySessions.has(stubSessionKey(candidate.source, candidate.session_id)))
|
|
210
221
|
.filter((candidate) => isAfterCursor(candidate, options.cursor) ||
|
|
@@ -266,6 +277,7 @@ function normalizeCodexCandidate(result) {
|
|
|
266
277
|
return {
|
|
267
278
|
source: "codex",
|
|
268
279
|
session_id: result.codex_session_id,
|
|
280
|
+
session_id_source: result.session_id_source,
|
|
269
281
|
file_path: result.file_path,
|
|
270
282
|
state: result.state,
|
|
271
283
|
reason: result.reason,
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Where on this machine is the transcript behind a stub? (BLI-11119)
|
|
3
|
+
*
|
|
4
|
+
* The stubs door names a session by the id inside its object key, and the
|
|
5
|
+
* collector wrote that key with the id it read from the transcript's CONTENT
|
|
6
|
+
* (`session_meta.id` for Codex). A rollout's file name does not always carry
|
|
7
|
+
* that id: a resumed or forked session is filed as
|
|
8
|
+
* `rollout-<time>-<parent>_<child>.jsonl` while its first line names another
|
|
9
|
+
* id, and Codex moves files from `sessions/` into `archived_sessions/`. So a
|
|
10
|
+
* stubbed session is located the way it was keyed, across every root the scan
|
|
11
|
+
* walked, never by file name alone:
|
|
12
|
+
*
|
|
13
|
+
* 1. a scanned session whose id matches (`found_by: session_meta`, or
|
|
14
|
+
* `file_name` when the file's first line gave no id and the scan fell back
|
|
15
|
+
* to the name, which is then the id the key used too);
|
|
16
|
+
* 2. otherwise a scanned file whose sha256 equals the cursor entry's
|
|
17
|
+
* `file_hash_sha256` for that session (`found_by: file_hash`); it is offered
|
|
18
|
+
* under the STUBBED session's id, because the server retires a stub only for
|
|
19
|
+
* a successor that names the same session;
|
|
20
|
+
* 3. otherwise `not_found`, recorded on the cursor so the scheduled sync stops
|
|
21
|
+
* rescanning history for it every tick (see `sync-stub-heal.ts`).
|
|
22
|
+
*/
|
|
23
|
+
import { CLAUDE_CURSOR_FILENAME, readRawEvidenceCursor, } from "../cursors/raw-evidence-cursor.js";
|
|
24
|
+
import { stubSessionKey } from "../evidence-stubs-client.js";
|
|
25
|
+
/** How many per-session lines the locate log carries; the counts cover the rest. */
|
|
26
|
+
const LOGGED_SESSION_LIMIT = 25;
|
|
27
|
+
/** `codex:<id>` -> the file hash the cursor last recorded for that stubbed session. */
|
|
28
|
+
export async function loadStubCursorHashes(paths, plan) {
|
|
29
|
+
const hashes = new Map();
|
|
30
|
+
if (plan.sessions.size === 0)
|
|
31
|
+
return hashes;
|
|
32
|
+
for (const source of ["codex", "claude_code"]) {
|
|
33
|
+
const filename = source === "claude_code" ? CLAUDE_CURSOR_FILENAME : undefined;
|
|
34
|
+
const cursor = await readRawEvidenceCursor(paths, { filename }).catch(() => null);
|
|
35
|
+
if (!cursor)
|
|
36
|
+
continue;
|
|
37
|
+
for (const key of plan.sessions.keys()) {
|
|
38
|
+
if (!key.startsWith(`${source}:`))
|
|
39
|
+
continue;
|
|
40
|
+
const hash = cursor.sessions[key.slice(source.length + 1)]?.file_hash_sha256;
|
|
41
|
+
if (hash)
|
|
42
|
+
hashes.set(key, hash);
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
return hashes;
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* Match every stubbed session to a scanned file, record how on `plan.found_by`,
|
|
49
|
+
* and return the scanned candidates plus one candidate per hash match, carrying
|
|
50
|
+
* the stubbed session's id. Logs the outcome, found or not.
|
|
51
|
+
*/
|
|
52
|
+
export function locateStubbedSessions(options) {
|
|
53
|
+
const { plan } = options;
|
|
54
|
+
if (plan.sessions.size === 0)
|
|
55
|
+
return options.candidates;
|
|
56
|
+
const byKey = new Map();
|
|
57
|
+
const byHash = new Map();
|
|
58
|
+
for (const candidate of options.candidates) {
|
|
59
|
+
byKey.set(stubSessionKey(candidate.source, candidate.session_id), candidate);
|
|
60
|
+
if (candidate.content_hash_sha256) {
|
|
61
|
+
const hashKey = `${candidate.source}:${candidate.content_hash_sha256}`;
|
|
62
|
+
if (!byHash.has(hashKey))
|
|
63
|
+
byHash.set(hashKey, candidate);
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
const extra = [];
|
|
67
|
+
const sessions = [];
|
|
68
|
+
for (const key of plan.sessions.keys()) {
|
|
69
|
+
const found = locateOne(key, byKey, byHash, options.cursorHashes);
|
|
70
|
+
plan.found_by.set(key, found.foundBy);
|
|
71
|
+
if (found.candidate)
|
|
72
|
+
extra.push(found.candidate);
|
|
73
|
+
sessions.push({ session: key, found_by: found.foundBy });
|
|
74
|
+
}
|
|
75
|
+
const counts = { session_meta: 0, file_name: 0, file_hash: 0, not_found: 0 };
|
|
76
|
+
for (const session of sessions)
|
|
77
|
+
counts[session.found_by] += 1;
|
|
78
|
+
console.error("[backfill] stubbed sessions located", JSON.stringify({
|
|
79
|
+
reason: counts.not_found === sessions.length ? "stub_sessions_not_found" : "stub_sessions_located",
|
|
80
|
+
sessions: sessions.length,
|
|
81
|
+
...counts,
|
|
82
|
+
scanned_candidates: options.candidates.length,
|
|
83
|
+
since_minutes: options.sinceMinutes,
|
|
84
|
+
found_by: sessions.slice(0, LOGGED_SESSION_LIMIT),
|
|
85
|
+
}));
|
|
86
|
+
return extra.length > 0 ? [...options.candidates, ...extra] : options.candidates;
|
|
87
|
+
}
|
|
88
|
+
function locateOne(key, byKey, byHash, cursorHashes) {
|
|
89
|
+
const byId = byKey.get(key);
|
|
90
|
+
if (byId) {
|
|
91
|
+
const fromContent = byId.source === "codex" && byId.session_id_source === "session_meta";
|
|
92
|
+
return { foundBy: fromContent ? "session_meta" : "file_name" };
|
|
93
|
+
}
|
|
94
|
+
const at = key.indexOf(":");
|
|
95
|
+
const source = key.slice(0, at);
|
|
96
|
+
const hash = cursorHashes.get(key);
|
|
97
|
+
const byContent = hash ? byHash.get(`${source}:${hash}`) : undefined;
|
|
98
|
+
if (!byContent)
|
|
99
|
+
return { foundBy: "not_found" };
|
|
100
|
+
return {
|
|
101
|
+
foundBy: "file_hash",
|
|
102
|
+
candidate: { ...byContent, session_id: key.slice(at + 1), session_id_source: undefined },
|
|
103
|
+
};
|
|
104
|
+
}
|
|
@@ -1,10 +1,18 @@
|
|
|
1
1
|
import { CLAUDE_CURSOR_FILENAME, readRawEvidenceCursor, writeRawEvidenceCursor, } from "../cursors/raw-evidence-cursor.js";
|
|
2
|
+
import { markCursorEntryStubbed } from "../cursors/raw-evidence-stub-entry.js";
|
|
2
3
|
import { fetchStubbedSessions, stubSessionKey } from "../evidence-stubs-client.js";
|
|
3
4
|
import { describeError } from "../health-detail.js";
|
|
4
5
|
import { readLocalCollectorSessionFile } from "../local-state.js";
|
|
5
6
|
import { candidateCursorKey } from "./backfill-candidates.js";
|
|
6
7
|
export function emptyStubReofferPlan(reason) {
|
|
7
|
-
return {
|
|
8
|
+
return {
|
|
9
|
+
status: "skipped",
|
|
10
|
+
reason,
|
|
11
|
+
sessions: new Map(),
|
|
12
|
+
stub_count: 0,
|
|
13
|
+
marked_local_sessions: 0,
|
|
14
|
+
found_by: new Map(),
|
|
15
|
+
};
|
|
8
16
|
}
|
|
9
17
|
/** Ask the server which of this operator's pointers are stubs. Never throws; writes nothing. */
|
|
10
18
|
export async function loadStubReofferPlan(options) {
|
|
@@ -25,6 +33,7 @@ export async function loadStubReofferPlan(options) {
|
|
|
25
33
|
sessions,
|
|
26
34
|
stub_count: answer.unique_stub_count,
|
|
27
35
|
marked_local_sessions: 0,
|
|
36
|
+
found_by: new Map(),
|
|
28
37
|
};
|
|
29
38
|
}
|
|
30
39
|
/**
|
|
@@ -34,7 +43,7 @@ export async function loadStubReofferPlan(options) {
|
|
|
34
43
|
export async function markStubReoffers(paths, plan, now) {
|
|
35
44
|
if (plan.sessions.size === 0)
|
|
36
45
|
return;
|
|
37
|
-
plan.marked_local_sessions = await markStubbedSessions(paths, plan
|
|
46
|
+
plan.marked_local_sessions = await markStubbedSessions(paths, plan, now);
|
|
38
47
|
}
|
|
39
48
|
/** Reopen stubbed sessions exactly like pointer-less ones: selected past the cursor, sorted first. */
|
|
40
49
|
export function reopenStubbedSessions(reopen, plan) {
|
|
@@ -67,6 +76,7 @@ export function summarizeStubReoffer(plan, candidates, durableCandidateKeys = ne
|
|
|
67
76
|
reoffered_sessions: reoffered,
|
|
68
77
|
reoffered_stubs: stubsOf(offered.filter((candidate) => attempted.includes(candidate))),
|
|
69
78
|
replaced_stubs: stubsOf(offered.filter((candidate) => durableCandidateKeys.has(candidateCursorKey(candidate)))),
|
|
79
|
+
found_by: countFoundBy(plan),
|
|
70
80
|
};
|
|
71
81
|
if (plan.sessions.size > 0) {
|
|
72
82
|
console.error("[backfill] redaction-stubbed sessions re-offered", JSON.stringify({
|
|
@@ -79,20 +89,43 @@ export function summarizeStubReoffer(plan, candidates, durableCandidateKeys = ne
|
|
|
79
89
|
}
|
|
80
90
|
return summary;
|
|
81
91
|
}
|
|
82
|
-
|
|
92
|
+
function countFoundBy(plan) {
|
|
93
|
+
const counts = { session_meta: 0, file_name: 0, file_hash: 0, not_found: 0 };
|
|
94
|
+
for (const method of plan.found_by.values())
|
|
95
|
+
counts[method] += 1;
|
|
96
|
+
return counts;
|
|
97
|
+
}
|
|
98
|
+
/**
|
|
99
|
+
* Mark every door-named session this cursor tracks, and make its entry tell
|
|
100
|
+
* the truth (BLI-11119): an entry written before the stub rule existed claimed
|
|
101
|
+
* the full file as uploaded. The door is the source of truth, so each such
|
|
102
|
+
* entry is reconciled to `stubbed: true` with the marker's bytes, counted and
|
|
103
|
+
* logged as `stub_cursor_reconciled`. A session the scan could not find in any
|
|
104
|
+
* root keeps `not_found_at`, so the scheduled sync waits before looking again.
|
|
105
|
+
*/
|
|
106
|
+
async function markStubbedSessions(paths, plan, now) {
|
|
83
107
|
let marked = 0;
|
|
108
|
+
let reconciled = 0;
|
|
84
109
|
for (const source of ["codex", "claude_code"]) {
|
|
85
110
|
const filename = source === "claude_code" ? CLAUDE_CURSOR_FILENAME : undefined;
|
|
86
111
|
try {
|
|
87
112
|
const cursor = await readRawEvidenceCursor(paths, { filename });
|
|
88
113
|
let changed = false;
|
|
89
|
-
for (const [key, pointerIds] of sessions) {
|
|
114
|
+
for (const [key, pointerIds] of plan.sessions) {
|
|
90
115
|
if (!key.startsWith(`${source}:`))
|
|
91
116
|
continue;
|
|
92
117
|
const entry = cursor.sessions[key.slice(source.length + 1)];
|
|
93
118
|
if (!entry)
|
|
94
119
|
continue;
|
|
95
|
-
|
|
120
|
+
const notFound = plan.found_by.get(key) === "not_found";
|
|
121
|
+
entry.stub_reoffer = {
|
|
122
|
+
reason: "stub_reoffered",
|
|
123
|
+
stub_pointer_ids: pointerIds,
|
|
124
|
+
marked_at: now.toISOString(),
|
|
125
|
+
...(notFound ? { not_found_at: now.toISOString() } : {}),
|
|
126
|
+
};
|
|
127
|
+
if (markCursorEntryStubbed(entry))
|
|
128
|
+
reconciled += 1;
|
|
96
129
|
changed = true;
|
|
97
130
|
marked += 1;
|
|
98
131
|
}
|
|
@@ -105,5 +138,6 @@ async function markStubbedSessions(paths, sessions, now) {
|
|
|
105
138
|
console.error("[backfill] stub mark not written to the cursor", JSON.stringify({ reason: "stub_mark_write_failed", source, ...describeError(error) }));
|
|
106
139
|
}
|
|
107
140
|
}
|
|
141
|
+
console.error("[backfill] stubbed sessions' cursor entries reconciled with the stubs door", JSON.stringify({ reason: "stub_cursor_reconciled", count: reconciled, marked_local_sessions: marked }));
|
|
108
142
|
return marked;
|
|
109
143
|
}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `cockpit sources check [--older-than <days>] [--json]` (BLI-11122).
|
|
3
|
+
*
|
|
4
|
+
* One verb, read-only. There is deliberately no `sources clean` and no
|
|
5
|
+
* `--apply`: Tower never deletes a person's raw transcripts; deleting one is
|
|
6
|
+
* the person's own act, outside Tower. A path is never accepted: the roots are
|
|
7
|
+
* the ones the collector reads (`sources-discovery.ts`).
|
|
8
|
+
*/
|
|
9
|
+
import { optionalNonEmpty, optionalPositiveInteger, optionalUrl, parseNamedArgs, } from "./local-arg-values.js";
|
|
10
|
+
export function parseSourcesArgs(args) {
|
|
11
|
+
const values = parseNamedArgs(args, {
|
|
12
|
+
allowedFlags: ["--older-than", "--home", "--dashboard-url", "--json"],
|
|
13
|
+
valueFlags: ["--older-than", "--home", "--dashboard-url"],
|
|
14
|
+
});
|
|
15
|
+
const action = values.positionals[0] ?? "check";
|
|
16
|
+
if (action !== "check" || values.positionals.length > 1) {
|
|
17
|
+
throw new Error("sources has one verb: `cockpit sources check [--older-than <days>] [--json]`.");
|
|
18
|
+
}
|
|
19
|
+
return {
|
|
20
|
+
kind: "sources",
|
|
21
|
+
action: "check",
|
|
22
|
+
olderThanDays: optionalPositiveInteger(values.flags.get("--older-than"), "--older-than") ?? null,
|
|
23
|
+
homeDir: optionalNonEmpty(values.flags.get("--home")),
|
|
24
|
+
dashboardUrl: optionalUrl(values.flags.get("--dashboard-url")),
|
|
25
|
+
json: values.booleans.has("--json"),
|
|
26
|
+
};
|
|
27
|
+
}
|
|
@@ -26,4 +26,5 @@
|
|
|
26
26
|
*/
|
|
27
27
|
export { parseOnboardArgs, parseUpdateArgs, parseDoctorArgs, parseInstallArgs, parseReleaseArgs, parseLoginArgs, parseLogoutArgs, } from "./local-args-collector-setup.js";
|
|
28
28
|
export { parseStartArgs, parseSyncArgs, parseAnalyzeArgs, parseBackfillArgs, } from "./local-args-collector-work.js";
|
|
29
|
-
export { parseStatusArgs, parseSessionsArgs, parseServeArgs, parseAutostartArgs, parseAgentRulesArgs, parseMemoryArgs, parseCleanArgs, } from "./local-args-collector-status.js";
|
|
29
|
+
export { parseStatusArgs, parseSessionsArgs, parseServeArgs, parseAutostartArgs, parseAgentRulesArgs, parseMemoryArgs, parseCleanArgs, } from "./local-args-collector-status.js";
|
|
30
|
+
export { parseSourcesArgs } from "./local-args-collector-sources.js";
|
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
*/
|
|
18
18
|
import { parseRootsArgs } from "./roots.js";
|
|
19
19
|
import { SURFACE_ALIASES } from "./local-help.js";
|
|
20
|
-
import { parseAgentRulesArgs, parseAnalyzeArgs, parseAutostartArgs, parseBackfillArgs, parseCleanArgs, parseDoctorArgs, parseInstallArgs, parseLoginArgs, parseMemoryArgs, parseLogoutArgs, parseOnboardArgs, parseReleaseArgs, parseServeArgs, parseSessionsArgs, parseStartArgs, parseStatusArgs, parseSyncArgs, parseUpdateArgs, } from "./local-args-collector.js";
|
|
20
|
+
import { parseAgentRulesArgs, parseAnalyzeArgs, parseAutostartArgs, parseBackfillArgs, parseCleanArgs, parseDoctorArgs, parseInstallArgs, parseLoginArgs, parseMemoryArgs, parseLogoutArgs, parseOnboardArgs, parseReleaseArgs, parseServeArgs, parseSessionsArgs, parseSourcesArgs, parseStartArgs, parseStatusArgs, parseSyncArgs, parseUpdateArgs, } from "./local-args-collector.js";
|
|
21
21
|
import { parseBriefArgs, parseCorrectArgs, parseDocsArgs, parseIssueArgs, parseJarvisArgs, parseMailArgs, parseCalArgs, parseModelArgs, parseMsgArgs, parseNotesArgs, parseOpsArgs, parseProjectArgs, parseModelsArgs, parseScoutArgs, parseSearchArgs, parseSettingsArgs, parseSlackArgs, parseTeamArgs, parseWorkbookArgs, parseUsageArgs, parseCareersArgs, } from "./local-args-tower.js";
|
|
22
22
|
// `normalizeUrl` has always been part of this module's surface — `local.ts` and
|
|
23
23
|
// `local-auth.ts` import it from here — so it stays exported from this address
|
|
@@ -107,6 +107,8 @@ export function parseLocalArgs(argv) {
|
|
|
107
107
|
return parseMemoryArgs(argv.slice(1));
|
|
108
108
|
case "clean":
|
|
109
109
|
return parseCleanArgs(argv.slice(1));
|
|
110
|
+
case "sources":
|
|
111
|
+
return parseSourcesArgs(argv.slice(1));
|
|
110
112
|
case "docs":
|
|
111
113
|
return parseDocsArgs(argv.slice(1));
|
|
112
114
|
case "msg":
|
|
@@ -366,6 +366,20 @@ export function localSubcommandHelp(command) {
|
|
|
366
366
|
"laptop that has been off.",
|
|
367
367
|
],
|
|
368
368
|
],
|
|
369
|
+
[
|
|
370
|
+
"sources",
|
|
371
|
+
[
|
|
372
|
+
"Usage: cockpit sources check [--older-than <days>] [--dashboard-url <url>] [--json]",
|
|
373
|
+
"",
|
|
374
|
+
"Answers, for every Codex and Claude transcript on this machine, whether Tower",
|
|
375
|
+
"holds it: stored in full, stored in part, or not stored, with the reason.",
|
|
376
|
+
"--older-than <days> checks only files not modified in that many days.",
|
|
377
|
+
"--json lists every file with its path, verdict and reason.",
|
|
378
|
+
"Read-only. Tower never deletes these files; deleting one is your own act.",
|
|
379
|
+
"Before deleting any transcript by hand, run this and delete only files it",
|
|
380
|
+
"answers \"stored in full\". See docs/runbooks/freeing-disk-on-a-collector-machine.md.",
|
|
381
|
+
],
|
|
382
|
+
],
|
|
369
383
|
[
|
|
370
384
|
"release",
|
|
371
385
|
[
|
|
@@ -65,6 +65,7 @@ export const rootCommandNames = new Set([
|
|
|
65
65
|
"agent-rules",
|
|
66
66
|
"memory",
|
|
67
67
|
"clean",
|
|
68
|
+
"sources",
|
|
68
69
|
"docs",
|
|
69
70
|
"msg",
|
|
70
71
|
"issue",
|
|
@@ -117,6 +118,7 @@ export function localCommandHelp(command) {
|
|
|
117
118
|
" cockpit agent-rules [install|uninstall|status] [--host codex|claude|all] [--workspace <path>] [--json]",
|
|
118
119
|
" cockpit memory [search \"<question>\"|save|update <id>|forget <id>|profile|install|status|log <win|loss|noise> --store <bli|supermemory|both> \"<reason>\"] [--container <tag>] [--limit <n>] [--no-recent] [--custom-id <id>] [--extract] [--content-stdin] [--reason <label>] [--file <path>] [--dashboard-url <url>] [--dry-run] [--json]",
|
|
119
120
|
" cockpit clean [--dry-run] [--all-committed] [--reconcile] [--dashboard-url <url>] [--json]",
|
|
121
|
+
" cockpit sources check [--older-than <days>] [--dashboard-url <url>] [--json]",
|
|
120
122
|
" cockpit docs [list|tree|read <id|slug>|create --title <t>|update <id>|delete <id|slug>] [--parent <id>|root|--clear-parent] [--query <text> [--word]] [--limit <n>] [--visibility org|private] [--file <path>|--body-stdin] [--allow-empty] [--dashboard-url <url>] [--json]",
|
|
121
123
|
" cockpit msg [channels|create <name>|dm <email>|read <channel>|send <channel>|thread <id> --channel <channel>] [--private] [--members a@x,b@y] [--description <text>] [--thread <id>] [--limit <n>] [--dashboard-url <url>] [--json]",
|
|
122
124
|
" cockpit issue [list|show <BLI-id>|create --title <t>|update <BLI-id>|move <BLI-id> <state>|comment <BLI-id>|history <BLI-id>] [--state <s>] [--assignee me|unassigned|<uuid>] [--project <name|id>] [--limit <n>] [--priority 0-4] [--size S|M|L] [--parent <BLI-id>] [--file <path>|--body-stdin] [--dashboard-url <url>] [--json]",
|
package/dist/commands/local.js
CHANGED
|
@@ -37,6 +37,7 @@ import { runMemoryLog } from "./memory-log.js";
|
|
|
37
37
|
import { runMemoryInstall } from "./memory-install.js";
|
|
38
38
|
import { runMemoryStore, runMemoryVerbList } from "./memory-store.js";
|
|
39
39
|
import { runClean } from "./clean.js";
|
|
40
|
+
import { runSourcesCheck } from "./sources-check.js";
|
|
40
41
|
import { runDocs } from "./docs.js";
|
|
41
42
|
import { runMsg } from "./msg.js";
|
|
42
43
|
import { runIssue } from "./issue.js";
|
|
@@ -213,6 +214,8 @@ export async function runLocalCockpitCli(argv, io = defaultIo()) {
|
|
|
213
214
|
return await runMemoryStore(command, io);
|
|
214
215
|
case "clean":
|
|
215
216
|
return await runClean(command, io);
|
|
217
|
+
case "sources":
|
|
218
|
+
return await runSourcesCheck(command, io);
|
|
216
219
|
case "docs":
|
|
217
220
|
return await runDocs(command, io);
|
|
218
221
|
case "msg":
|
|
@@ -21,7 +21,7 @@ export async function runCockpitCli(argv, io) {
|
|
|
21
21
|
}
|
|
22
22
|
|
|
23
23
|
if (command === "--version" || command === "-V" || command === "version") {
|
|
24
|
-
writeLine(io?.stdout ?? process.stdout, "0.2.
|
|
24
|
+
writeLine(io?.stdout ?? process.stdout, "0.2.159");
|
|
25
25
|
return 0;
|
|
26
26
|
}
|
|
27
27
|
|
|
@@ -11,6 +11,7 @@ import { DEFAULT_DASHBOARD_URL, getCollectorRuntimePaths, readLocalCollectorConf
|
|
|
11
11
|
import { normalizeDashboardUrl } from "../upload-http.js";
|
|
12
12
|
import { flushPendingCodexSessionReports, queueCodexSessionReport, } from "../upload.js";
|
|
13
13
|
import { CLAUDE_CURSOR_FILENAME, countStaleSessions, readRawEvidenceCursor, recordSessionObservation, writeRawEvidenceCursor, } from "../cursors/raw-evidence-cursor.js";
|
|
14
|
+
import { isRedactionStubObjectKey, markCursorEntryStubbed } from "../cursors/raw-evidence-stub-entry.js";
|
|
14
15
|
import { claudeAttributionReadFailureCount, codexAttributionReadFailureCount, } from "./agent-session-report.js";
|
|
15
16
|
import { liveSyncCursorEntryRequiresRetry } from "./session-sync-attribution.js";
|
|
16
17
|
/**
|
|
@@ -331,5 +332,16 @@ function recordSourceObservation(options, result) {
|
|
|
331
332
|
uploaded_byte_size: uploadedByteSize,
|
|
332
333
|
last_seen_at: options.now.toISOString(),
|
|
333
334
|
};
|
|
335
|
+
// BLI-11119: a pointer that is the redaction-crash marker is not a delivery.
|
|
336
|
+
// The entry records the marker's bytes, and keeps the re-offer mark while
|
|
337
|
+
// the stub still stands.
|
|
338
|
+
if (isRedactionStubObjectKey(uploadedObjectKey)) {
|
|
339
|
+
markCursorEntryStubbed(entry);
|
|
340
|
+
if (prior?.stub_reoffer)
|
|
341
|
+
entry.stub_reoffer = prior.stub_reoffer;
|
|
342
|
+
if (uploadedThisSync) {
|
|
343
|
+
console.error("[collector sync] session delivered only as a redaction-crash stub", JSON.stringify({ reason: "stub_cursor_recorded", source: options.source, byte_size: result.byte_size }));
|
|
344
|
+
}
|
|
345
|
+
}
|
|
334
346
|
recordSessionObservation(options.cursor, sessionId, entry);
|
|
335
347
|
}
|
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
import { writeLine } from "./cli-io.js";
|
|
2
|
+
import { CLAUDE_CURSOR_FILENAME, readRawEvidenceCursor, } from "../cursors/raw-evidence-cursor.js";
|
|
3
|
+
import { fetchTranscriptObjects } from "../evidence-transcripts-client.js";
|
|
4
|
+
import { DEFAULT_DASHBOARD_URL, getCollectorRuntimePaths, readLocalCollectorConfig, readLocalCollectorSessionFile, } from "../local-state.js";
|
|
5
|
+
import { discoverLocalTranscripts } from "./sources-discovery.js";
|
|
6
|
+
import { fileFacts, judgeSource } from "./sources-verdict.js";
|
|
7
|
+
const VERDICTS = ["stored_full", "stored_partial", "not_stored"];
|
|
8
|
+
const DAY_MS = 24 * 60 * 60 * 1000;
|
|
9
|
+
export async function runSourcesCheck(command, io, now = new Date()) {
|
|
10
|
+
const startedAt = Date.now();
|
|
11
|
+
const paths = getCollectorRuntimePaths(command.homeDir);
|
|
12
|
+
const config = await readLocalCollectorConfig(paths).catch(() => null);
|
|
13
|
+
const dashboardUrl = command.dashboardUrl ?? config?.dashboard_url ?? DEFAULT_DASHBOARD_URL;
|
|
14
|
+
const session = await readLocalCollectorSessionFile(paths).catch(() => null);
|
|
15
|
+
if (!session || session.session_state !== "valid" || typeof session.device_token !== "string" || !session.device_token) {
|
|
16
|
+
console.error("[collector sources] check not run", JSON.stringify({ reason: "collector_not_paired" }));
|
|
17
|
+
return fail(io, command, "collector_not_paired", "This machine is not signed in to Tower, so Tower cannot be asked. Run `cockpit doctor`.");
|
|
18
|
+
}
|
|
19
|
+
const remote = await fetchTranscriptObjects({ dashboardUrl, deviceToken: session.device_token, fetchImpl: io.fetch });
|
|
20
|
+
if (remote.status === "unavailable") {
|
|
21
|
+
console.error("[collector sources] check not answered", JSON.stringify({ reason: `transcript_door_unavailable:${remote.reason}` }));
|
|
22
|
+
return fail(io, command, `transcript_door_unavailable:${remote.reason}`, `Tower could not be asked which transcripts it holds (${remote.reason}). No file is answered; nothing here means "not stored".`);
|
|
23
|
+
}
|
|
24
|
+
const homeDir = command.homeDir ?? paths.home_dir;
|
|
25
|
+
const olderThanMs = command.olderThanDays === null ? null : now.getTime() - command.olderThanDays * DAY_MS;
|
|
26
|
+
const local = await discoverLocalTranscripts({ homeDir, olderThanMs });
|
|
27
|
+
const cursors = {
|
|
28
|
+
codex: await readRawEvidenceCursor(paths),
|
|
29
|
+
claude_code: await readRawEvidenceCursor(paths, { filename: CLAUDE_CURSOR_FILENAME, sessionsOnly: true }),
|
|
30
|
+
};
|
|
31
|
+
const report = emptyReport(command, remote, local.newer_count, local.file_read_failed + local.directory_read_failed);
|
|
32
|
+
for (const file of local.files) {
|
|
33
|
+
const judged = await judgeSource({
|
|
34
|
+
file,
|
|
35
|
+
remotes: remote.files.get(file.key) ?? [],
|
|
36
|
+
cursorEntry: file.kind === "claude_subagent" ? null : (cursors[file.source].sessions[file.session_id] ?? null),
|
|
37
|
+
facts: fileFacts(file),
|
|
38
|
+
}).catch((error) => {
|
|
39
|
+
report.local.unreadable += 1;
|
|
40
|
+
console.error("[collector sources] a transcript could not be judged; left out of the answer", JSON.stringify({ reason: "file_unreadable", kind: file.kind, byte_size: file.byte_size, error_name: error?.name ?? "Error" }));
|
|
41
|
+
return null;
|
|
42
|
+
});
|
|
43
|
+
if (!judged)
|
|
44
|
+
continue;
|
|
45
|
+
report.local.examined += 1;
|
|
46
|
+
report.counts[judged.verdict].files += 1;
|
|
47
|
+
report.counts[judged.verdict].bytes += file.byte_size;
|
|
48
|
+
report.reasons[judged.reason] = (report.reasons[judged.reason] ?? 0) + 1;
|
|
49
|
+
report.files.push({
|
|
50
|
+
path: file.file_path,
|
|
51
|
+
source: file.source,
|
|
52
|
+
kind: file.kind,
|
|
53
|
+
session_id: file.session_id,
|
|
54
|
+
byte_size: file.byte_size,
|
|
55
|
+
modified_at: new Date(file.mtime_ms).toISOString(),
|
|
56
|
+
verdict: judged.verdict,
|
|
57
|
+
reason: judged.reason,
|
|
58
|
+
remote_byte_size: judged.remote_byte_size,
|
|
59
|
+
});
|
|
60
|
+
}
|
|
61
|
+
report.files.sort((a, b) => VERDICTS.indexOf(a.verdict) - VERDICTS.indexOf(b.verdict) || a.path.localeCompare(b.path));
|
|
62
|
+
// Counts and bytes only: never a path, a session id or a byte of content.
|
|
63
|
+
console.error("[collector sources] check", JSON.stringify({
|
|
64
|
+
reason: `sources_check_${report.status}`,
|
|
65
|
+
older_than_days: report.older_than_days,
|
|
66
|
+
remote_status: remote.status,
|
|
67
|
+
remote_unique: remote.unique,
|
|
68
|
+
examined: report.local.examined,
|
|
69
|
+
newer_skipped: report.local.newer_skipped,
|
|
70
|
+
unreadable: report.local.unreadable,
|
|
71
|
+
stored_full: report.counts.stored_full,
|
|
72
|
+
stored_partial: report.counts.stored_partial,
|
|
73
|
+
not_stored: report.counts.not_stored,
|
|
74
|
+
reasons: report.reasons,
|
|
75
|
+
elapsed_ms: Date.now() - startedAt,
|
|
76
|
+
}));
|
|
77
|
+
if (command.json)
|
|
78
|
+
writeLine(io.stdout, JSON.stringify(report, null, 2));
|
|
79
|
+
else
|
|
80
|
+
for (const line of sourcesCheckLines(report))
|
|
81
|
+
writeLine(io.stdout, line);
|
|
82
|
+
return 0;
|
|
83
|
+
}
|
|
84
|
+
function emptyReport(command, remote, newerSkipped, unreadable) {
|
|
85
|
+
const { files, ...remoteCounts } = remote;
|
|
86
|
+
return {
|
|
87
|
+
status: remote.status === "exhausted" ? "complete" : "incomplete",
|
|
88
|
+
reason: remote.status === "exhausted" ? "ok" : `transcript_list_${remote.status}${remote.reason === "ok" ? "" : `:${remote.reason}`}`,
|
|
89
|
+
older_than_days: command.olderThanDays,
|
|
90
|
+
remote: { ...remoteCounts, file_count: files.size },
|
|
91
|
+
local: { examined: 0, newer_skipped: newerSkipped, unreadable },
|
|
92
|
+
counts: {
|
|
93
|
+
stored_full: { files: 0, bytes: 0 },
|
|
94
|
+
stored_partial: { files: 0, bytes: 0 },
|
|
95
|
+
not_stored: { files: 0, bytes: 0 },
|
|
96
|
+
},
|
|
97
|
+
reasons: {},
|
|
98
|
+
files: [],
|
|
99
|
+
};
|
|
100
|
+
}
|
|
101
|
+
function fail(io, command, reason, sentence) {
|
|
102
|
+
if (command.json) {
|
|
103
|
+
writeLine(io.stdout, JSON.stringify({ status: "unavailable", reason, older_than_days: command.olderThanDays }, null, 2));
|
|
104
|
+
}
|
|
105
|
+
else {
|
|
106
|
+
writeLine(io.stderr, sentence);
|
|
107
|
+
}
|
|
108
|
+
return 1;
|
|
109
|
+
}
|
|
110
|
+
/** What a person reads: counts and bytes per verdict, then why. Paths only with --json. */
|
|
111
|
+
export function sourcesCheckLines(report) {
|
|
112
|
+
const scope = report.older_than_days === null ? "every transcript" : `transcripts not modified in ${report.older_than_days} days`;
|
|
113
|
+
const lines = [
|
|
114
|
+
`Tower sources check: ${scope} on this machine`,
|
|
115
|
+
` stored in full ${report.counts.stored_full.files} file(s), ${mb(report.counts.stored_full.bytes)} MB`,
|
|
116
|
+
` stored in part ${report.counts.stored_partial.files} file(s), ${mb(report.counts.stored_partial.bytes)} MB`,
|
|
117
|
+
` not stored ${report.counts.not_stored.files} file(s), ${mb(report.counts.not_stored.bytes)} MB`,
|
|
118
|
+
];
|
|
119
|
+
const reasons = Object.entries(report.reasons).sort((a, b) => b[1] - a[1]);
|
|
120
|
+
if (reasons.length > 0)
|
|
121
|
+
lines.push(` why ${reasons.map(([reason, count]) => `${reason} ${count}`).join(", ")}`);
|
|
122
|
+
if (report.local.newer_skipped > 0)
|
|
123
|
+
lines.push(` newer ${report.local.newer_skipped} file(s) modified inside the window, not checked`);
|
|
124
|
+
if (report.local.unreadable > 0)
|
|
125
|
+
lines.push(` unreadable ${report.local.unreadable} file(s) could not be read and are not answered`);
|
|
126
|
+
lines.push(` Tower's list ${report.remote.status}: ${report.remote.unique} current transcript object(s) over ${report.remote.pages} page(s)`);
|
|
127
|
+
if (report.status === "incomplete") {
|
|
128
|
+
lines.push(` INCOMPLETE Tower's list was cut short (${report.reason}); "not stored" and "stored in part" may be understated. Run it again.`);
|
|
129
|
+
}
|
|
130
|
+
lines.push("Tower never deletes these files. `--json` lists every file with its path and verdict. Only a \"stored in full\" file is safe to delete by your own hand; see docs/runbooks/freeing-disk-on-a-collector-machine.md.");
|
|
131
|
+
return lines;
|
|
132
|
+
}
|
|
133
|
+
function mb(bytes) {
|
|
134
|
+
return (bytes / (1024 * 1024)).toFixed(1);
|
|
135
|
+
}
|
|
@@ -0,0 +1,232 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Every agent transcript on this machine, named the way the collector names
|
|
3
|
+
* its upload (BLI-11122).
|
|
4
|
+
*
|
|
5
|
+
* `cockpit sources check` has to pair each local file with the object Tower
|
|
6
|
+
* holds for it, and an object key carries the session id the collector read
|
|
7
|
+
* from the transcript's CONTENT, not its file name (a resumed or forked Codex
|
|
8
|
+
* rollout is filed under one id and names another; see
|
|
9
|
+
* `backfill-stub-locate.ts`). So each file is named by the same rules the
|
|
10
|
+
* readers use, kept here as pure functions so both path layouts (macOS and
|
|
11
|
+
* Windows) are testable without either file system:
|
|
12
|
+
*
|
|
13
|
+
* - Codex (`~/.codex/sessions/**`, `~/.codex/archived_sessions/**`): the first
|
|
14
|
+
* `session_meta` record's `payload.id`, else the uuid in the file name, else
|
|
15
|
+
* a hash of the path (`codex-attribution.ts`).
|
|
16
|
+
* - Claude main (`~/.claude/projects/<project>/<uuid>.jsonl`): the first
|
|
17
|
+
* record's `sessionId`, else the file name's uuid (`claude-attribution-score.ts`).
|
|
18
|
+
* - Claude subagent (`<project>/<uuid>/subagents/agent-*.jsonl`): the main's
|
|
19
|
+
* session id plus the file stem (`raw-evidence-claude-reader.ts`).
|
|
20
|
+
*
|
|
21
|
+
* This module reads files and never writes, moves or deletes one.
|
|
22
|
+
*/
|
|
23
|
+
import fs from "node:fs/promises";
|
|
24
|
+
import path from "node:path";
|
|
25
|
+
import { createReadStream } from "node:fs";
|
|
26
|
+
import readline from "node:readline";
|
|
27
|
+
import { transcriptFileKey } from "@bli-cockpit/telemetry-core";
|
|
28
|
+
import { SESSION_FILE_UUID_PATTERN, sanitizeSessionId, sessionIdFromFileName, shortHash, } from "../adapters/attribution-core.js";
|
|
29
|
+
import { isSecretLikePath, safeKeySegment } from "../adapters/raw-evidence-keys.js";
|
|
30
|
+
import { describeError } from "../health-detail.js";
|
|
31
|
+
export function sourcesRoots(homeDir, pathApi = path) {
|
|
32
|
+
return {
|
|
33
|
+
codex: [
|
|
34
|
+
pathApi.join(homeDir, ".codex", "sessions"),
|
|
35
|
+
pathApi.join(homeDir, ".codex", "archived_sessions"),
|
|
36
|
+
],
|
|
37
|
+
claudeProjects: pathApi.join(homeDir, ".claude", "projects"),
|
|
38
|
+
};
|
|
39
|
+
}
|
|
40
|
+
/** Which kind of transcript a path is, or null for anything the collector does not upload as one. */
|
|
41
|
+
export function classifyTranscriptPath(filePath, roots, pathApi = path) {
|
|
42
|
+
if (!filePath.toLowerCase().endsWith(".jsonl"))
|
|
43
|
+
return null;
|
|
44
|
+
for (const root of roots.codex) {
|
|
45
|
+
const rel = pathApi.relative(root, filePath);
|
|
46
|
+
if (rel && !rel.startsWith("..") && !pathApi.isAbsolute(rel))
|
|
47
|
+
return { kind: "codex_rollout" };
|
|
48
|
+
}
|
|
49
|
+
const rel = pathApi.relative(roots.claudeProjects, filePath);
|
|
50
|
+
if (!rel || rel.startsWith("..") || pathApi.isAbsolute(rel))
|
|
51
|
+
return null;
|
|
52
|
+
const parts = rel.split(pathApi.sep);
|
|
53
|
+
if (parts.length === 2 && SESSION_FILE_UUID_PATTERN.test(parts[1]))
|
|
54
|
+
return { kind: "claude_main" };
|
|
55
|
+
if (parts.length === 4 && parts[2] === "subagents" && parts[3].startsWith("agent-")) {
|
|
56
|
+
return {
|
|
57
|
+
kind: "claude_subagent",
|
|
58
|
+
main_session_dir: parts[1],
|
|
59
|
+
sidecar_stem: parts[3].replace(/\.jsonl$/i, ""),
|
|
60
|
+
};
|
|
61
|
+
}
|
|
62
|
+
return null;
|
|
63
|
+
}
|
|
64
|
+
/** The session id a reader would key this file's session by, from its content id when it has one. */
|
|
65
|
+
export function sessionIdFor(filePath, contentId, pathApi = path) {
|
|
66
|
+
const fromContent = sanitizeSessionId(contentId ?? undefined);
|
|
67
|
+
if (fromContent)
|
|
68
|
+
return { session_id: fromContent, session_id_source: "content" };
|
|
69
|
+
const fromName = sessionIdFromFileName(pathApi.basename(filePath));
|
|
70
|
+
if (fromName)
|
|
71
|
+
return { session_id: fromName, session_id_source: "file_name" };
|
|
72
|
+
return { session_id: shortHash(filePath), session_id_source: "path_hash" };
|
|
73
|
+
}
|
|
74
|
+
/** The key of the object a file would be uploaded as; see `raw-evidence-*-reader.ts`. */
|
|
75
|
+
export function transcriptKeyFor(source, sessionId, sidecarStem) {
|
|
76
|
+
const identity = {
|
|
77
|
+
source,
|
|
78
|
+
session_id: safeKeySegment(sessionId),
|
|
79
|
+
sidecar: sidecarStem === null ? null : isSecretLikePath(sidecarStem) ? "redacted-file-name" : safeKeySegment(sidecarStem),
|
|
80
|
+
};
|
|
81
|
+
return transcriptFileKey(identity);
|
|
82
|
+
}
|
|
83
|
+
/**
|
|
84
|
+
* The first content id in a transcript: Codex's first `session_meta`
|
|
85
|
+
* `payload.id`, Claude's first `sessionId`. Streams line by line and stops at
|
|
86
|
+
* the first hit; a line that does not parse is skipped, as the readers do.
|
|
87
|
+
*/
|
|
88
|
+
export async function readContentSessionId(filePath, source) {
|
|
89
|
+
const stream = createReadStream(filePath, { encoding: "utf8" });
|
|
90
|
+
const lines = readline.createInterface({ input: stream, crlfDelay: Infinity });
|
|
91
|
+
try {
|
|
92
|
+
for await (const line of lines) {
|
|
93
|
+
const id = contentIdFromLine(line, source);
|
|
94
|
+
if (id)
|
|
95
|
+
return id;
|
|
96
|
+
}
|
|
97
|
+
return null;
|
|
98
|
+
}
|
|
99
|
+
finally {
|
|
100
|
+
lines.close();
|
|
101
|
+
stream.destroy();
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
export function contentIdFromLine(line, source) {
|
|
105
|
+
if (!line.trim())
|
|
106
|
+
return null;
|
|
107
|
+
let record;
|
|
108
|
+
try {
|
|
109
|
+
record = JSON.parse(line);
|
|
110
|
+
}
|
|
111
|
+
catch {
|
|
112
|
+
return null;
|
|
113
|
+
}
|
|
114
|
+
if (!record || typeof record !== "object" || Array.isArray(record))
|
|
115
|
+
return null;
|
|
116
|
+
const entry = record;
|
|
117
|
+
if (source === "claude_code")
|
|
118
|
+
return trimmed(entry["sessionId"]);
|
|
119
|
+
if (entry["type"] !== "session_meta")
|
|
120
|
+
return null;
|
|
121
|
+
const payload = entry["payload"];
|
|
122
|
+
if (!payload || typeof payload !== "object")
|
|
123
|
+
return null;
|
|
124
|
+
return trimmed(payload["id"]);
|
|
125
|
+
}
|
|
126
|
+
function trimmed(value) {
|
|
127
|
+
return typeof value === "string" && value.trim() ? value.trim() : null;
|
|
128
|
+
}
|
|
129
|
+
/**
|
|
130
|
+
* Walk every root, name every transcript. `olderThanMs` keeps only files whose
|
|
131
|
+
* mtime is before it; the rest are counted, never read.
|
|
132
|
+
*/
|
|
133
|
+
export async function discoverLocalTranscripts(options) {
|
|
134
|
+
const roots = sourcesRoots(options.homeDir);
|
|
135
|
+
const result = { files: [], newer_count: 0, directory_read_failed: 0, file_read_failed: 0 };
|
|
136
|
+
const found = [];
|
|
137
|
+
let firstFailure = null;
|
|
138
|
+
const stack = [...roots.codex, roots.claudeProjects];
|
|
139
|
+
const seen = new Set();
|
|
140
|
+
while (stack.length > 0) {
|
|
141
|
+
const dir = stack.pop();
|
|
142
|
+
let entries;
|
|
143
|
+
try {
|
|
144
|
+
entries = await fs.readdir(dir, { withFileTypes: true });
|
|
145
|
+
}
|
|
146
|
+
catch (error) {
|
|
147
|
+
if (error?.code !== "ENOENT") {
|
|
148
|
+
result.directory_read_failed += 1;
|
|
149
|
+
firstFailure ??= describeError(error);
|
|
150
|
+
}
|
|
151
|
+
continue;
|
|
152
|
+
}
|
|
153
|
+
for (const entry of entries) {
|
|
154
|
+
const full = path.join(dir, entry.name);
|
|
155
|
+
if (entry.isDirectory()) {
|
|
156
|
+
stack.push(full);
|
|
157
|
+
continue;
|
|
158
|
+
}
|
|
159
|
+
if (!entry.isFile())
|
|
160
|
+
continue;
|
|
161
|
+
const shape = classifyTranscriptPath(full, roots);
|
|
162
|
+
if (!shape || seen.has(full))
|
|
163
|
+
continue;
|
|
164
|
+
seen.add(full);
|
|
165
|
+
const stat = await fs.stat(full).catch((error) => {
|
|
166
|
+
result.file_read_failed += 1;
|
|
167
|
+
firstFailure ??= describeError(error);
|
|
168
|
+
return null;
|
|
169
|
+
});
|
|
170
|
+
if (!stat)
|
|
171
|
+
continue;
|
|
172
|
+
if (options.olderThanMs !== null && stat.mtimeMs >= options.olderThanMs) {
|
|
173
|
+
result.newer_count += 1;
|
|
174
|
+
continue;
|
|
175
|
+
}
|
|
176
|
+
found.push({ file: full, size: stat.size, mtimeMs: stat.mtimeMs, shape });
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
// Claude mains first, so a subagent file can take its main's content id.
|
|
180
|
+
const claudeMainIds = new Map();
|
|
181
|
+
found.sort((a, b) => shapeOrder(a.shape) - shapeOrder(b.shape) || a.file.localeCompare(b.file));
|
|
182
|
+
for (const item of found) {
|
|
183
|
+
const source = item.shape.kind === "codex_rollout" ? "codex" : "claude_code";
|
|
184
|
+
let contentId = null;
|
|
185
|
+
if (item.shape.kind !== "claude_subagent") {
|
|
186
|
+
const read = await readContentSessionId(item.file, source).then((id) => ({ ok: true, id }), (error) => {
|
|
187
|
+
result.file_read_failed += 1;
|
|
188
|
+
firstFailure ??= describeError(error);
|
|
189
|
+
return { ok: false, id: null };
|
|
190
|
+
});
|
|
191
|
+
if (!read.ok)
|
|
192
|
+
continue;
|
|
193
|
+
contentId = read.id;
|
|
194
|
+
}
|
|
195
|
+
let named;
|
|
196
|
+
if (item.shape.kind === "claude_subagent") {
|
|
197
|
+
// The reader keys a subagent file under its MAIN session's id.
|
|
198
|
+
const mainPath = path.join(path.dirname(path.dirname(path.dirname(item.file))), `${item.shape.main_session_dir}.jsonl`);
|
|
199
|
+
const known = claudeMainIds.get(mainPath);
|
|
200
|
+
named = known
|
|
201
|
+
? { session_id: known, session_id_source: "content" }
|
|
202
|
+
: sessionIdFor(mainPath, await readContentSessionId(mainPath, "claude_code").catch(() => null));
|
|
203
|
+
}
|
|
204
|
+
else {
|
|
205
|
+
named = sessionIdFor(item.file, contentId);
|
|
206
|
+
if (item.shape.kind === "claude_main")
|
|
207
|
+
claudeMainIds.set(item.file, named.session_id);
|
|
208
|
+
}
|
|
209
|
+
result.files.push({
|
|
210
|
+
source,
|
|
211
|
+
kind: item.shape.kind,
|
|
212
|
+
file_path: item.file,
|
|
213
|
+
byte_size: item.size,
|
|
214
|
+
mtime_ms: item.mtimeMs,
|
|
215
|
+
...named,
|
|
216
|
+
key: transcriptKeyFor(source, named.session_id, item.shape.kind === "claude_subagent" ? item.shape.sidecar_stem : null),
|
|
217
|
+
});
|
|
218
|
+
}
|
|
219
|
+
if (result.directory_read_failed > 0 || result.file_read_failed > 0) {
|
|
220
|
+
console.error("[collector sources] some transcripts could not be read; they are left out of the answer", JSON.stringify({
|
|
221
|
+
reason: "sources_discovery_incomplete",
|
|
222
|
+
directory_read_failed: result.directory_read_failed,
|
|
223
|
+
file_read_failed: result.file_read_failed,
|
|
224
|
+
found_count: result.files.length,
|
|
225
|
+
...(firstFailure ? { first_failure: firstFailure } : {}),
|
|
226
|
+
}));
|
|
227
|
+
}
|
|
228
|
+
return result;
|
|
229
|
+
}
|
|
230
|
+
function shapeOrder(shape) {
|
|
231
|
+
return shape.kind === "claude_main" ? 0 : shape.kind === "codex_rollout" ? 1 : 2;
|
|
232
|
+
}
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Does Tower hold this transcript, in full, in part, or not at all?
|
|
3
|
+
* (BLI-11122)
|
|
4
|
+
*
|
|
5
|
+
* One file, the current objects Tower lists for it, and this machine's cursor
|
|
6
|
+
* entry for its session. The rules, cheapest first; every answer names why:
|
|
7
|
+
*
|
|
8
|
+
* not_stored no current object for the file (`no_object`), or only the
|
|
9
|
+
* redaction-crash stub (`stub_only`): a stub holds no byte.
|
|
10
|
+
* stored_full the cursor's uploaded object is still current, the file
|
|
11
|
+
* is the size it was when uploaded and its hash is the one the
|
|
12
|
+
* cursor last recorded (`unchanged_since_upload`); or the
|
|
13
|
+
* file, redacted exactly as the upload redacts it, has the
|
|
14
|
+
* hash of a current object (`content_hash_match`) or is no
|
|
15
|
+
* larger than one (`remote_at_least_sanitized_size`).
|
|
16
|
+
* stored_partial a current object exists but is smaller than the redacted
|
|
17
|
+
* file (`changed_since_upload` when the cursor saw the file
|
|
18
|
+
* grow after its upload, else `remote_smaller`), or its size
|
|
19
|
+
* is not recorded (`remote_size_unknown`).
|
|
20
|
+
*
|
|
21
|
+
* A file too large to redact in memory is compared by its raw size
|
|
22
|
+
* (`remote_at_least_file_size`). A file whose redaction would crash to the stub
|
|
23
|
+
* is compared the same way, never by the stub's 42 bytes.
|
|
24
|
+
*
|
|
25
|
+
* Read-only: this module reads the file and never writes, moves or deletes it.
|
|
26
|
+
*/
|
|
27
|
+
import crypto from "node:crypto";
|
|
28
|
+
import fs from "node:fs/promises";
|
|
29
|
+
import path from "node:path";
|
|
30
|
+
import { RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES, REDACTION_STUB_SHA256, } from "@bli-cockpit/telemetry-core";
|
|
31
|
+
import { sanitizeTextEvidenceForUpload } from "../adapters/raw-evidence-sanitize.js";
|
|
32
|
+
import { isSecretLikePath } from "../adapters/raw-evidence-keys.js";
|
|
33
|
+
export async function judgeSource(input) {
|
|
34
|
+
const { file, remotes, cursorEntry } = input;
|
|
35
|
+
if (remotes.length === 0)
|
|
36
|
+
return { verdict: "not_stored", reason: "no_object", remote_byte_size: null };
|
|
37
|
+
const held = remotes.filter((remote) => !remote.stub);
|
|
38
|
+
if (held.length === 0)
|
|
39
|
+
return { verdict: "not_stored", reason: "stub_only", remote_byte_size: null };
|
|
40
|
+
const sizes = held.map((remote) => remote.byte_size).filter((size) => size !== null);
|
|
41
|
+
const best = sizes.length > 0 ? Math.max(...sizes) : null;
|
|
42
|
+
const full = (reason) => ({ verdict: "stored_full", reason, remote_byte_size: best });
|
|
43
|
+
const partial = (reason) => ({
|
|
44
|
+
verdict: "stored_partial",
|
|
45
|
+
reason,
|
|
46
|
+
remote_byte_size: best,
|
|
47
|
+
});
|
|
48
|
+
if (cursorEntry &&
|
|
49
|
+
!cursorEntry.stubbed &&
|
|
50
|
+
cursorEntry.uploaded_object_key &&
|
|
51
|
+
held.some((remote) => remote.object_key === cursorEntry.uploaded_object_key) &&
|
|
52
|
+
cursorEntry.uploaded_byte_size === file.byte_size &&
|
|
53
|
+
cursorEntry.file_hash_sha256 &&
|
|
54
|
+
cursorEntry.file_hash_sha256 === (await input.facts.rawSha256())) {
|
|
55
|
+
return full("unchanged_since_upload");
|
|
56
|
+
}
|
|
57
|
+
const sanitized = await input.facts.sanitized();
|
|
58
|
+
if (sanitized) {
|
|
59
|
+
if (held.some((remote) => remote.content_hash_sha256 === sanitized.sha256))
|
|
60
|
+
return full("content_hash_match");
|
|
61
|
+
if (best === null)
|
|
62
|
+
return partial("remote_size_unknown");
|
|
63
|
+
if (best >= sanitized.byte_size)
|
|
64
|
+
return full("remote_at_least_sanitized_size");
|
|
65
|
+
}
|
|
66
|
+
else {
|
|
67
|
+
if (best === null)
|
|
68
|
+
return partial("remote_size_unknown");
|
|
69
|
+
if (best >= file.byte_size)
|
|
70
|
+
return full("remote_at_least_file_size");
|
|
71
|
+
}
|
|
72
|
+
const grewAfterUpload = cursorEntry?.uploaded_object_key != null &&
|
|
73
|
+
cursorEntry.uploaded_byte_size != null &&
|
|
74
|
+
cursorEntry.uploaded_byte_size < file.byte_size;
|
|
75
|
+
return partial(grewAfterUpload ? "changed_since_upload" : "remote_smaller");
|
|
76
|
+
}
|
|
77
|
+
/**
|
|
78
|
+
* Reads the file at most once, on first ask, and redacts it exactly as the
|
|
79
|
+
* upload does (`raw-evidence-transcript-file.ts`): same sanitizer, same
|
|
80
|
+
* secret-like-name rule, same byte cap.
|
|
81
|
+
*/
|
|
82
|
+
export function fileFacts(file, maxBytes = RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES) {
|
|
83
|
+
let bytes = null;
|
|
84
|
+
const read = () => {
|
|
85
|
+
bytes ??= file.byte_size > maxBytes ? Promise.resolve(null) : fs.readFile(file.file_path);
|
|
86
|
+
return bytes;
|
|
87
|
+
};
|
|
88
|
+
let rawHash = null;
|
|
89
|
+
let sanitized = null;
|
|
90
|
+
return {
|
|
91
|
+
rawSha256() {
|
|
92
|
+
rawHash ??= read().then(async (buffer) => (buffer ? sha256(buffer) : await streamSha256(file.file_path)));
|
|
93
|
+
return rawHash;
|
|
94
|
+
},
|
|
95
|
+
sanitized() {
|
|
96
|
+
sanitized ??= read().then((buffer) => {
|
|
97
|
+
if (!buffer)
|
|
98
|
+
return null;
|
|
99
|
+
const result = sanitizeTextEvidenceForUpload({
|
|
100
|
+
text: buffer.toString("utf8"),
|
|
101
|
+
originalBytes: buffer,
|
|
102
|
+
redactedFields: [`${file.kind === "codex_rollout" ? "codex_jsonl" : file.kind === "claude_main" ? "claude_jsonl" : "claude_jsonl_sidecar"}.body`],
|
|
103
|
+
secretLikeFileName: isSecretLikePath(path.basename(file.file_path)),
|
|
104
|
+
});
|
|
105
|
+
const hash = sha256(result.bytes);
|
|
106
|
+
if (hash === REDACTION_STUB_SHA256)
|
|
107
|
+
return null;
|
|
108
|
+
return { sha256: hash, byte_size: result.bytes.byteLength };
|
|
109
|
+
});
|
|
110
|
+
return sanitized;
|
|
111
|
+
},
|
|
112
|
+
};
|
|
113
|
+
}
|
|
114
|
+
function sha256(buffer) {
|
|
115
|
+
return crypto.createHash("sha256").update(buffer).digest("hex");
|
|
116
|
+
}
|
|
117
|
+
async function streamSha256(filePath) {
|
|
118
|
+
const hash = crypto.createHash("sha256");
|
|
119
|
+
const handle = await fs.open(filePath, "r");
|
|
120
|
+
try {
|
|
121
|
+
for await (const chunk of handle.createReadStream())
|
|
122
|
+
hash.update(chunk);
|
|
123
|
+
}
|
|
124
|
+
finally {
|
|
125
|
+
await handle.close().catch(() => undefined);
|
|
126
|
+
}
|
|
127
|
+
return hash.digest("hex");
|
|
128
|
+
}
|
|
@@ -8,6 +8,13 @@ import { DEFAULT_DASHBOARD_URL, getCollectorRuntimePaths, readLocalCollectorConf
|
|
|
8
8
|
/** At most this many stubbed sessions are re-offered per tick (one backfill batch). */
|
|
9
9
|
export const SYNC_STUB_SESSIONS_PER_TICK = 25;
|
|
10
10
|
const DAY_MS = 24 * 60 * 60_000;
|
|
11
|
+
/**
|
|
12
|
+
* BLI-11119: a stubbed session a re-offer could not find in any root, by
|
|
13
|
+
* content id or file hash, is looked for again after this long. Without it
|
|
14
|
+
* every tick rescanned (and hashed) every transcript back to the session's
|
|
15
|
+
* date for a file that is gone.
|
|
16
|
+
*/
|
|
17
|
+
export const STUB_NOT_FOUND_RETRY_MS = DAY_MS;
|
|
11
18
|
const DEFAULT_DEPS = { runBackfill, loadPlan: loadStubReofferPlan };
|
|
12
19
|
export async function healStubsDuringTick(command, io, deps = DEFAULT_DEPS, now = new Date()) {
|
|
13
20
|
const summary = {
|
|
@@ -17,6 +24,7 @@ export async function healStubsDuringTick(command, io, deps = DEFAULT_DEPS, now
|
|
|
17
24
|
stubs_remaining: 0,
|
|
18
25
|
sessions_local: 0,
|
|
19
26
|
sessions_selected: 0,
|
|
27
|
+
sessions_not_found_recently: 0,
|
|
20
28
|
};
|
|
21
29
|
try {
|
|
22
30
|
summary.reason = await runStubPass(command, io, deps, now, summary);
|
|
@@ -54,13 +62,18 @@ async function runStubPass(command, io, deps, now, summary) {
|
|
|
54
62
|
summary.sessions_local = local.length;
|
|
55
63
|
if (local.length === 0)
|
|
56
64
|
return "stubs_not_on_this_machine";
|
|
57
|
-
const
|
|
65
|
+
const lookable = local.filter((session) => !notFoundRecently(session.notFoundAt, now));
|
|
66
|
+
summary.sessions_not_found_recently = local.length - lookable.length;
|
|
67
|
+
if (lookable.length === 0)
|
|
68
|
+
return "stubs_not_found_recently";
|
|
69
|
+
const selected = lookable.slice(0, SYNC_STUB_SESSIONS_PER_TICK);
|
|
58
70
|
summary.sessions_selected = selected.length;
|
|
59
71
|
const trimmed = {
|
|
60
72
|
...plan,
|
|
61
73
|
sessions: new Map(selected.map((session) => [session.key, session.pointerIds])),
|
|
62
74
|
stub_count: selected.reduce((total, session) => total + session.pointerIds.length, 0),
|
|
63
75
|
marked_local_sessions: 0,
|
|
76
|
+
found_by: new Map(),
|
|
64
77
|
};
|
|
65
78
|
const window = stubWindow(selected, sessionFile.paired_at, now);
|
|
66
79
|
const result = await deps.runBackfill({
|
|
@@ -90,11 +103,18 @@ async function locallyTrackedStubSessions(paths, plan) {
|
|
|
90
103
|
const at = key.indexOf(":");
|
|
91
104
|
const source = key.slice(0, at);
|
|
92
105
|
const entry = cursors[source]?.sessions[key.slice(at + 1)];
|
|
93
|
-
if (entry)
|
|
94
|
-
local.push({ key, pointerIds, mtimeMs: entry.file_mtime_ms });
|
|
106
|
+
if (entry) {
|
|
107
|
+
local.push({ key, pointerIds, mtimeMs: entry.file_mtime_ms, notFoundAt: entry.stub_reoffer?.not_found_at ?? null });
|
|
108
|
+
}
|
|
95
109
|
}
|
|
96
110
|
return local;
|
|
97
111
|
}
|
|
112
|
+
function notFoundRecently(notFoundAt, now) {
|
|
113
|
+
if (!notFoundAt)
|
|
114
|
+
return false;
|
|
115
|
+
const at = Date.parse(notFoundAt);
|
|
116
|
+
return Number.isFinite(at) && now.getTime() - at < STUB_NOT_FOUND_RETRY_MS;
|
|
117
|
+
}
|
|
98
118
|
/**
|
|
99
119
|
* The narrowest backfill window that still reaches every selected session:
|
|
100
120
|
* back to the oldest one's file time plus a day. A backfill window never
|
|
@@ -196,6 +196,7 @@ function parseSessionEntry(value) {
|
|
|
196
196
|
uploaded_byte_size: optionalNumberOrNull(record["uploaded_byte_size"]),
|
|
197
197
|
last_seen_at: lastSeenAt,
|
|
198
198
|
...parseStubReoffer(record["stub_reoffer"]),
|
|
199
|
+
...(record["stubbed"] === true ? { stubbed: true } : {}),
|
|
199
200
|
};
|
|
200
201
|
}
|
|
201
202
|
function parseStubReoffer(value) {
|
|
@@ -208,7 +209,15 @@ function parseStubReoffer(value) {
|
|
|
208
209
|
: [];
|
|
209
210
|
if (record["reason"] !== "stub_reoffered" || !markedAt || ids.length === 0)
|
|
210
211
|
return {};
|
|
211
|
-
|
|
212
|
+
const notFoundAt = optionalString(record["not_found_at"]);
|
|
213
|
+
return {
|
|
214
|
+
stub_reoffer: {
|
|
215
|
+
reason: "stub_reoffered",
|
|
216
|
+
stub_pointer_ids: ids,
|
|
217
|
+
marked_at: markedAt,
|
|
218
|
+
...(notFoundAt ? { not_found_at: notFoundAt } : {}),
|
|
219
|
+
},
|
|
220
|
+
};
|
|
212
221
|
}
|
|
213
222
|
function parseCheckpoints(value) {
|
|
214
223
|
if (!value || typeof value !== "object")
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A redaction-crash stub is a promise to come back, not a delivery (BLI-11119).
|
|
3
|
+
*
|
|
4
|
+
* When the redactor crashed, the collector uploaded the 42-byte marker
|
|
5
|
+
* `REDACTION_STUB_TEXT` in place of the transcript, and the cursor entry then
|
|
6
|
+
* recorded the ORIGINAL file size as uploaded. Nothing local ever considered
|
|
7
|
+
* the session undelivered. This module is the one place a cursor entry is told
|
|
8
|
+
* the truth about a stub: `stubbed: true`, `uploaded_byte_size` the marker's
|
|
9
|
+
* 42 bytes, and `byte_offset` 0 because no session byte is durable.
|
|
10
|
+
*
|
|
11
|
+
* Two ways a stub is recognised, both deterministic:
|
|
12
|
+
* - at write time, by the object key: every transcript key's last segment
|
|
13
|
+
* starts with the first 16 hex of its content hash (a re-keyed pointer keeps
|
|
14
|
+
* that prefix), and the marker always hashes to `REDACTION_STUB_SHA256`;
|
|
15
|
+
* - after the fact, by the stubs door's answer, which is the source of truth
|
|
16
|
+
* for every entry written before this rule existed.
|
|
17
|
+
*/
|
|
18
|
+
import { REDACTION_STUB_BYTE_SIZE, REDACTION_STUB_SHA256, } from "@bli-cockpit/telemetry-core";
|
|
19
|
+
const STUB_KEY_PREFIX = REDACTION_STUB_SHA256.slice(0, 16);
|
|
20
|
+
/** True when this object key names the redaction-crash marker, not a transcript. */
|
|
21
|
+
export function isRedactionStubObjectKey(objectKey) {
|
|
22
|
+
if (!objectKey)
|
|
23
|
+
return false;
|
|
24
|
+
const segment = objectKey.slice(objectKey.lastIndexOf("/") + 1);
|
|
25
|
+
return segment.startsWith(STUB_KEY_PREFIX);
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* Make the entry say what the server holds: a stub. Returns whether anything
|
|
29
|
+
* changed, so a caller reconciling old entries can count what it corrected.
|
|
30
|
+
*/
|
|
31
|
+
export function markCursorEntryStubbed(entry) {
|
|
32
|
+
const changed = entry.stubbed !== true ||
|
|
33
|
+
entry.uploaded_byte_size !== REDACTION_STUB_BYTE_SIZE ||
|
|
34
|
+
entry.byte_offset !== 0;
|
|
35
|
+
entry.stubbed = true;
|
|
36
|
+
entry.uploaded_byte_size = REDACTION_STUB_BYTE_SIZE;
|
|
37
|
+
entry.byte_offset = 0;
|
|
38
|
+
return changed;
|
|
39
|
+
}
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Asks `POST /api/ambient/evidence/transcripts` for every current transcript
|
|
3
|
+
* object this operator owns, and groups the answer by local transcript file
|
|
4
|
+
* (BLI-11122). `cockpit sources check` compares each group to the file on disk.
|
|
5
|
+
*
|
|
6
|
+
* Never throws. A door that cannot be asked (an older dashboard answers 404, a
|
|
7
|
+
* network error, a malformed page) returns `unavailable` with the reason and
|
|
8
|
+
* the check says so instead of guessing.
|
|
9
|
+
*
|
|
10
|
+
* Bounded pages of up to 500, walked by the server's keyset cursor, at most
|
|
11
|
+
* `maxPages` of them. The result says whether the list was exhausted, capped by
|
|
12
|
+
* `maxPages`, or cut short by a failed page after some landed, and carries the
|
|
13
|
+
* requested, returned and unique counts so completeness is a number, not a claim.
|
|
14
|
+
*/
|
|
15
|
+
import { EVIDENCE_TRANSCRIPTS_MAX_PAGE, EVIDENCE_TRANSCRIPTS_SCHEMA_VERSION, EvidenceTranscriptsResponseSchema, transcriptFileFromObjectKey, transcriptFileKey, } from "@bli-cockpit/telemetry-core";
|
|
16
|
+
import { describeError } from "./health-detail.js";
|
|
17
|
+
const TRANSCRIPTS_PAGE_TIMEOUT_MS = 30_000;
|
|
18
|
+
/** 400 pages of 500 is 200,000 current transcript objects: far past any operator today. */
|
|
19
|
+
export const TRANSCRIPTS_DEFAULT_MAX_PAGES = 400;
|
|
20
|
+
export async function fetchTranscriptObjects(options) {
|
|
21
|
+
const maxPages = options.maxPages ?? TRANSCRIPTS_DEFAULT_MAX_PAGES;
|
|
22
|
+
const pageSize = Math.min(options.pageSize ?? EVIDENCE_TRANSCRIPTS_MAX_PAGE, EVIDENCE_TRANSCRIPTS_MAX_PAGE);
|
|
23
|
+
const files = new Map();
|
|
24
|
+
const seen = new Set();
|
|
25
|
+
let returned = 0;
|
|
26
|
+
let unkeyed = 0;
|
|
27
|
+
let pages = 0;
|
|
28
|
+
let after = null;
|
|
29
|
+
for (;;) {
|
|
30
|
+
if (pages >= maxPages)
|
|
31
|
+
return finish("capped", "ok");
|
|
32
|
+
const page = await requestPage({ ...options, after, limit: pageSize });
|
|
33
|
+
if (!page.ok)
|
|
34
|
+
return finish(pages === 0 ? "unavailable" : "failed_partial", page.reason);
|
|
35
|
+
pages += 1;
|
|
36
|
+
for (const object of page.objects) {
|
|
37
|
+
returned += 1;
|
|
38
|
+
if (seen.has(object.ref_id))
|
|
39
|
+
continue;
|
|
40
|
+
seen.add(object.ref_id);
|
|
41
|
+
addObject(object);
|
|
42
|
+
}
|
|
43
|
+
// Advance by what the server returned; a page that names no cursor is the last.
|
|
44
|
+
if (!page.nextAfter)
|
|
45
|
+
return finish("exhausted", "ok");
|
|
46
|
+
after = page.nextAfter;
|
|
47
|
+
}
|
|
48
|
+
function addObject(object) {
|
|
49
|
+
const identity = transcriptFileFromObjectKey(object.object_key);
|
|
50
|
+
if (!identity) {
|
|
51
|
+
unkeyed += 1;
|
|
52
|
+
return;
|
|
53
|
+
}
|
|
54
|
+
const key = transcriptFileKey(identity);
|
|
55
|
+
const list = files.get(key) ?? [];
|
|
56
|
+
list.push({
|
|
57
|
+
object_key: object.object_key,
|
|
58
|
+
byte_size: object.byte_size,
|
|
59
|
+
content_hash_sha256: object.content_hash_sha256,
|
|
60
|
+
stub: object.stub,
|
|
61
|
+
});
|
|
62
|
+
files.set(key, list);
|
|
63
|
+
}
|
|
64
|
+
function finish(status, reason) {
|
|
65
|
+
const result = {
|
|
66
|
+
status,
|
|
67
|
+
reason,
|
|
68
|
+
pages,
|
|
69
|
+
requested: pages * pageSize,
|
|
70
|
+
returned,
|
|
71
|
+
unique: seen.size,
|
|
72
|
+
unkeyed,
|
|
73
|
+
files,
|
|
74
|
+
};
|
|
75
|
+
console.error("[collector sources] transcript objects listed", JSON.stringify({
|
|
76
|
+
reason: status === "unavailable" ? `transcript_door_unavailable:${reason}` : `transcripts_${status}`,
|
|
77
|
+
pages,
|
|
78
|
+
requested: result.requested,
|
|
79
|
+
returned,
|
|
80
|
+
unique: seen.size,
|
|
81
|
+
unkeyed,
|
|
82
|
+
file_count: files.size,
|
|
83
|
+
...(status === "failed_partial" ? { failure: reason } : {}),
|
|
84
|
+
}));
|
|
85
|
+
return result;
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
async function requestPage(options) {
|
|
89
|
+
const controller = new AbortController();
|
|
90
|
+
const timeout = setTimeout(() => controller.abort(), TRANSCRIPTS_PAGE_TIMEOUT_MS);
|
|
91
|
+
try {
|
|
92
|
+
const response = await options.fetchImpl(`${options.dashboardUrl}/api/ambient/evidence/transcripts`, {
|
|
93
|
+
method: "POST",
|
|
94
|
+
headers: {
|
|
95
|
+
"Content-Type": "application/json",
|
|
96
|
+
Authorization: `Bearer ${options.deviceToken}`,
|
|
97
|
+
},
|
|
98
|
+
body: JSON.stringify({
|
|
99
|
+
schema_version: EVIDENCE_TRANSCRIPTS_SCHEMA_VERSION,
|
|
100
|
+
after: options.after,
|
|
101
|
+
limit: options.limit,
|
|
102
|
+
}),
|
|
103
|
+
signal: controller.signal,
|
|
104
|
+
});
|
|
105
|
+
if (!response.ok)
|
|
106
|
+
return { ok: false, reason: `http_${response.status}` };
|
|
107
|
+
const parsed = EvidenceTranscriptsResponseSchema.safeParse(await response.json().catch(() => null));
|
|
108
|
+
if (!parsed.success)
|
|
109
|
+
return { ok: false, reason: "invalid_response" };
|
|
110
|
+
return { ok: true, objects: parsed.data.objects, nextAfter: parsed.data.next_after };
|
|
111
|
+
}
|
|
112
|
+
catch (error) {
|
|
113
|
+
return { ok: false, reason: describeError(error).error_name };
|
|
114
|
+
}
|
|
115
|
+
finally {
|
|
116
|
+
clearTimeout(timeout);
|
|
117
|
+
}
|
|
118
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@bli-cockpit/cli",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.159",
|
|
4
4
|
"private": false,
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
@@ -27,8 +27,8 @@
|
|
|
27
27
|
"test": "node dist/cli.js --help && node ../../scripts/assert-public-cli-routing.mjs && node ../../scripts/assert-public-cli-verb-help.mjs && node ../../scripts/assert-public-cli-surface-doors.mjs && node ../../scripts/assert-public-cli-runtime-files.mjs && node ../../scripts/assert-public-cli-exit-contract.mjs && node ../../scripts/assert-public-cli-no-fleet-posts.mjs && node ../../scripts/assert-public-cli-doctor-json.mjs && node ../../scripts/assert-public-package-pack.mjs --workspace=@bli-cockpit/cli"
|
|
28
28
|
},
|
|
29
29
|
"dependencies": {
|
|
30
|
-
"@bli-cockpit/memory-mcp": "0.1.
|
|
31
|
-
"@bli-cockpit/mcp": "0.1.
|
|
32
|
-
"@bli-cockpit/telemetry-core": "0.1.
|
|
30
|
+
"@bli-cockpit/memory-mcp": "0.1.43",
|
|
31
|
+
"@bli-cockpit/mcp": "0.1.77",
|
|
32
|
+
"@bli-cockpit/telemetry-core": "0.1.55"
|
|
33
33
|
}
|
|
34
34
|
}
|