@bli-cockpit/cli 0.2.157 → 0.2.158
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/codex-attribution.js +2 -0
- package/dist/commands/backfill-checkpoint.js +11 -2
- package/dist/commands/backfill-scan.js +18 -6
- package/dist/commands/backfill-stub-locate.js +104 -0
- package/dist/commands/backfill-stub-reoffer.js +39 -5
- package/dist/commands/public-root.js +1 -1
- package/dist/commands/session-sync-record.js +12 -0
- package/dist/commands/sync-stub-heal.js +23 -3
- package/dist/cursors/raw-evidence-cursor.js +10 -1
- package/dist/cursors/raw-evidence-stub-entry.js +39 -0
- package/package.json +1 -1
|
@@ -167,6 +167,7 @@ async function attributeOneSession(file, worktrees, collectionRoots) {
|
|
|
167
167
|
file_path: file.file,
|
|
168
168
|
file_name: fileName,
|
|
169
169
|
codex_session_id: sessionIdFromFileName(fileName) ?? shortHash(file.file),
|
|
170
|
+
session_id_source: sessionIdFromFileName(fileName) ? "file_name" : "path_hash",
|
|
170
171
|
cwd_basename: null,
|
|
171
172
|
cwd_hash: null,
|
|
172
173
|
session_file_mtime: new Date(file.mtimeMs).toISOString(),
|
|
@@ -197,6 +198,7 @@ async function attributeOneSession(file, worktrees, collectionRoots) {
|
|
|
197
198
|
const metaSessionId = sanitizeSessionId(signals.session_ids[0]);
|
|
198
199
|
if (metaSessionId) {
|
|
199
200
|
base.codex_session_id = metaSessionId;
|
|
201
|
+
base.session_id_source = "session_meta";
|
|
200
202
|
}
|
|
201
203
|
const primaryCwd = signals.cwds[0] ?? null;
|
|
202
204
|
if (primaryCwd) {
|
|
@@ -10,6 +10,7 @@
|
|
|
10
10
|
*/
|
|
11
11
|
import { recordBackfillCursorObservations, } from "../cursors/backfill-cursor.js";
|
|
12
12
|
import { CLAUDE_CURSOR_FILENAME, readRawEvidenceCursor, writeRawEvidenceCursor, } from "../cursors/raw-evidence-cursor.js";
|
|
13
|
+
import { isRedactionStubObjectKey, markCursorEntryStubbed } from "../cursors/raw-evidence-stub-entry.js";
|
|
13
14
|
import { isRawEvidenceUploadableAttributionState } from "../raw-evidence-attribution-policy.js";
|
|
14
15
|
import { candidateCursorKey, compareBackfillCandidates, isAfterCursor, } from "./backfill-candidates.js";
|
|
15
16
|
import { indexDurableMainObjectKeys } from "./backfill-evidence-outcomes.js";
|
|
@@ -34,10 +35,11 @@ export async function recordBackfillDurableSessionPointers(options) {
|
|
|
34
35
|
// No pointer, no prior entry, or already pointed: nothing this pass owes.
|
|
35
36
|
// BLI-11102: a pointer the server named a redaction stub is not a
|
|
36
37
|
// delivery, so a real pointer replaces it and the mark clears.
|
|
37
|
-
|
|
38
|
+
// BLI-11119: an entry the cursor itself knows is `stubbed` counts too.
|
|
39
|
+
const replacesStub = Boolean(prior?.stub_reoffer || prior?.stubbed) && prior?.uploaded_object_key !== objectKey;
|
|
38
40
|
if (!objectKey || !prior || (prior.uploaded_object_key && !replacesStub))
|
|
39
41
|
continue;
|
|
40
|
-
if (replacesStub)
|
|
42
|
+
if (replacesStub && !isRedactionStubObjectKey(objectKey))
|
|
41
43
|
stubsReplaced += 1;
|
|
42
44
|
cursor.sessions[candidate.session_id] = deliveredCursorEntry(prior, candidate, objectKey, options.now);
|
|
43
45
|
changed = true;
|
|
@@ -71,8 +73,15 @@ function deliveredCursorEntry(prior, candidate, objectKey, now) {
|
|
|
71
73
|
uploaded_byte_size: candidate.byte_size,
|
|
72
74
|
last_seen_at: now.toISOString(),
|
|
73
75
|
};
|
|
76
|
+
// BLI-11119: a re-offer that stubbed again earned another marker, not the
|
|
77
|
+
// transcript. The entry says so and keeps its mark for the next pass.
|
|
78
|
+
if (isRedactionStubObjectKey(objectKey)) {
|
|
79
|
+
markCursorEntryStubbed(entry);
|
|
80
|
+
return entry;
|
|
81
|
+
}
|
|
74
82
|
// The mark says "this pointer is a stub"; the pointer written above is not.
|
|
75
83
|
delete entry.stub_reoffer;
|
|
84
|
+
delete entry.stubbed;
|
|
76
85
|
return entry;
|
|
77
86
|
}
|
|
78
87
|
export function advanceBackfillCursorThroughResolvedPrefix(options) {
|
|
@@ -21,6 +21,7 @@ import { labelOutsideRootCandidates, outsideRootsWorkspace, } from "./backfill-o
|
|
|
21
21
|
import { reasonCountsFor } from "./backfill-reasons.js";
|
|
22
22
|
import { blockedBackfillResult } from "./backfill-result.js";
|
|
23
23
|
import { emptyStubReofferPlan, loadStubReofferPlan, reopenStubbedSessions, } from "./backfill-stub-reoffer.js";
|
|
24
|
+
import { loadStubCursorHashes, locateStubbedSessions } from "./backfill-stub-locate.js";
|
|
24
25
|
import { resolveBackfillScope } from "./backfill-scope.js";
|
|
25
26
|
import { stubSessionKey } from "../evidence-stubs-client.js";
|
|
26
27
|
/**
|
|
@@ -64,6 +65,7 @@ export async function scanBackfillRun(command, io, now) {
|
|
|
64
65
|
fetchImpl: io.fetch,
|
|
65
66
|
});
|
|
66
67
|
reopenStubbedSessions(pointerlessTerminalSessions, stubReoffer);
|
|
68
|
+
const stubCursorHashes = await loadStubCursorHashes(paths, stubReoffer);
|
|
67
69
|
const scan = await scanBackfillSessions({
|
|
68
70
|
command,
|
|
69
71
|
homeDir: command.homeDir ?? os.homedir(),
|
|
@@ -73,6 +75,8 @@ export async function scanBackfillRun(command, io, now) {
|
|
|
73
75
|
window: scope.window,
|
|
74
76
|
cursor,
|
|
75
77
|
pointerlessTerminalSessions,
|
|
78
|
+
stubReoffer,
|
|
79
|
+
stubCursorHashes,
|
|
76
80
|
now,
|
|
77
81
|
});
|
|
78
82
|
const reasonCounts = await auditScannedCandidates(scan, worktreeDiscovery.incomplete_reasons);
|
|
@@ -129,9 +133,19 @@ async function scanBackfillSessions(options) {
|
|
|
129
133
|
const claudeAttribution = options.sources.includes("claude_code")
|
|
130
134
|
? await scanClaudeHistory(options, claudeProjectsDir, scanLimit)
|
|
131
135
|
: null;
|
|
136
|
+
// BLI-11119: a stubbed session is found by the id in its content or by the
|
|
137
|
+
// file hash its cursor entry recorded, across every root the scan walked.
|
|
138
|
+
const scanned = locateStubbedSessions({
|
|
139
|
+
plan: options.stubReoffer,
|
|
140
|
+
candidates: [
|
|
141
|
+
...(codexAttribution?.results.map(normalizeCodexCandidate) ?? []),
|
|
142
|
+
...(claudeAttribution?.results.map(normalizeClaudeCandidate) ?? []),
|
|
143
|
+
],
|
|
144
|
+
cursorHashes: options.stubCursorHashes,
|
|
145
|
+
sinceMinutes: options.window.since_minutes,
|
|
146
|
+
});
|
|
132
147
|
const selection = selectBackfillCandidates({
|
|
133
|
-
|
|
134
|
-
claudeAttribution,
|
|
148
|
+
scanned,
|
|
135
149
|
cursor: options.cursor,
|
|
136
150
|
pointerlessTerminalSessions: options.pointerlessTerminalSessions,
|
|
137
151
|
maxFiles: options.command.maxFiles,
|
|
@@ -201,10 +215,7 @@ async function scanClaudeHistory(request, projectsDir, scanLimit) {
|
|
|
201
215
|
* so a `--max-files` cap spends its budget on them before newer history.
|
|
202
216
|
*/
|
|
203
217
|
function selectBackfillCandidates(options) {
|
|
204
|
-
const allCandidates =
|
|
205
|
-
...(options.codexAttribution?.results.map(normalizeCodexCandidate) ?? []),
|
|
206
|
-
...(options.claudeAttribution?.results.map(normalizeClaudeCandidate) ?? []),
|
|
207
|
-
]
|
|
218
|
+
const allCandidates = options.scanned
|
|
208
219
|
.filter((candidate) => !options.onlySessions ||
|
|
209
220
|
options.onlySessions.has(stubSessionKey(candidate.source, candidate.session_id)))
|
|
210
221
|
.filter((candidate) => isAfterCursor(candidate, options.cursor) ||
|
|
@@ -266,6 +277,7 @@ function normalizeCodexCandidate(result) {
|
|
|
266
277
|
return {
|
|
267
278
|
source: "codex",
|
|
268
279
|
session_id: result.codex_session_id,
|
|
280
|
+
session_id_source: result.session_id_source,
|
|
269
281
|
file_path: result.file_path,
|
|
270
282
|
state: result.state,
|
|
271
283
|
reason: result.reason,
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Where on this machine is the transcript behind a stub? (BLI-11119)
|
|
3
|
+
*
|
|
4
|
+
* The stubs door names a session by the id inside its object key, and the
|
|
5
|
+
* collector wrote that key with the id it read from the transcript's CONTENT
|
|
6
|
+
* (`session_meta.id` for Codex). A rollout's file name does not always carry
|
|
7
|
+
* that id: a resumed or forked session is filed as
|
|
8
|
+
* `rollout-<time>-<parent>_<child>.jsonl` while its first line names another
|
|
9
|
+
* id, and Codex moves files from `sessions/` into `archived_sessions/`. So a
|
|
10
|
+
* stubbed session is located the way it was keyed, across every root the scan
|
|
11
|
+
* walked, never by file name alone:
|
|
12
|
+
*
|
|
13
|
+
* 1. a scanned session whose id matches (`found_by: session_meta`, or
|
|
14
|
+
* `file_name` when the file's first line gave no id and the scan fell back
|
|
15
|
+
* to the name, which is then the id the key used too);
|
|
16
|
+
* 2. otherwise a scanned file whose sha256 equals the cursor entry's
|
|
17
|
+
* `file_hash_sha256` for that session (`found_by: file_hash`); it is offered
|
|
18
|
+
* under the STUBBED session's id, because the server retires a stub only for
|
|
19
|
+
* a successor that names the same session;
|
|
20
|
+
* 3. otherwise `not_found`, recorded on the cursor so the scheduled sync stops
|
|
21
|
+
* rescanning history for it every tick (see `sync-stub-heal.ts`).
|
|
22
|
+
*/
|
|
23
|
+
import { CLAUDE_CURSOR_FILENAME, readRawEvidenceCursor, } from "../cursors/raw-evidence-cursor.js";
|
|
24
|
+
import { stubSessionKey } from "../evidence-stubs-client.js";
|
|
25
|
+
/** How many per-session lines the locate log carries; the counts cover the rest. */
|
|
26
|
+
const LOGGED_SESSION_LIMIT = 25;
|
|
27
|
+
/** `codex:<id>` -> the file hash the cursor last recorded for that stubbed session. */
|
|
28
|
+
export async function loadStubCursorHashes(paths, plan) {
|
|
29
|
+
const hashes = new Map();
|
|
30
|
+
if (plan.sessions.size === 0)
|
|
31
|
+
return hashes;
|
|
32
|
+
for (const source of ["codex", "claude_code"]) {
|
|
33
|
+
const filename = source === "claude_code" ? CLAUDE_CURSOR_FILENAME : undefined;
|
|
34
|
+
const cursor = await readRawEvidenceCursor(paths, { filename }).catch(() => null);
|
|
35
|
+
if (!cursor)
|
|
36
|
+
continue;
|
|
37
|
+
for (const key of plan.sessions.keys()) {
|
|
38
|
+
if (!key.startsWith(`${source}:`))
|
|
39
|
+
continue;
|
|
40
|
+
const hash = cursor.sessions[key.slice(source.length + 1)]?.file_hash_sha256;
|
|
41
|
+
if (hash)
|
|
42
|
+
hashes.set(key, hash);
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
return hashes;
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* Match every stubbed session to a scanned file, record how on `plan.found_by`,
|
|
49
|
+
* and return the scanned candidates plus one candidate per hash match, carrying
|
|
50
|
+
* the stubbed session's id. Logs the outcome, found or not.
|
|
51
|
+
*/
|
|
52
|
+
export function locateStubbedSessions(options) {
|
|
53
|
+
const { plan } = options;
|
|
54
|
+
if (plan.sessions.size === 0)
|
|
55
|
+
return options.candidates;
|
|
56
|
+
const byKey = new Map();
|
|
57
|
+
const byHash = new Map();
|
|
58
|
+
for (const candidate of options.candidates) {
|
|
59
|
+
byKey.set(stubSessionKey(candidate.source, candidate.session_id), candidate);
|
|
60
|
+
if (candidate.content_hash_sha256) {
|
|
61
|
+
const hashKey = `${candidate.source}:${candidate.content_hash_sha256}`;
|
|
62
|
+
if (!byHash.has(hashKey))
|
|
63
|
+
byHash.set(hashKey, candidate);
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
const extra = [];
|
|
67
|
+
const sessions = [];
|
|
68
|
+
for (const key of plan.sessions.keys()) {
|
|
69
|
+
const found = locateOne(key, byKey, byHash, options.cursorHashes);
|
|
70
|
+
plan.found_by.set(key, found.foundBy);
|
|
71
|
+
if (found.candidate)
|
|
72
|
+
extra.push(found.candidate);
|
|
73
|
+
sessions.push({ session: key, found_by: found.foundBy });
|
|
74
|
+
}
|
|
75
|
+
const counts = { session_meta: 0, file_name: 0, file_hash: 0, not_found: 0 };
|
|
76
|
+
for (const session of sessions)
|
|
77
|
+
counts[session.found_by] += 1;
|
|
78
|
+
console.error("[backfill] stubbed sessions located", JSON.stringify({
|
|
79
|
+
reason: counts.not_found === sessions.length ? "stub_sessions_not_found" : "stub_sessions_located",
|
|
80
|
+
sessions: sessions.length,
|
|
81
|
+
...counts,
|
|
82
|
+
scanned_candidates: options.candidates.length,
|
|
83
|
+
since_minutes: options.sinceMinutes,
|
|
84
|
+
found_by: sessions.slice(0, LOGGED_SESSION_LIMIT),
|
|
85
|
+
}));
|
|
86
|
+
return extra.length > 0 ? [...options.candidates, ...extra] : options.candidates;
|
|
87
|
+
}
|
|
88
|
+
function locateOne(key, byKey, byHash, cursorHashes) {
|
|
89
|
+
const byId = byKey.get(key);
|
|
90
|
+
if (byId) {
|
|
91
|
+
const fromContent = byId.source === "codex" && byId.session_id_source === "session_meta";
|
|
92
|
+
return { foundBy: fromContent ? "session_meta" : "file_name" };
|
|
93
|
+
}
|
|
94
|
+
const at = key.indexOf(":");
|
|
95
|
+
const source = key.slice(0, at);
|
|
96
|
+
const hash = cursorHashes.get(key);
|
|
97
|
+
const byContent = hash ? byHash.get(`${source}:${hash}`) : undefined;
|
|
98
|
+
if (!byContent)
|
|
99
|
+
return { foundBy: "not_found" };
|
|
100
|
+
return {
|
|
101
|
+
foundBy: "file_hash",
|
|
102
|
+
candidate: { ...byContent, session_id: key.slice(at + 1), session_id_source: undefined },
|
|
103
|
+
};
|
|
104
|
+
}
|
|
@@ -1,10 +1,18 @@
|
|
|
1
1
|
import { CLAUDE_CURSOR_FILENAME, readRawEvidenceCursor, writeRawEvidenceCursor, } from "../cursors/raw-evidence-cursor.js";
|
|
2
|
+
import { markCursorEntryStubbed } from "../cursors/raw-evidence-stub-entry.js";
|
|
2
3
|
import { fetchStubbedSessions, stubSessionKey } from "../evidence-stubs-client.js";
|
|
3
4
|
import { describeError } from "../health-detail.js";
|
|
4
5
|
import { readLocalCollectorSessionFile } from "../local-state.js";
|
|
5
6
|
import { candidateCursorKey } from "./backfill-candidates.js";
|
|
6
7
|
export function emptyStubReofferPlan(reason) {
|
|
7
|
-
return {
|
|
8
|
+
return {
|
|
9
|
+
status: "skipped",
|
|
10
|
+
reason,
|
|
11
|
+
sessions: new Map(),
|
|
12
|
+
stub_count: 0,
|
|
13
|
+
marked_local_sessions: 0,
|
|
14
|
+
found_by: new Map(),
|
|
15
|
+
};
|
|
8
16
|
}
|
|
9
17
|
/** Ask the server which of this operator's pointers are stubs. Never throws; writes nothing. */
|
|
10
18
|
export async function loadStubReofferPlan(options) {
|
|
@@ -25,6 +33,7 @@ export async function loadStubReofferPlan(options) {
|
|
|
25
33
|
sessions,
|
|
26
34
|
stub_count: answer.unique_stub_count,
|
|
27
35
|
marked_local_sessions: 0,
|
|
36
|
+
found_by: new Map(),
|
|
28
37
|
};
|
|
29
38
|
}
|
|
30
39
|
/**
|
|
@@ -34,7 +43,7 @@ export async function loadStubReofferPlan(options) {
|
|
|
34
43
|
export async function markStubReoffers(paths, plan, now) {
|
|
35
44
|
if (plan.sessions.size === 0)
|
|
36
45
|
return;
|
|
37
|
-
plan.marked_local_sessions = await markStubbedSessions(paths, plan
|
|
46
|
+
plan.marked_local_sessions = await markStubbedSessions(paths, plan, now);
|
|
38
47
|
}
|
|
39
48
|
/** Reopen stubbed sessions exactly like pointer-less ones: selected past the cursor, sorted first. */
|
|
40
49
|
export function reopenStubbedSessions(reopen, plan) {
|
|
@@ -67,6 +76,7 @@ export function summarizeStubReoffer(plan, candidates, durableCandidateKeys = ne
|
|
|
67
76
|
reoffered_sessions: reoffered,
|
|
68
77
|
reoffered_stubs: stubsOf(offered.filter((candidate) => attempted.includes(candidate))),
|
|
69
78
|
replaced_stubs: stubsOf(offered.filter((candidate) => durableCandidateKeys.has(candidateCursorKey(candidate)))),
|
|
79
|
+
found_by: countFoundBy(plan),
|
|
70
80
|
};
|
|
71
81
|
if (plan.sessions.size > 0) {
|
|
72
82
|
console.error("[backfill] redaction-stubbed sessions re-offered", JSON.stringify({
|
|
@@ -79,20 +89,43 @@ export function summarizeStubReoffer(plan, candidates, durableCandidateKeys = ne
|
|
|
79
89
|
}
|
|
80
90
|
return summary;
|
|
81
91
|
}
|
|
82
|
-
|
|
92
|
+
function countFoundBy(plan) {
|
|
93
|
+
const counts = { session_meta: 0, file_name: 0, file_hash: 0, not_found: 0 };
|
|
94
|
+
for (const method of plan.found_by.values())
|
|
95
|
+
counts[method] += 1;
|
|
96
|
+
return counts;
|
|
97
|
+
}
|
|
98
|
+
/**
|
|
99
|
+
* Mark every door-named session this cursor tracks, and make its entry tell
|
|
100
|
+
* the truth (BLI-11119): an entry written before the stub rule existed claimed
|
|
101
|
+
* the full file as uploaded. The door is the source of truth, so each such
|
|
102
|
+
* entry is reconciled to `stubbed: true` with the marker's bytes, counted and
|
|
103
|
+
* logged as `stub_cursor_reconciled`. A session the scan could not find in any
|
|
104
|
+
* root keeps `not_found_at`, so the scheduled sync waits before looking again.
|
|
105
|
+
*/
|
|
106
|
+
async function markStubbedSessions(paths, plan, now) {
|
|
83
107
|
let marked = 0;
|
|
108
|
+
let reconciled = 0;
|
|
84
109
|
for (const source of ["codex", "claude_code"]) {
|
|
85
110
|
const filename = source === "claude_code" ? CLAUDE_CURSOR_FILENAME : undefined;
|
|
86
111
|
try {
|
|
87
112
|
const cursor = await readRawEvidenceCursor(paths, { filename });
|
|
88
113
|
let changed = false;
|
|
89
|
-
for (const [key, pointerIds] of sessions) {
|
|
114
|
+
for (const [key, pointerIds] of plan.sessions) {
|
|
90
115
|
if (!key.startsWith(`${source}:`))
|
|
91
116
|
continue;
|
|
92
117
|
const entry = cursor.sessions[key.slice(source.length + 1)];
|
|
93
118
|
if (!entry)
|
|
94
119
|
continue;
|
|
95
|
-
|
|
120
|
+
const notFound = plan.found_by.get(key) === "not_found";
|
|
121
|
+
entry.stub_reoffer = {
|
|
122
|
+
reason: "stub_reoffered",
|
|
123
|
+
stub_pointer_ids: pointerIds,
|
|
124
|
+
marked_at: now.toISOString(),
|
|
125
|
+
...(notFound ? { not_found_at: now.toISOString() } : {}),
|
|
126
|
+
};
|
|
127
|
+
if (markCursorEntryStubbed(entry))
|
|
128
|
+
reconciled += 1;
|
|
96
129
|
changed = true;
|
|
97
130
|
marked += 1;
|
|
98
131
|
}
|
|
@@ -105,5 +138,6 @@ async function markStubbedSessions(paths, sessions, now) {
|
|
|
105
138
|
console.error("[backfill] stub mark not written to the cursor", JSON.stringify({ reason: "stub_mark_write_failed", source, ...describeError(error) }));
|
|
106
139
|
}
|
|
107
140
|
}
|
|
141
|
+
console.error("[backfill] stubbed sessions' cursor entries reconciled with the stubs door", JSON.stringify({ reason: "stub_cursor_reconciled", count: reconciled, marked_local_sessions: marked }));
|
|
108
142
|
return marked;
|
|
109
143
|
}
|
|
@@ -21,7 +21,7 @@ export async function runCockpitCli(argv, io) {
|
|
|
21
21
|
}
|
|
22
22
|
|
|
23
23
|
if (command === "--version" || command === "-V" || command === "version") {
|
|
24
|
-
writeLine(io?.stdout ?? process.stdout, "0.2.
|
|
24
|
+
writeLine(io?.stdout ?? process.stdout, "0.2.158");
|
|
25
25
|
return 0;
|
|
26
26
|
}
|
|
27
27
|
|
|
@@ -11,6 +11,7 @@ import { DEFAULT_DASHBOARD_URL, getCollectorRuntimePaths, readLocalCollectorConf
|
|
|
11
11
|
import { normalizeDashboardUrl } from "../upload-http.js";
|
|
12
12
|
import { flushPendingCodexSessionReports, queueCodexSessionReport, } from "../upload.js";
|
|
13
13
|
import { CLAUDE_CURSOR_FILENAME, countStaleSessions, readRawEvidenceCursor, recordSessionObservation, writeRawEvidenceCursor, } from "../cursors/raw-evidence-cursor.js";
|
|
14
|
+
import { isRedactionStubObjectKey, markCursorEntryStubbed } from "../cursors/raw-evidence-stub-entry.js";
|
|
14
15
|
import { claudeAttributionReadFailureCount, codexAttributionReadFailureCount, } from "./agent-session-report.js";
|
|
15
16
|
import { liveSyncCursorEntryRequiresRetry } from "./session-sync-attribution.js";
|
|
16
17
|
/**
|
|
@@ -331,5 +332,16 @@ function recordSourceObservation(options, result) {
|
|
|
331
332
|
uploaded_byte_size: uploadedByteSize,
|
|
332
333
|
last_seen_at: options.now.toISOString(),
|
|
333
334
|
};
|
|
335
|
+
// BLI-11119: a pointer that is the redaction-crash marker is not a delivery.
|
|
336
|
+
// The entry records the marker's bytes, and keeps the re-offer mark while
|
|
337
|
+
// the stub still stands.
|
|
338
|
+
if (isRedactionStubObjectKey(uploadedObjectKey)) {
|
|
339
|
+
markCursorEntryStubbed(entry);
|
|
340
|
+
if (prior?.stub_reoffer)
|
|
341
|
+
entry.stub_reoffer = prior.stub_reoffer;
|
|
342
|
+
if (uploadedThisSync) {
|
|
343
|
+
console.error("[collector sync] session delivered only as a redaction-crash stub", JSON.stringify({ reason: "stub_cursor_recorded", source: options.source, byte_size: result.byte_size }));
|
|
344
|
+
}
|
|
345
|
+
}
|
|
334
346
|
recordSessionObservation(options.cursor, sessionId, entry);
|
|
335
347
|
}
|
|
@@ -8,6 +8,13 @@ import { DEFAULT_DASHBOARD_URL, getCollectorRuntimePaths, readLocalCollectorConf
|
|
|
8
8
|
/** At most this many stubbed sessions are re-offered per tick (one backfill batch). */
|
|
9
9
|
export const SYNC_STUB_SESSIONS_PER_TICK = 25;
|
|
10
10
|
const DAY_MS = 24 * 60 * 60_000;
|
|
11
|
+
/**
|
|
12
|
+
* BLI-11119: a stubbed session a re-offer could not find in any root, by
|
|
13
|
+
* content id or file hash, is looked for again after this long. Without it
|
|
14
|
+
* every tick rescanned (and hashed) every transcript back to the session's
|
|
15
|
+
* date for a file that is gone.
|
|
16
|
+
*/
|
|
17
|
+
export const STUB_NOT_FOUND_RETRY_MS = DAY_MS;
|
|
11
18
|
const DEFAULT_DEPS = { runBackfill, loadPlan: loadStubReofferPlan };
|
|
12
19
|
export async function healStubsDuringTick(command, io, deps = DEFAULT_DEPS, now = new Date()) {
|
|
13
20
|
const summary = {
|
|
@@ -17,6 +24,7 @@ export async function healStubsDuringTick(command, io, deps = DEFAULT_DEPS, now
|
|
|
17
24
|
stubs_remaining: 0,
|
|
18
25
|
sessions_local: 0,
|
|
19
26
|
sessions_selected: 0,
|
|
27
|
+
sessions_not_found_recently: 0,
|
|
20
28
|
};
|
|
21
29
|
try {
|
|
22
30
|
summary.reason = await runStubPass(command, io, deps, now, summary);
|
|
@@ -54,13 +62,18 @@ async function runStubPass(command, io, deps, now, summary) {
|
|
|
54
62
|
summary.sessions_local = local.length;
|
|
55
63
|
if (local.length === 0)
|
|
56
64
|
return "stubs_not_on_this_machine";
|
|
57
|
-
const
|
|
65
|
+
const lookable = local.filter((session) => !notFoundRecently(session.notFoundAt, now));
|
|
66
|
+
summary.sessions_not_found_recently = local.length - lookable.length;
|
|
67
|
+
if (lookable.length === 0)
|
|
68
|
+
return "stubs_not_found_recently";
|
|
69
|
+
const selected = lookable.slice(0, SYNC_STUB_SESSIONS_PER_TICK);
|
|
58
70
|
summary.sessions_selected = selected.length;
|
|
59
71
|
const trimmed = {
|
|
60
72
|
...plan,
|
|
61
73
|
sessions: new Map(selected.map((session) => [session.key, session.pointerIds])),
|
|
62
74
|
stub_count: selected.reduce((total, session) => total + session.pointerIds.length, 0),
|
|
63
75
|
marked_local_sessions: 0,
|
|
76
|
+
found_by: new Map(),
|
|
64
77
|
};
|
|
65
78
|
const window = stubWindow(selected, sessionFile.paired_at, now);
|
|
66
79
|
const result = await deps.runBackfill({
|
|
@@ -90,11 +103,18 @@ async function locallyTrackedStubSessions(paths, plan) {
|
|
|
90
103
|
const at = key.indexOf(":");
|
|
91
104
|
const source = key.slice(0, at);
|
|
92
105
|
const entry = cursors[source]?.sessions[key.slice(at + 1)];
|
|
93
|
-
if (entry)
|
|
94
|
-
local.push({ key, pointerIds, mtimeMs: entry.file_mtime_ms });
|
|
106
|
+
if (entry) {
|
|
107
|
+
local.push({ key, pointerIds, mtimeMs: entry.file_mtime_ms, notFoundAt: entry.stub_reoffer?.not_found_at ?? null });
|
|
108
|
+
}
|
|
95
109
|
}
|
|
96
110
|
return local;
|
|
97
111
|
}
|
|
112
|
+
function notFoundRecently(notFoundAt, now) {
|
|
113
|
+
if (!notFoundAt)
|
|
114
|
+
return false;
|
|
115
|
+
const at = Date.parse(notFoundAt);
|
|
116
|
+
return Number.isFinite(at) && now.getTime() - at < STUB_NOT_FOUND_RETRY_MS;
|
|
117
|
+
}
|
|
98
118
|
/**
|
|
99
119
|
* The narrowest backfill window that still reaches every selected session:
|
|
100
120
|
* back to the oldest one's file time plus a day. A backfill window never
|
|
@@ -196,6 +196,7 @@ function parseSessionEntry(value) {
|
|
|
196
196
|
uploaded_byte_size: optionalNumberOrNull(record["uploaded_byte_size"]),
|
|
197
197
|
last_seen_at: lastSeenAt,
|
|
198
198
|
...parseStubReoffer(record["stub_reoffer"]),
|
|
199
|
+
...(record["stubbed"] === true ? { stubbed: true } : {}),
|
|
199
200
|
};
|
|
200
201
|
}
|
|
201
202
|
function parseStubReoffer(value) {
|
|
@@ -208,7 +209,15 @@ function parseStubReoffer(value) {
|
|
|
208
209
|
: [];
|
|
209
210
|
if (record["reason"] !== "stub_reoffered" || !markedAt || ids.length === 0)
|
|
210
211
|
return {};
|
|
211
|
-
|
|
212
|
+
const notFoundAt = optionalString(record["not_found_at"]);
|
|
213
|
+
return {
|
|
214
|
+
stub_reoffer: {
|
|
215
|
+
reason: "stub_reoffered",
|
|
216
|
+
stub_pointer_ids: ids,
|
|
217
|
+
marked_at: markedAt,
|
|
218
|
+
...(notFoundAt ? { not_found_at: notFoundAt } : {}),
|
|
219
|
+
},
|
|
220
|
+
};
|
|
212
221
|
}
|
|
213
222
|
function parseCheckpoints(value) {
|
|
214
223
|
if (!value || typeof value !== "object")
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A redaction-crash stub is a promise to come back, not a delivery (BLI-11119).
|
|
3
|
+
*
|
|
4
|
+
* When the redactor crashed, the collector uploaded the 42-byte marker
|
|
5
|
+
* `REDACTION_STUB_TEXT` in place of the transcript, and the cursor entry then
|
|
6
|
+
* recorded the ORIGINAL file size as uploaded. Nothing local ever considered
|
|
7
|
+
* the session undelivered. This module is the one place a cursor entry is told
|
|
8
|
+
* the truth about a stub: `stubbed: true`, `uploaded_byte_size` the marker's
|
|
9
|
+
* 42 bytes, and `byte_offset` 0 because no session byte is durable.
|
|
10
|
+
*
|
|
11
|
+
* Two ways a stub is recognised, both deterministic:
|
|
12
|
+
* - at write time, by the object key: every transcript key's last segment
|
|
13
|
+
* starts with the first 16 hex of its content hash (a re-keyed pointer keeps
|
|
14
|
+
* that prefix), and the marker always hashes to `REDACTION_STUB_SHA256`;
|
|
15
|
+
* - after the fact, by the stubs door's answer, which is the source of truth
|
|
16
|
+
* for every entry written before this rule existed.
|
|
17
|
+
*/
|
|
18
|
+
import { REDACTION_STUB_BYTE_SIZE, REDACTION_STUB_SHA256, } from "@bli-cockpit/telemetry-core";
|
|
19
|
+
const STUB_KEY_PREFIX = REDACTION_STUB_SHA256.slice(0, 16);
|
|
20
|
+
/** True when this object key names the redaction-crash marker, not a transcript. */
|
|
21
|
+
export function isRedactionStubObjectKey(objectKey) {
|
|
22
|
+
if (!objectKey)
|
|
23
|
+
return false;
|
|
24
|
+
const segment = objectKey.slice(objectKey.lastIndexOf("/") + 1);
|
|
25
|
+
return segment.startsWith(STUB_KEY_PREFIX);
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* Make the entry say what the server holds: a stub. Returns whether anything
|
|
29
|
+
* changed, so a caller reconciling old entries can count what it corrected.
|
|
30
|
+
*/
|
|
31
|
+
export function markCursorEntryStubbed(entry) {
|
|
32
|
+
const changed = entry.stubbed !== true ||
|
|
33
|
+
entry.uploaded_byte_size !== REDACTION_STUB_BYTE_SIZE ||
|
|
34
|
+
entry.byte_offset !== 0;
|
|
35
|
+
entry.stubbed = true;
|
|
36
|
+
entry.uploaded_byte_size = REDACTION_STUB_BYTE_SIZE;
|
|
37
|
+
entry.byte_offset = 0;
|
|
38
|
+
return changed;
|
|
39
|
+
}
|