@bli-cockpit/cli 0.2.155 → 0.2.157
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/local-sources.js +1 -0
- package/dist/adapters/raw-evidence-collection-state.js +1 -0
- package/dist/adapters/raw-evidence-snapshot-supersede.js +29 -0
- package/dist/adapters/raw-evidence-transcript-file.js +13 -3
- package/dist/commands/backfill-checkpoint.js +12 -3
- package/dist/commands/backfill-report.js +23 -11
- package/dist/commands/backfill-scan.js +23 -0
- package/dist/commands/backfill-stub-reoffer.js +109 -0
- package/dist/commands/backfill-upload.js +14 -3
- package/dist/commands/backfill.js +36 -17
- package/dist/commands/public-root.js +1 -1
- package/dist/commands/session-sync-upload.js +13 -9
- package/dist/commands/session-sync.js +1 -0
- package/dist/commands/sync-run.js +8 -1
- package/dist/commands/sync-stub-heal.js +135 -0
- package/dist/commands/sync-yield.js +61 -0
- package/dist/cursors/raw-evidence-cursor.js +13 -0
- package/dist/evidence-stubs-client.js +115 -0
- package/dist/upload-envelope-build.js +1 -0
- package/package.json +4 -4
|
@@ -35,6 +35,7 @@ export async function runLocalSourceCollectors(options) {
|
|
|
35
35
|
claudeAttributionScan: options.rawEvidenceClaudeAttributionScan,
|
|
36
36
|
skipContentHashes: options.rawEvidenceSkipContentHashes,
|
|
37
37
|
priorSnapshots: options.rawEvidencePriorSnapshots,
|
|
38
|
+
stubReoffers: options.rawEvidenceStubReoffers,
|
|
38
39
|
byteBudget: options.rawEvidenceByteBudget,
|
|
39
40
|
objectBudget: options.rawEvidenceObjectBudget,
|
|
40
41
|
budget: options.rawEvidenceBudget,
|
|
@@ -74,6 +74,7 @@ export async function openCollection(context, options, places) {
|
|
|
74
74
|
caps: startingCaps({ byteBudget, objectBudget, budget: options.budget }),
|
|
75
75
|
skipContentHashes: options.skipContentHashes ?? new Set(),
|
|
76
76
|
priorSnapshots: options.priorSnapshots ?? new Map(),
|
|
77
|
+
stubReoffers: options.stubReoffers ?? new Map(),
|
|
77
78
|
budget: options.budget ?? {
|
|
78
79
|
remainingBytes: byteBudget,
|
|
79
80
|
remainingObjects: objectBudget,
|
|
@@ -20,6 +20,7 @@
|
|
|
20
20
|
* says so: `snapshot_rewritten`, both sizes, and the byte range the first
|
|
21
21
|
* difference falls in. Never content.
|
|
22
22
|
*/
|
|
23
|
+
import { REDACTION_STUB_LABEL } from "@bli-cockpit/telemetry-core";
|
|
23
24
|
import { firstDifferenceRange, isPrefixSuccessor, prefixCheckpoints, } from "@bli-cockpit/telemetry-core/snapshot-supersede";
|
|
24
25
|
/**
|
|
25
26
|
* The newest committed snapshot per source key, from the cursor's objects map,
|
|
@@ -79,4 +80,32 @@ export function supersedeClaimFor(options) {
|
|
|
79
80
|
first_differing_offset_below: range.below,
|
|
80
81
|
}));
|
|
81
82
|
return null;
|
|
83
|
+
}
|
|
84
|
+
/**
|
|
85
|
+
* BLI-11102: the claim a re-offered transcript makes when the server named its
|
|
86
|
+
* stored copy a redaction-crash stub. Only a main transcript that redacted
|
|
87
|
+
* cleanly this time claims; one that stubbed again claims nothing and keeps
|
|
88
|
+
* the ordinary path. Logs the decision either way.
|
|
89
|
+
*/
|
|
90
|
+
export function stubReplacementClaim(options) {
|
|
91
|
+
if (!options.stubPointerId || (options.kind !== "codex_jsonl" && options.kind !== "claude_jsonl"))
|
|
92
|
+
return null;
|
|
93
|
+
if (options.completenessLabel === REDACTION_STUB_LABEL) {
|
|
94
|
+
console.error("[evidence-upload] stubbed session re-offered but it stubbed again; no claim", JSON.stringify({
|
|
95
|
+
reason: "stub_reoffer_still_stubbed",
|
|
96
|
+
kind: options.kind,
|
|
97
|
+
byte_size: options.byteSize,
|
|
98
|
+
}));
|
|
99
|
+
return null;
|
|
100
|
+
}
|
|
101
|
+
console.error("[evidence-upload] stubbed session re-offered; claiming to replace the stub", JSON.stringify({
|
|
102
|
+
reason: "stub_reoffered",
|
|
103
|
+
rule: "stub_replaced",
|
|
104
|
+
kind: options.kind,
|
|
105
|
+
byte_size: options.byteSize,
|
|
106
|
+
// The same bytes may already be durable (a later sync healed it without
|
|
107
|
+
// knowing about the stub); they are staged anyway so the claim has a carrier.
|
|
108
|
+
already_durable: options.alreadyDurable,
|
|
109
|
+
}));
|
|
110
|
+
return { pointer_id: options.stubPointerId, rule: "stub_replaced" };
|
|
82
111
|
}
|
|
@@ -19,7 +19,7 @@ import { isSecretLikePath, sha256, shortHash, } from "./raw-evidence-keys.js";
|
|
|
19
19
|
import { maskSecretsInTranscript, } from "./raw-evidence-sanitize.js";
|
|
20
20
|
import { markBudgetCapApplied, markCapApplied, } from "./raw-evidence-completeness.js";
|
|
21
21
|
import { evidenceEntry } from "./raw-evidence-manifest.js";
|
|
22
|
-
import { describeSnapshot, supersedeClaimFor } from "./raw-evidence-snapshot-supersede.js";
|
|
22
|
+
import { describeSnapshot, stubReplacementClaim, supersedeClaimFor, } from "./raw-evidence-snapshot-supersede.js";
|
|
23
23
|
import { admitToBudget, stageEvidenceBytes, } from "./raw-evidence-collection-state.js";
|
|
24
24
|
/**
|
|
25
25
|
* Reads, secret-guards, content-addresses, budget-checks, and copies one
|
|
@@ -64,7 +64,16 @@ export async function collectOneEvidenceFile(collection, options) {
|
|
|
64
64
|
secretLikeFileName: names.secretLikeFileName,
|
|
65
65
|
});
|
|
66
66
|
const contentHash = sha256(sanitized.bytes);
|
|
67
|
-
|
|
67
|
+
// BLI-11102: a session the server named a redaction stub is re-offered even
|
|
68
|
+
// when these bytes are already durable, because the stub is not a delivery.
|
|
69
|
+
const stubClaim = stubReplacementClaim({
|
|
70
|
+
kind: options.kind,
|
|
71
|
+
stubPointerId: collection.stubReoffers.get(options.sessionId),
|
|
72
|
+
completenessLabel: sanitized.completenessLabel,
|
|
73
|
+
byteSize: sanitized.bytes.byteLength,
|
|
74
|
+
alreadyDurable: collection.skipContentHashes.has(contentHash),
|
|
75
|
+
});
|
|
76
|
+
if (collection.skipContentHashes.has(contentHash) && !stubClaim) {
|
|
68
77
|
collection.reused.push({
|
|
69
78
|
kind: options.kind,
|
|
70
79
|
label: names.evidenceLabel,
|
|
@@ -83,6 +92,7 @@ export async function collectOneEvidenceFile(collection, options) {
|
|
|
83
92
|
names,
|
|
84
93
|
sanitized,
|
|
85
94
|
contentHash,
|
|
95
|
+
stubClaim,
|
|
86
96
|
});
|
|
87
97
|
return true;
|
|
88
98
|
}
|
|
@@ -139,7 +149,7 @@ async function stageOneTranscript(collection, file) {
|
|
|
139
149
|
// BLI-11088: remember this snapshot's shape, and claim the last one when
|
|
140
150
|
// these bytes rebuild it.
|
|
141
151
|
snapshot: describeSnapshot(file.sanitized.bytes),
|
|
142
|
-
supersedes: supersedeClaimFor({
|
|
152
|
+
supersedes: file.stubClaim ?? supersedeClaimFor({
|
|
143
153
|
bytes: file.sanitized.bytes,
|
|
144
154
|
contentHash: file.contentHash,
|
|
145
155
|
kind: file.kind,
|
|
@@ -21,6 +21,7 @@ import { indexDurableMainObjectKeys } from "./backfill-evidence-outcomes.js";
|
|
|
21
21
|
export async function recordBackfillDurableSessionPointers(options) {
|
|
22
22
|
const durableObjectBySession = indexDurableMainObjectKeys(options.syncResults);
|
|
23
23
|
let recordedCount = 0;
|
|
24
|
+
let stubsReplaced = 0;
|
|
24
25
|
for (const source of ["codex", "claude_code"]) {
|
|
25
26
|
const filename = source === "claude_code" ? CLAUDE_CURSOR_FILENAME : undefined;
|
|
26
27
|
const cursor = await readRawEvidenceCursor(options.paths, { filename });
|
|
@@ -31,8 +32,13 @@ export async function recordBackfillDurableSessionPointers(options) {
|
|
|
31
32
|
const objectKey = durableObjectBySession.get(`${source}:${candidate.session_id}`);
|
|
32
33
|
const prior = cursor.sessions[candidate.session_id];
|
|
33
34
|
// No pointer, no prior entry, or already pointed: nothing this pass owes.
|
|
34
|
-
|
|
35
|
+
// BLI-11102: a pointer the server named a redaction stub is not a
|
|
36
|
+
// delivery, so a real pointer replaces it and the mark clears.
|
|
37
|
+
const replacesStub = Boolean(prior?.stub_reoffer) && prior?.uploaded_object_key !== objectKey;
|
|
38
|
+
if (!objectKey || !prior || (prior.uploaded_object_key && !replacesStub))
|
|
35
39
|
continue;
|
|
40
|
+
if (replacesStub)
|
|
41
|
+
stubsReplaced += 1;
|
|
36
42
|
cursor.sessions[candidate.session_id] = deliveredCursorEntry(prior, candidate, objectKey, options.now);
|
|
37
43
|
changed = true;
|
|
38
44
|
recordedCount += 1;
|
|
@@ -45,12 +51,12 @@ export async function recordBackfillDurableSessionPointers(options) {
|
|
|
45
51
|
}
|
|
46
52
|
}
|
|
47
53
|
if (recordedCount > 0) {
|
|
48
|
-
console.error("[backfill] terminal session pointers recorded", JSON.stringify({ count: recordedCount }));
|
|
54
|
+
console.error("[backfill] terminal session pointers recorded", JSON.stringify({ count: recordedCount, stub_pointers_replaced: stubsReplaced }));
|
|
49
55
|
}
|
|
50
56
|
}
|
|
51
57
|
/** The cursor entry for a session backfill has just delivered in full. */
|
|
52
58
|
function deliveredCursorEntry(prior, candidate, objectKey, now) {
|
|
53
|
-
|
|
59
|
+
const entry = {
|
|
54
60
|
...prior,
|
|
55
61
|
file_hash_sha256: candidate.content_hash_sha256,
|
|
56
62
|
file_mtime_ms: candidate.session_file_mtime_ms,
|
|
@@ -65,6 +71,9 @@ function deliveredCursorEntry(prior, candidate, objectKey, now) {
|
|
|
65
71
|
uploaded_byte_size: candidate.byte_size,
|
|
66
72
|
last_seen_at: now.toISOString(),
|
|
67
73
|
};
|
|
74
|
+
// The mark says "this pointer is a stub"; the pointer written above is not.
|
|
75
|
+
delete entry.stub_reoffer;
|
|
76
|
+
return entry;
|
|
68
77
|
}
|
|
69
78
|
export function advanceBackfillCursorThroughResolvedPrefix(options) {
|
|
70
79
|
if (!options.discoveryComplete)
|
|
@@ -7,6 +7,7 @@ import { advanceBackfillCursorThroughResolvedPrefix, recordBackfillDurableSessio
|
|
|
7
7
|
import { blockingScanIssues, cappedSidecarCount } from "./backfill-issues.js";
|
|
8
8
|
import { backfillResultBaseArgs, baseBackfillResult, emptyReport, } from "./backfill-result.js";
|
|
9
9
|
import { buildBackfillSessionReport } from "./backfill-session-report.js";
|
|
10
|
+
import { summarizeStubReoffer } from "./backfill-stub-reoffer.js";
|
|
10
11
|
/**
|
|
11
12
|
* REPORT: post the session report, record durable pointers, advance the
|
|
12
13
|
* cursor through the resolved contiguous prefix, decide completion, write the
|
|
@@ -25,7 +26,11 @@ export async function reportBackfillOutcome(command, io, ctx, upload) {
|
|
|
25
26
|
failureReason ??= sessionReportFailureReason(posted);
|
|
26
27
|
}
|
|
27
28
|
if (posted.acknowledged) {
|
|
28
|
-
|
|
29
|
+
// BLI-11112: the sync's stub pass saw only stubbed sessions, so it records
|
|
30
|
+
// their pointers and never moves the history cursor past anything else.
|
|
31
|
+
await checkpointResolvedBackfillProgress(ctx, upload, {
|
|
32
|
+
advanceCursor: !command.stubPass,
|
|
33
|
+
});
|
|
29
34
|
}
|
|
30
35
|
const completion = summarizeBackfillCompletion({
|
|
31
36
|
ctx,
|
|
@@ -34,17 +39,22 @@ export async function reportBackfillOutcome(command, io, ctx, upload) {
|
|
|
34
39
|
stoppedEarly: Boolean(blockedAt),
|
|
35
40
|
failureReason,
|
|
36
41
|
});
|
|
37
|
-
if (completion.status === "complete" && command.all) {
|
|
42
|
+
if (completion.status === "complete" && command.all && !command.stubPass) {
|
|
38
43
|
await writeAllHistoryCompletionMarker(ctx);
|
|
39
44
|
}
|
|
40
|
-
return
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
45
|
+
return {
|
|
46
|
+
...assembleBackfillResult({
|
|
47
|
+
command,
|
|
48
|
+
ctx,
|
|
49
|
+
upload,
|
|
50
|
+
posted,
|
|
51
|
+
completion,
|
|
52
|
+
blockedAt,
|
|
53
|
+
}),
|
|
54
|
+
stub_reoffer: summarizeStubReoffer(ctx.stubReoffer, upload.uploadable, posted.acknowledged ? upload.durableCandidateKeys : new Set(),
|
|
55
|
+
// One sync result per batch the loop reached; a stopped run reached fewer.
|
|
56
|
+
upload.batches.slice(0, upload.syncResults.length).flatMap((batch) => batch.candidates)),
|
|
57
|
+
};
|
|
48
58
|
}
|
|
49
59
|
/**
|
|
50
60
|
* Tells the server what this run observed about every session, uploaded or
|
|
@@ -110,7 +120,7 @@ function sessionReportFailureReason(posted) {
|
|
|
110
120
|
* both writes wait on the report acknowledgement, because a cursor advanced
|
|
111
121
|
* past a session the server never recorded can never be walked back.
|
|
112
122
|
*/
|
|
113
|
-
async function checkpointResolvedBackfillProgress(ctx, upload) {
|
|
123
|
+
async function checkpointResolvedBackfillProgress(ctx, upload, options) {
|
|
114
124
|
const { paths, scan, cursor, now, deterministicSkipCandidateKeys } = ctx;
|
|
115
125
|
if (upload.durableCandidateKeys.size > 0) {
|
|
116
126
|
await recordBackfillDurableSessionPointers({
|
|
@@ -120,6 +130,8 @@ async function checkpointResolvedBackfillProgress(ctx, upload) {
|
|
|
120
130
|
now,
|
|
121
131
|
});
|
|
122
132
|
}
|
|
133
|
+
if (!options.advanceCursor)
|
|
134
|
+
return;
|
|
123
135
|
const cursorAdvanced = advanceBackfillCursorThroughResolvedPrefix({
|
|
124
136
|
cursor,
|
|
125
137
|
candidates: scan.candidates,
|
|
@@ -20,7 +20,9 @@ import { addReadOnlyGuardIssues, addRepoDiscoveryIssues, backfillScanIssues, cou
|
|
|
20
20
|
import { labelOutsideRootCandidates, outsideRootsWorkspace, } from "./backfill-outside-roots.js";
|
|
21
21
|
import { reasonCountsFor } from "./backfill-reasons.js";
|
|
22
22
|
import { blockedBackfillResult } from "./backfill-result.js";
|
|
23
|
+
import { emptyStubReofferPlan, loadStubReofferPlan, reopenStubbedSessions, } from "./backfill-stub-reoffer.js";
|
|
23
24
|
import { resolveBackfillScope } from "./backfill-scope.js";
|
|
25
|
+
import { stubSessionKey } from "../evidence-stubs-client.js";
|
|
24
26
|
/**
|
|
25
27
|
* SCAN: is this machine paired, what roots/sources/window apply, and what did
|
|
26
28
|
* the archived Codex + Claude session stores actually contain. Returns either
|
|
@@ -47,6 +49,21 @@ export async function scanBackfillRun(command, io, now) {
|
|
|
47
49
|
const scopedCursor = prepareBackfillCursorForScope(storedCursor, collectionRoots, sources);
|
|
48
50
|
const cursor = scopedCursor.cursor;
|
|
49
51
|
const pointerlessTerminalSessions = await loadPointerlessTerminalSessions(paths, sources);
|
|
52
|
+
// BLI-11102: a redaction-crash stub is not a delivery. The server names
|
|
53
|
+
// them; those sessions reopen exactly like pointer-less ones. A dry run
|
|
54
|
+
// keeps its one-request contract (the reachability probe) and does not ask.
|
|
55
|
+
// BLI-11112: the sync's stub pass brings the door's answer with it.
|
|
56
|
+
const stubReoffer = command.stubPass
|
|
57
|
+
? command.stubPass.plan
|
|
58
|
+
: command.dryRun
|
|
59
|
+
? emptyStubReofferPlan("dry_run")
|
|
60
|
+
: await loadStubReofferPlan({
|
|
61
|
+
paths,
|
|
62
|
+
dashboardUrl: scope.dashboardUrl,
|
|
63
|
+
sources,
|
|
64
|
+
fetchImpl: io.fetch,
|
|
65
|
+
});
|
|
66
|
+
reopenStubbedSessions(pointerlessTerminalSessions, stubReoffer);
|
|
50
67
|
const scan = await scanBackfillSessions({
|
|
51
68
|
command,
|
|
52
69
|
homeDir: command.homeDir ?? os.homedir(),
|
|
@@ -73,6 +90,7 @@ export async function scanBackfillRun(command, io, now) {
|
|
|
73
90
|
cursor,
|
|
74
91
|
scan,
|
|
75
92
|
reasonCounts,
|
|
93
|
+
stubReoffer,
|
|
76
94
|
oversizedCandidateKeys: oversizedBackfillCandidateKeys(scan.candidates),
|
|
77
95
|
deterministicSkipCandidateKeys: deterministicSkipCandidateKeys(scan.candidates),
|
|
78
96
|
};
|
|
@@ -117,6 +135,9 @@ async function scanBackfillSessions(options) {
|
|
|
117
135
|
cursor: options.cursor,
|
|
118
136
|
pointerlessTerminalSessions: options.pointerlessTerminalSessions,
|
|
119
137
|
maxFiles: options.command.maxFiles,
|
|
138
|
+
onlySessions: options.command.stubPass
|
|
139
|
+
? new Set(options.command.stubPass.plan.sessions.keys())
|
|
140
|
+
: undefined,
|
|
120
141
|
});
|
|
121
142
|
// BLI-11091: a session outside every approved root is labelled, not
|
|
122
143
|
// refused. It rides the synthetic outside-roots workspace from here on, so
|
|
@@ -184,6 +205,8 @@ function selectBackfillCandidates(options) {
|
|
|
184
205
|
...(options.codexAttribution?.results.map(normalizeCodexCandidate) ?? []),
|
|
185
206
|
...(options.claudeAttribution?.results.map(normalizeClaudeCandidate) ?? []),
|
|
186
207
|
]
|
|
208
|
+
.filter((candidate) => !options.onlySessions ||
|
|
209
|
+
options.onlySessions.has(stubSessionKey(candidate.source, candidate.session_id)))
|
|
187
210
|
.filter((candidate) => isAfterCursor(candidate, options.cursor) ||
|
|
188
211
|
isPointerlessTerminalRetry(candidate, options.pointerlessTerminalSessions))
|
|
189
212
|
.sort((a, b) => compareBackfillCandidatesForRetry(a, b, options.pointerlessTerminalSessions));
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
import { CLAUDE_CURSOR_FILENAME, readRawEvidenceCursor, writeRawEvidenceCursor, } from "../cursors/raw-evidence-cursor.js";
|
|
2
|
+
import { fetchStubbedSessions, stubSessionKey } from "../evidence-stubs-client.js";
|
|
3
|
+
import { describeError } from "../health-detail.js";
|
|
4
|
+
import { readLocalCollectorSessionFile } from "../local-state.js";
|
|
5
|
+
import { candidateCursorKey } from "./backfill-candidates.js";
|
|
6
|
+
export function emptyStubReofferPlan(reason) {
|
|
7
|
+
return { status: "skipped", reason, sessions: new Map(), stub_count: 0, marked_local_sessions: 0 };
|
|
8
|
+
}
|
|
9
|
+
/** Ask the server which of this operator's pointers are stubs. Never throws; writes nothing. */
|
|
10
|
+
export async function loadStubReofferPlan(options) {
|
|
11
|
+
const session = await readLocalCollectorSessionFile(options.paths).catch(() => null);
|
|
12
|
+
if (!session || session.session_state !== "valid" || typeof session.device_token !== "string" || !session.device_token) {
|
|
13
|
+
console.error("[backfill] redaction stubs not asked", JSON.stringify({ reason: "no_device_session" }));
|
|
14
|
+
return emptyStubReofferPlan("no_device_session");
|
|
15
|
+
}
|
|
16
|
+
const answer = await fetchStubbedSessions({
|
|
17
|
+
dashboardUrl: options.dashboardUrl,
|
|
18
|
+
deviceToken: session.device_token,
|
|
19
|
+
fetchImpl: options.fetchImpl,
|
|
20
|
+
});
|
|
21
|
+
const sessions = new Map([...answer.sessions].filter(([key]) => options.sources.some((source) => key.startsWith(`${source}:`))));
|
|
22
|
+
return {
|
|
23
|
+
status: answer.status,
|
|
24
|
+
reason: answer.reason,
|
|
25
|
+
sessions,
|
|
26
|
+
stub_count: answer.unique_stub_count,
|
|
27
|
+
marked_local_sessions: 0,
|
|
28
|
+
};
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* Mark every stubbed session this machine tracks as not delivered. The caller
|
|
32
|
+
* holds the collection lock: sync writes the same cursor files.
|
|
33
|
+
*/
|
|
34
|
+
export async function markStubReoffers(paths, plan, now) {
|
|
35
|
+
if (plan.sessions.size === 0)
|
|
36
|
+
return;
|
|
37
|
+
plan.marked_local_sessions = await markStubbedSessions(paths, plan.sessions, now);
|
|
38
|
+
}
|
|
39
|
+
/** Reopen stubbed sessions exactly like pointer-less ones: selected past the cursor, sorted first. */
|
|
40
|
+
export function reopenStubbedSessions(reopen, plan) {
|
|
41
|
+
for (const key of plan.sessions.keys()) {
|
|
42
|
+
const at = key.indexOf(":");
|
|
43
|
+
const source = key.slice(0, at);
|
|
44
|
+
reopen[source]?.add(key.slice(at + 1));
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
/** Session id -> the stub pointer the new upload claims to replace (the server retires the rest). */
|
|
48
|
+
export function stubClaimsForCandidates(candidates, plan) {
|
|
49
|
+
const claims = new Map();
|
|
50
|
+
for (const candidate of candidates) {
|
|
51
|
+
const pointer = plan.sessions.get(stubSessionKey(candidate.source, candidate.session_id))?.[0];
|
|
52
|
+
if (pointer)
|
|
53
|
+
claims.set(candidate.session_id, pointer);
|
|
54
|
+
}
|
|
55
|
+
return claims;
|
|
56
|
+
}
|
|
57
|
+
export function summarizeStubReoffer(plan, candidates, durableCandidateKeys = new Set(), attempted = candidates) {
|
|
58
|
+
const offered = candidates.filter((candidate) => plan.sessions.has(stubSessionKey(candidate.source, candidate.session_id)));
|
|
59
|
+
const stubsOf = (list) => list.reduce((total, candidate) => total + (plan.sessions.get(stubSessionKey(candidate.source, candidate.session_id))?.length ?? 0), 0);
|
|
60
|
+
const reoffered = offered.length;
|
|
61
|
+
const summary = {
|
|
62
|
+
status: plan.status,
|
|
63
|
+
reason: plan.reason,
|
|
64
|
+
stubs: plan.stub_count,
|
|
65
|
+
sessions: plan.sessions.size,
|
|
66
|
+
marked_local_sessions: plan.marked_local_sessions,
|
|
67
|
+
reoffered_sessions: reoffered,
|
|
68
|
+
reoffered_stubs: stubsOf(offered.filter((candidate) => attempted.includes(candidate))),
|
|
69
|
+
replaced_stubs: stubsOf(offered.filter((candidate) => durableCandidateKeys.has(candidateCursorKey(candidate)))),
|
|
70
|
+
};
|
|
71
|
+
if (plan.sessions.size > 0) {
|
|
72
|
+
console.error("[backfill] redaction-stubbed sessions re-offered", JSON.stringify({
|
|
73
|
+
...summary,
|
|
74
|
+
reason: "stub_reoffered",
|
|
75
|
+
stub_door: `${summary.status}:${summary.reason}`,
|
|
76
|
+
// Stubbed on the server, but no file for it in this run's scan window.
|
|
77
|
+
not_found_locally: plan.sessions.size - reoffered,
|
|
78
|
+
}));
|
|
79
|
+
}
|
|
80
|
+
return summary;
|
|
81
|
+
}
|
|
82
|
+
async function markStubbedSessions(paths, sessions, now) {
|
|
83
|
+
let marked = 0;
|
|
84
|
+
for (const source of ["codex", "claude_code"]) {
|
|
85
|
+
const filename = source === "claude_code" ? CLAUDE_CURSOR_FILENAME : undefined;
|
|
86
|
+
try {
|
|
87
|
+
const cursor = await readRawEvidenceCursor(paths, { filename });
|
|
88
|
+
let changed = false;
|
|
89
|
+
for (const [key, pointerIds] of sessions) {
|
|
90
|
+
if (!key.startsWith(`${source}:`))
|
|
91
|
+
continue;
|
|
92
|
+
const entry = cursor.sessions[key.slice(source.length + 1)];
|
|
93
|
+
if (!entry)
|
|
94
|
+
continue;
|
|
95
|
+
entry.stub_reoffer = { reason: "stub_reoffered", stub_pointer_ids: pointerIds, marked_at: now.toISOString() };
|
|
96
|
+
changed = true;
|
|
97
|
+
marked += 1;
|
|
98
|
+
}
|
|
99
|
+
if (changed) {
|
|
100
|
+
await writeRawEvidenceCursor(paths, cursor, { filename, sessionsOnly: source === "claude_code" });
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
catch (error) {
|
|
104
|
+
// The mark is bookkeeping; the server's answer still drives this run.
|
|
105
|
+
console.error("[backfill] stub mark not written to the cursor", JSON.stringify({ reason: "stub_mark_write_failed", source, ...describeError(error) }));
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
return marked;
|
|
109
|
+
}
|
|
@@ -7,6 +7,7 @@ import { countSessionUploadFailures, deferredEvidenceCount, durableBackfillCandi
|
|
|
7
7
|
import { blockingScanIssues } from "./backfill-issues.js";
|
|
8
8
|
import { ensureOutsideRootsWorkspace, isOutsideRootsWorkspace, } from "./backfill-outside-roots.js";
|
|
9
9
|
import { writeLine } from "./backfill-output.js";
|
|
10
|
+
import { stubClaimsForCandidates } from "./backfill-stub-reoffer.js";
|
|
10
11
|
const BACKFILL_MAX_CONSECUTIVE_FAILURES = 3;
|
|
11
12
|
/**
|
|
12
13
|
* UPLOAD: sync every uploadable candidate in fixed-size batches, heartbeating
|
|
@@ -19,10 +20,11 @@ const BACKFILL_MAX_CONSECUTIVE_FAILURES = 3;
|
|
|
19
20
|
*/
|
|
20
21
|
export async function uploadBackfillBatches(command, io, lock, ctx) {
|
|
21
22
|
const uploadable = uploadableCandidates(ctx.scan.candidates);
|
|
22
|
-
// Doctor needs a boundary after each session
|
|
23
|
-
//
|
|
23
|
+
// Doctor needs a boundary after each session, and so does the scheduled
|
|
24
|
+
// sync's stub pass (BLI-11112), which hands the lock to a waiting backfill
|
|
25
|
+
// at the next one. Ordinary backfill retains its existing batching.
|
|
24
26
|
const plannedBatches = buildBackfillBatches(uploadable);
|
|
25
|
-
const batches = command.deadlineMs !== undefined || command.onProgress
|
|
27
|
+
const batches = command.deadlineMs !== undefined || command.onProgress || command.stubPass
|
|
26
28
|
? plannedBatches.flatMap((batch) => batch.candidates.map((candidate) => ({
|
|
27
29
|
worktree: batch.worktree, candidates: [candidate],
|
|
28
30
|
})))
|
|
@@ -71,6 +73,12 @@ export async function uploadBackfillBatches(command, io, lock, ctx) {
|
|
|
71
73
|
failureReason = "deferred_budget_exhausted";
|
|
72
74
|
break;
|
|
73
75
|
}
|
|
76
|
+
if (command.stubPass?.shouldYield && (await command.stubPass.shouldYield())) {
|
|
77
|
+
blockedAt = stoppedAt("yielded to a waiting backfill", index);
|
|
78
|
+
failureReason = "yielded_to_backfill";
|
|
79
|
+
console.error("[collector stubs] stub pass yielded the collection lock to a waiting backfill", JSON.stringify({ reason: "backfill_waiting", sessions_done: index, sessions_left: batches.length - index }));
|
|
80
|
+
break;
|
|
81
|
+
}
|
|
74
82
|
const outcome = await uploadOneBackfillBatch(command, io, ctx, batch, rawEvidenceBudget);
|
|
75
83
|
foldBatchIntoTally(tally, batch, outcome);
|
|
76
84
|
reportProgress();
|
|
@@ -101,6 +109,7 @@ async function uploadOneBackfillBatch(command, io, ctx, batch, rawEvidenceBudget
|
|
|
101
109
|
codexAttribution: ctx.scan.codexAttribution,
|
|
102
110
|
claudeAttribution: ctx.scan.claudeAttribution,
|
|
103
111
|
rawEvidenceBudget,
|
|
112
|
+
stubClaims: stubClaimsForCandidates(batch.candidates, ctx.stubReoffer),
|
|
104
113
|
fetchImpl: io.fetch,
|
|
105
114
|
});
|
|
106
115
|
const durableInBatch = durableBackfillCandidateKeys(batch, sync);
|
|
@@ -216,6 +225,8 @@ function backfillBatchSyncOptions(options) {
|
|
|
216
225
|
})),
|
|
217
226
|
claudeAttributionScan: options.claudeAttribution ?? undefined,
|
|
218
227
|
rawEvidenceBudget: options.rawEvidenceBudget,
|
|
228
|
+
// BLI-11102: session id -> the redaction stub this upload replaces.
|
|
229
|
+
rawEvidenceStubReoffers: options.stubClaims,
|
|
219
230
|
// `cockpit backfill` is only ever an operator asking — by hand, through
|
|
220
231
|
// onboarding, through doctor, or by running the retry command Cockpit
|
|
221
232
|
// printed. The delivery-backoff window is the scheduler's cadence, so it
|
|
@@ -25,6 +25,7 @@
|
|
|
25
25
|
* backfill-evidence-outcomes.ts what one upload achieved per session
|
|
26
26
|
* backfill-session-report.ts one attribution row per session seen
|
|
27
27
|
* backfill-checkpoint.ts the durable cursor and pointer writes
|
|
28
|
+
* backfill-stub-reoffer.ts redaction-crash stubs: re-offered, not delivered
|
|
28
29
|
* backfill-result.ts the `--json` payload
|
|
29
30
|
* backfill-output.ts every line and prompt a person sees
|
|
30
31
|
*
|
|
@@ -45,6 +46,7 @@ import { planBackfillRun } from "./backfill-plan.js";
|
|
|
45
46
|
import { reportBackfillOutcome } from "./backfill-report.js";
|
|
46
47
|
import { lockHeldResult } from "./backfill-result.js";
|
|
47
48
|
import { scanBackfillRun } from "./backfill-scan.js";
|
|
49
|
+
import { markStubReoffers } from "./backfill-stub-reoffer.js";
|
|
48
50
|
import { uploadBackfillBatches } from "./backfill-upload.js";
|
|
49
51
|
// The command's public surface. Callers (doctor, onboard, local, the CLI
|
|
50
52
|
// router, the BLI-3272 regression test) import these from `./backfill.js`;
|
|
@@ -101,7 +103,14 @@ export async function runBackfill(command, io) {
|
|
|
101
103
|
if (plan.kind === "result")
|
|
102
104
|
return plan.result;
|
|
103
105
|
const { paths } = scanned;
|
|
104
|
-
|
|
106
|
+
// BLI-11112: the sync's stub pass does not take the backfill lock. It runs
|
|
107
|
+
// under the collection lock, which already serialises it against any
|
|
108
|
+
// backfill, and the backfill lock is the flag a WAITING backfill raises: a
|
|
109
|
+
// person's backfill that arrives mid-pass must wait its turn, not bounce off
|
|
110
|
+
// `backfill_already_running`.
|
|
111
|
+
const lock = command.stubPass
|
|
112
|
+
? { acquired: true, handle: { release: async () => undefined, heartbeat: async () => undefined } }
|
|
113
|
+
: await acquireBackfillLock(paths, now);
|
|
105
114
|
if (!lock.acquired) {
|
|
106
115
|
return lockHeldResult(command, scanned, {
|
|
107
116
|
failureReason: "backfill_already_running",
|
|
@@ -110,31 +119,41 @@ export async function runBackfill(command, io) {
|
|
|
110
119
|
}
|
|
111
120
|
// BLI-11091: the scheduled sync holds this lock most of the time an agent
|
|
112
121
|
// is working, so wait for it (bounded) instead of exiting at batch 0. Giving
|
|
113
|
-
// up is `sync_lock_held`, a blocked result and so a non-zero exit.
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
catch (error) {
|
|
119
|
-
await lock.handle.release();
|
|
120
|
-
throw error;
|
|
122
|
+
// up is `sync_lock_held`, a blocked result and so a non-zero exit. The
|
|
123
|
+
// sync's own stub pass (BLI-11112) runs inside the sync's lock and skips it.
|
|
124
|
+
let collectionLock;
|
|
125
|
+
if (command.stubPass) {
|
|
126
|
+
collectionLock = { handle: { release: async () => undefined } };
|
|
121
127
|
}
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
+
else {
|
|
129
|
+
let waited;
|
|
130
|
+
try {
|
|
131
|
+
waited = await acquireSyncLockWithWait(paths, command.syncLockWaitMs ?? BACKFILL_SYNC_LOCK_WAIT_MS);
|
|
132
|
+
}
|
|
133
|
+
catch (error) {
|
|
134
|
+
await lock.handle.release();
|
|
135
|
+
throw error;
|
|
136
|
+
}
|
|
137
|
+
if (!waited.acquired) {
|
|
138
|
+
await lock.handle.release();
|
|
139
|
+
return lockHeldResult(command, scanned, {
|
|
140
|
+
failureReason: "sync_lock_held",
|
|
141
|
+
what: "sync collection lock held",
|
|
142
|
+
});
|
|
143
|
+
}
|
|
144
|
+
collectionLock = waited.lock;
|
|
128
145
|
}
|
|
129
|
-
const collectionLock = waited.lock;
|
|
130
146
|
try {
|
|
131
147
|
// Backfill and scheduled/manual sync share upload spool and raw-evidence
|
|
132
148
|
// cursors. Holding the same collection lock makes their read-modify-write
|
|
133
149
|
// updates serial, while the dedicated backfill lock still prevents two
|
|
134
150
|
// historical scans from running together.
|
|
135
|
-
if (scanned.scopedCursor.reset) {
|
|
151
|
+
if (scanned.scopedCursor.reset && !command.stubPass) {
|
|
136
152
|
await writeBackfillCursor(paths, scanned.cursor);
|
|
137
153
|
}
|
|
154
|
+
// BLI-11102: mark server-named stubs as not delivered, under the lock
|
|
155
|
+
// because sync writes the same cursor files.
|
|
156
|
+
await markStubReoffers(paths, scanned.stubReoffer, now);
|
|
138
157
|
const upload = await uploadBackfillBatches(command, io, lock, scanned);
|
|
139
158
|
return await reportBackfillOutcome(command, io, scanned, upload);
|
|
140
159
|
}
|
|
@@ -21,7 +21,7 @@ export async function runCockpitCli(argv, io) {
|
|
|
21
21
|
}
|
|
22
22
|
|
|
23
23
|
if (command === "--version" || command === "-V" || command === "version") {
|
|
24
|
-
writeLine(io?.stdout ?? process.stdout, "0.2.
|
|
24
|
+
writeLine(io?.stdout ?? process.stdout, "0.2.157");
|
|
25
25
|
return 0;
|
|
26
26
|
}
|
|
27
27
|
|
|
@@ -10,6 +10,7 @@ import { startLocalWorkContext, startLocalWorkContextForAttributedTarget, } from
|
|
|
10
10
|
import { LocalUploadBlockedError, syncLocalAmbientEnvelope, } from "../upload.js";
|
|
11
11
|
import { RAW_EVIDENCE_DEFAULT_BYTE_BUDGET, RAW_EVIDENCE_DEFAULT_OBJECT_BUDGET, } from "../adapters/raw-evidence.js";
|
|
12
12
|
import { liveSyncTargetKey, liveSyncTargetWorktrees, matchesLiveSyncWorktree, } from "./session-sync-attribution.js";
|
|
13
|
+
import { runYieldingAtBoundaries } from "./sync-yield.js";
|
|
13
14
|
const CLAUDE_DAMP_GROWTH_BYTES = 256 * 1024;
|
|
14
15
|
const CLAUDE_DAMP_MAX_AGE_MS = 6 * 60 * 60 * 1000;
|
|
15
16
|
/**
|
|
@@ -33,10 +34,13 @@ export async function syncEveryTargetWorktree(options) {
|
|
|
33
34
|
remainingBytes: RAW_EVIDENCE_DEFAULT_BYTE_BUDGET,
|
|
34
35
|
remainingObjects: RAW_EVIDENCE_DEFAULT_OBJECT_BUDGET,
|
|
35
36
|
};
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
37
|
+
// BLI-11112: a waiting backfill takes the lock at the next worktree
|
|
38
|
+
// boundary; the worktrees not reached stay pending for it or the next tick.
|
|
39
|
+
const loop = await runYieldingAtBoundaries({
|
|
40
|
+
items: syncWorktrees,
|
|
41
|
+
shouldYield: run.yieldCheck,
|
|
42
|
+
describe: (outcome) => ({ status: outcome.sync.status }),
|
|
43
|
+
runOne: (worktree) => syncOneWorktree({
|
|
40
44
|
run,
|
|
41
45
|
now: options.now,
|
|
42
46
|
plan: options.plan,
|
|
@@ -50,16 +54,16 @@ export async function syncEveryTargetWorktree(options) {
|
|
|
50
54
|
recordDampedClaudeSession: (sessionId, priorPointer) => {
|
|
51
55
|
dampedClaudePointers.set(sessionId, priorPointer);
|
|
52
56
|
},
|
|
53
|
-
})
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
}
|
|
57
|
+
}),
|
|
58
|
+
});
|
|
59
|
+
const outcomes = loop.results;
|
|
60
|
+
const everyWorktreeUploaded = outcomes.every((outcome) => outcome.sync.status === "uploaded");
|
|
58
61
|
return {
|
|
59
62
|
outcomes,
|
|
60
63
|
everyWorktreeUploaded,
|
|
61
64
|
claudePriorDurablePointers: claudeDurablePointersFromCursor(options.plan.claudeCursorBefore),
|
|
62
65
|
growthDampedSessionCount: dampedClaudePointers.size,
|
|
66
|
+
yieldedWorktreesLeft: loop.left,
|
|
63
67
|
};
|
|
64
68
|
}
|
|
65
69
|
/** Every Claude session the cursor already holds a durable pointer for. */
|
|
@@ -151,6 +151,7 @@ export async function runAttributedWorktreeSync(options) {
|
|
|
151
151
|
failure_records: health.failure_records,
|
|
152
152
|
notice: health.notice,
|
|
153
153
|
hold: health.hold,
|
|
154
|
+
yielded_worktrees_left: worktreePass.yieldedWorktreesLeft,
|
|
154
155
|
sessions_observed: sessions.length,
|
|
155
156
|
sessions_outside_root: health.sessions_outside_root,
|
|
156
157
|
outside_root_directories: health.outside_root_directories,
|
|
@@ -3,6 +3,8 @@ import { discoverCommandWorktrees } from "./local-discovery.js";
|
|
|
3
3
|
import { runAttributedWorktreeSync } from "./session-sync.js";
|
|
4
4
|
import { reportMultiRepoSync, reportNoWorktreeSync, reportSingleRepoSync, } from "./sync-report.js";
|
|
5
5
|
import { resolveSyncCollectionRoots } from "./sync-roots.js";
|
|
6
|
+
import { healStubsDuringTick } from "./sync-stub-heal.js";
|
|
7
|
+
import { backfillWaitingCheck } from "./sync-yield.js";
|
|
6
8
|
import { getCollectorRuntimePaths } from "../local-state.js";
|
|
7
9
|
import { rawEvidenceDedupSummary, sweepDuplicateStagedRawEvidence, } from "../raw-evidence-gc.js";
|
|
8
10
|
/**
|
|
@@ -18,7 +20,8 @@ export async function runSyncLocked(command, io) {
|
|
|
18
20
|
// holds 559 copies of one rollout needs the disk back before it stages
|
|
19
21
|
// anything else (BLI-3066). Safe by construction — a duplicate is identical
|
|
20
22
|
// by content hash to the survivor.
|
|
21
|
-
const
|
|
23
|
+
const paths = getCollectorRuntimePaths(command.homeDir);
|
|
24
|
+
const dedup = await sweepDuplicateStagedRawEvidence(paths, io.env);
|
|
22
25
|
if (!dedup.skipped && dedup.removed_dirs > 0) {
|
|
23
26
|
writeLine(io.stdout, rawEvidenceDedupSummary(dedup));
|
|
24
27
|
}
|
|
@@ -37,7 +40,11 @@ export async function runSyncLocked(command, io) {
|
|
|
37
40
|
startContexts: false,
|
|
38
41
|
worktrees,
|
|
39
42
|
fetchImpl: io.fetch,
|
|
43
|
+
yieldCheck: backfillWaitingCheck(paths),
|
|
40
44
|
});
|
|
45
|
+
// After the tick's own uploads, still under the lock: re-offer up to 25 of
|
|
46
|
+
// this machine's stubbed sessions. Never fails the tick.
|
|
47
|
+
await healStubsDuringTick(command, io);
|
|
41
48
|
if (discoveryNotice) {
|
|
42
49
|
run.discovery_partial = true;
|
|
43
50
|
run.notice = [discoveryNotice, run.notice].filter(Boolean).join("; ");
|
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
import { runBackfill } from "./backfill.js";
|
|
2
|
+
import { loadStubReofferPlan } from "./backfill-stub-reoffer.js";
|
|
3
|
+
import { backfillWaitingCheck } from "./sync-yield.js";
|
|
4
|
+
import { inspectBackfillLock } from "../backfill-lock.js";
|
|
5
|
+
import { CLAUDE_CURSOR_FILENAME, readRawEvidenceCursor, } from "../cursors/raw-evidence-cursor.js";
|
|
6
|
+
import { describeError } from "../health-detail.js";
|
|
7
|
+
import { DEFAULT_DASHBOARD_URL, getCollectorRuntimePaths, readLocalCollectorConfig, readLocalCollectorSessionFile, } from "../local-state.js";
|
|
8
|
+
/** At most this many stubbed sessions are re-offered per tick (one backfill batch). */
|
|
9
|
+
export const SYNC_STUB_SESSIONS_PER_TICK = 25;
|
|
10
|
+
const DAY_MS = 24 * 60 * 60_000;
|
|
11
|
+
const DEFAULT_DEPS = { runBackfill, loadPlan: loadStubReofferPlan };
|
|
12
|
+
export async function healStubsDuringTick(command, io, deps = DEFAULT_DEPS, now = new Date()) {
|
|
13
|
+
const summary = {
|
|
14
|
+
reason: "unknown",
|
|
15
|
+
stubs_asked: 0,
|
|
16
|
+
stubs_reoffered: 0,
|
|
17
|
+
stubs_remaining: 0,
|
|
18
|
+
sessions_local: 0,
|
|
19
|
+
sessions_selected: 0,
|
|
20
|
+
};
|
|
21
|
+
try {
|
|
22
|
+
summary.reason = await runStubPass(command, io, deps, now, summary);
|
|
23
|
+
}
|
|
24
|
+
catch (error) {
|
|
25
|
+
summary.reason = "stub_pass_threw";
|
|
26
|
+
console.error("[collector stubs] tick stub pass threw; the tick is unaffected", JSON.stringify({ reason: "stub_pass_threw", ...describeError(error) }));
|
|
27
|
+
}
|
|
28
|
+
console.error("[collector stubs] tick", JSON.stringify(summary));
|
|
29
|
+
return summary;
|
|
30
|
+
}
|
|
31
|
+
async function runStubPass(command, io, deps, now, summary) {
|
|
32
|
+
const paths = getCollectorRuntimePaths(command.homeDir);
|
|
33
|
+
// A backfill waiting for the lock heals stubs itself, with the whole door.
|
|
34
|
+
if ((await inspectBackfillLock(paths)).held)
|
|
35
|
+
return "backfill_running";
|
|
36
|
+
const config = await readLocalCollectorConfig(paths).catch(() => null);
|
|
37
|
+
const sessionFile = await readLocalCollectorSessionFile(paths).catch(() => null);
|
|
38
|
+
if (!sessionFile)
|
|
39
|
+
return "no_device_session";
|
|
40
|
+
const sources = config?.collect_claude_jsonl === false ? ["codex"] : ["codex", "claude_code"];
|
|
41
|
+
const dashboardUrl = (sessionFile.dashboard_url ?? config?.dashboard_url ?? DEFAULT_DASHBOARD_URL)
|
|
42
|
+
.trim()
|
|
43
|
+
.replace(/\/+$/, "");
|
|
44
|
+
const plan = await deps.loadPlan({ paths, dashboardUrl, sources, fetchImpl: io.fetch });
|
|
45
|
+
summary.stubs_asked = plan.stub_count;
|
|
46
|
+
summary.stubs_remaining = plan.stub_count;
|
|
47
|
+
if (plan.status === "skipped")
|
|
48
|
+
return plan.reason;
|
|
49
|
+
if (plan.status === "unavailable")
|
|
50
|
+
return `stub_door_unavailable:${plan.reason}`;
|
|
51
|
+
if (plan.sessions.size === 0)
|
|
52
|
+
return "no_stubs";
|
|
53
|
+
const local = await locallyTrackedStubSessions(paths, plan);
|
|
54
|
+
summary.sessions_local = local.length;
|
|
55
|
+
if (local.length === 0)
|
|
56
|
+
return "stubs_not_on_this_machine";
|
|
57
|
+
const selected = local.slice(0, SYNC_STUB_SESSIONS_PER_TICK);
|
|
58
|
+
summary.sessions_selected = selected.length;
|
|
59
|
+
const trimmed = {
|
|
60
|
+
...plan,
|
|
61
|
+
sessions: new Map(selected.map((session) => [session.key, session.pointerIds])),
|
|
62
|
+
stub_count: selected.reduce((total, session) => total + session.pointerIds.length, 0),
|
|
63
|
+
marked_local_sessions: 0,
|
|
64
|
+
};
|
|
65
|
+
const window = stubWindow(selected, sessionFile.paired_at, now);
|
|
66
|
+
const result = await deps.runBackfill({
|
|
67
|
+
homeDir: command.homeDir,
|
|
68
|
+
repoRoot: command.repoRoot,
|
|
69
|
+
maxDepth: command.maxDepth,
|
|
70
|
+
maxRepos: command.maxRepos,
|
|
71
|
+
...window,
|
|
72
|
+
dryRun: false,
|
|
73
|
+
yes: true,
|
|
74
|
+
json: true,
|
|
75
|
+
stubPass: { plan: trimmed, shouldYield: backfillWaitingCheck(paths) },
|
|
76
|
+
}, backfillIoFor(io));
|
|
77
|
+
const outcome = result.stub_reoffer;
|
|
78
|
+
summary.stubs_reoffered = outcome?.reoffered_stubs ?? 0;
|
|
79
|
+
summary.stubs_remaining = Math.max(0, plan.stub_count - (outcome?.replaced_stubs ?? 0));
|
|
80
|
+
return stubPassReason(result);
|
|
81
|
+
}
|
|
82
|
+
/** Door-order list of the stubbed sessions whose cursor entry lives on this machine. */
|
|
83
|
+
async function locallyTrackedStubSessions(paths, plan) {
|
|
84
|
+
const cursors = {
|
|
85
|
+
codex: await readRawEvidenceCursor(paths).catch(() => null),
|
|
86
|
+
claude_code: await readRawEvidenceCursor(paths, { filename: CLAUDE_CURSOR_FILENAME }).catch(() => null),
|
|
87
|
+
};
|
|
88
|
+
const local = [];
|
|
89
|
+
for (const [key, pointerIds] of plan.sessions) {
|
|
90
|
+
const at = key.indexOf(":");
|
|
91
|
+
const source = key.slice(0, at);
|
|
92
|
+
const entry = cursors[source]?.sessions[key.slice(at + 1)];
|
|
93
|
+
if (entry)
|
|
94
|
+
local.push({ key, pointerIds, mtimeMs: entry.file_mtime_ms });
|
|
95
|
+
}
|
|
96
|
+
return local;
|
|
97
|
+
}
|
|
98
|
+
/**
|
|
99
|
+
* The narrowest backfill window that still reaches every selected session:
|
|
100
|
+
* back to the oldest one's file time plus a day. A backfill window never
|
|
101
|
+
* reaches before pairing, so an older session (or one with no recorded time)
|
|
102
|
+
* asks for all history instead.
|
|
103
|
+
*/
|
|
104
|
+
function stubWindow(selected, pairedAt, now) {
|
|
105
|
+
const times = selected.map((session) => session.mtimeMs);
|
|
106
|
+
const oldest = Math.min(...times);
|
|
107
|
+
const paired = pairedAt ? Date.parse(pairedAt) : Number.NaN;
|
|
108
|
+
if (times.some((time) => !Number.isFinite(time) || time <= 0))
|
|
109
|
+
return { all: true };
|
|
110
|
+
if (!Number.isFinite(paired) || oldest < paired)
|
|
111
|
+
return { all: true };
|
|
112
|
+
return { all: false, sinceDays: Math.max(1, Math.ceil((now.getTime() - oldest) / DAY_MS) + 1) };
|
|
113
|
+
}
|
|
114
|
+
function stubPassReason(result) {
|
|
115
|
+
const outcome = result.stub_reoffer;
|
|
116
|
+
if (result.failure_reason === "yielded_to_backfill")
|
|
117
|
+
return "yielded_to_backfill";
|
|
118
|
+
if (result.status === "blocked")
|
|
119
|
+
return `stub_backfill_blocked:${result.failure_reason ?? "unknown"}`;
|
|
120
|
+
if (!outcome || outcome.reoffered_sessions === 0)
|
|
121
|
+
return "stubs_not_found_locally";
|
|
122
|
+
if (outcome.replaced_stubs < outcome.reoffered_stubs) {
|
|
123
|
+
return `stubs_partly_replaced:${result.failure_reason ?? "unknown"}`;
|
|
124
|
+
}
|
|
125
|
+
return "stubs_reoffered";
|
|
126
|
+
}
|
|
127
|
+
/**
|
|
128
|
+
* The backfill writes its `--json` result and progress to stdout; the sync's
|
|
129
|
+
* stdout is its own `--json` receipt, so those lines go nowhere and the
|
|
130
|
+
* backfill's own `[backfill]` logs still reach stderr.
|
|
131
|
+
*/
|
|
132
|
+
function backfillIoFor(io) {
|
|
133
|
+
const discard = { write: () => true };
|
|
134
|
+
return { stdin: io.stdin, stdout: discard, stderr: io.stderr, fetch: io.fetch };
|
|
135
|
+
}
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* BLI-11112: the scheduled sync hands the collection lock to a waiting
|
|
3
|
+
* backfill at its next worktree boundary.
|
|
4
|
+
*
|
|
5
|
+
* A tick on a machine with a backlog now runs 10 to 15 minutes, longer than
|
|
6
|
+
* the backfill's old wait, so a person who typed `cockpit backfill` sat out the
|
|
7
|
+
* whole tick and got `sync_lock_held`. The rule is plain: a backfill that wants
|
|
8
|
+
* the lock takes the backfill lock FIRST and then waits for the collection
|
|
9
|
+
* lock (`backfill-lock-wait.ts`). The backfill lock being held is the waiter's
|
|
10
|
+
* flag. Before each worktree upload the sync asks whether that flag is up; if
|
|
11
|
+
* it is, it finishes nothing new, writes down what it already did, and
|
|
12
|
+
* releases the lock. The worktrees it did not reach stay pending in the cursor
|
|
13
|
+
* and are exactly what the backfill (or the next tick) picks up. A worktree
|
|
14
|
+
* already uploading is never interrupted, so the wait is bounded by one
|
|
15
|
+
* worktree's upload, which is why each one logs its own elapsed time.
|
|
16
|
+
*/
|
|
17
|
+
import { inspectBackfillLock } from "../backfill-lock.js";
|
|
18
|
+
/** The waiter's flag: the backfill lock is held while its owner waits for ours. */
|
|
19
|
+
export function backfillWaitingCheck(paths, inspect = inspectBackfillLock) {
|
|
20
|
+
return async () => {
|
|
21
|
+
try {
|
|
22
|
+
return (await inspect(paths)).held;
|
|
23
|
+
}
|
|
24
|
+
catch {
|
|
25
|
+
// A lock probe that cannot read the file means nobody is waiting; the
|
|
26
|
+
// sync carries on exactly as it did before this rule existed.
|
|
27
|
+
return false;
|
|
28
|
+
}
|
|
29
|
+
};
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* Run `runOne` over `items` in order, asking `shouldYield` before each one.
|
|
33
|
+
* The first `true` stops the loop; nothing already started is interrupted.
|
|
34
|
+
*/
|
|
35
|
+
export async function runYieldingAtBoundaries(options) {
|
|
36
|
+
const now = options.now ?? (() => Date.now());
|
|
37
|
+
const results = [];
|
|
38
|
+
const started = now();
|
|
39
|
+
for (const [index, item] of options.items.entries()) {
|
|
40
|
+
if (options.shouldYield && (await options.shouldYield())) {
|
|
41
|
+
const left = options.items.length - index;
|
|
42
|
+
console.error("[session-sync] yielded the collection lock to a waiting backfill", JSON.stringify({
|
|
43
|
+
reason: "backfill_waiting",
|
|
44
|
+
worktrees_done: index,
|
|
45
|
+
worktrees_left: left,
|
|
46
|
+
elapsed_ms: now() - started,
|
|
47
|
+
}));
|
|
48
|
+
return { results, yielded: true, left };
|
|
49
|
+
}
|
|
50
|
+
const itemStarted = now();
|
|
51
|
+
const result = await options.runOne(item);
|
|
52
|
+
results.push(result);
|
|
53
|
+
console.error("[session-sync] worktree synced", JSON.stringify({
|
|
54
|
+
index: index + 1,
|
|
55
|
+
total: options.items.length,
|
|
56
|
+
elapsed_ms: now() - itemStarted,
|
|
57
|
+
...(options.describe ? options.describe(result) : {}),
|
|
58
|
+
}));
|
|
59
|
+
}
|
|
60
|
+
return { results, yielded: false, left: 0 };
|
|
61
|
+
}
|
|
@@ -195,8 +195,21 @@ function parseSessionEntry(value) {
|
|
|
195
195
|
uploaded_at: optionalString(record["uploaded_at"]),
|
|
196
196
|
uploaded_byte_size: optionalNumberOrNull(record["uploaded_byte_size"]),
|
|
197
197
|
last_seen_at: lastSeenAt,
|
|
198
|
+
...parseStubReoffer(record["stub_reoffer"]),
|
|
198
199
|
};
|
|
199
200
|
}
|
|
201
|
+
function parseStubReoffer(value) {
|
|
202
|
+
if (!value || typeof value !== "object")
|
|
203
|
+
return {};
|
|
204
|
+
const record = value;
|
|
205
|
+
const markedAt = optionalString(record["marked_at"]);
|
|
206
|
+
const ids = Array.isArray(record["stub_pointer_ids"])
|
|
207
|
+
? record["stub_pointer_ids"].filter((id) => typeof id === "string" && id.length > 0)
|
|
208
|
+
: [];
|
|
209
|
+
if (record["reason"] !== "stub_reoffered" || !markedAt || ids.length === 0)
|
|
210
|
+
return {};
|
|
211
|
+
return { stub_reoffer: { reason: "stub_reoffered", stub_pointer_ids: ids, marked_at: markedAt } };
|
|
212
|
+
}
|
|
200
213
|
function parseCheckpoints(value) {
|
|
201
214
|
if (!value || typeof value !== "object")
|
|
202
215
|
return null;
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Asks `POST /api/ambient/evidence/stubs` which of this operator's stored
|
|
3
|
+
* transcripts are only the redaction-crash stub, and groups the answer by agent
|
|
4
|
+
* session (BLI-11102).
|
|
5
|
+
*
|
|
6
|
+
* A stub is a promise to come back, not a delivery: the local cursor recorded
|
|
7
|
+
* an upload, so nothing local would ever offer that session again. The server
|
|
8
|
+
* holds the label, so the server is asked. `cockpit backfill` re-offers every
|
|
9
|
+
* session this returns.
|
|
10
|
+
*
|
|
11
|
+
* Never throws. A door that cannot be asked (an older dashboard answers 404, a
|
|
12
|
+
* network error, a malformed page) returns `unavailable` with the reason, and
|
|
13
|
+
* the backfill runs exactly as it did before. That is also the release gate:
|
|
14
|
+
* the collector only ever sends a `stub_replaced` claim for a stub this door
|
|
15
|
+
* named, so it cannot send one to a server that does not know the rule.
|
|
16
|
+
*
|
|
17
|
+
* Bounded: pages of up to 500, at most `maxPages` of them, walked by the
|
|
18
|
+
* server's keyset cursor. The result says whether the list was exhausted,
|
|
19
|
+
* capped by `maxPages`, or cut short by a failed page after some landed.
|
|
20
|
+
*/
|
|
21
|
+
import { EVIDENCE_STUBS_MAX_PAGE, EVIDENCE_STUBS_SCHEMA_VERSION, EvidenceStubsResponseSchema, transcriptSessionFromObjectKey, } from "@bli-cockpit/telemetry-core";
|
|
22
|
+
import { describeError } from "./health-detail.js";
|
|
23
|
+
const STUBS_PAGE_TIMEOUT_MS = 15_000;
|
|
24
|
+
export const STUBS_DEFAULT_MAX_PAGES = 20;
|
|
25
|
+
export function stubSessionKey(source, sessionId) {
|
|
26
|
+
return `${source}:${sessionId}`;
|
|
27
|
+
}
|
|
28
|
+
export async function fetchStubbedSessions(options) {
|
|
29
|
+
const maxPages = options.maxPages ?? STUBS_DEFAULT_MAX_PAGES;
|
|
30
|
+
const sessions = new Map();
|
|
31
|
+
const seen = new Set();
|
|
32
|
+
let stubCount = 0;
|
|
33
|
+
let unkeyed = 0;
|
|
34
|
+
let pages = 0;
|
|
35
|
+
let after = null;
|
|
36
|
+
for (;;) {
|
|
37
|
+
if (pages >= maxPages) {
|
|
38
|
+
return finish("capped", "ok");
|
|
39
|
+
}
|
|
40
|
+
const page = await requestPage({ ...options, after });
|
|
41
|
+
if (!page.ok) {
|
|
42
|
+
return finish(pages === 0 ? "unavailable" : "failed_partial", page.reason);
|
|
43
|
+
}
|
|
44
|
+
pages += 1;
|
|
45
|
+
for (const stub of page.stubs) {
|
|
46
|
+
stubCount += 1;
|
|
47
|
+
if (seen.has(stub.ref_id))
|
|
48
|
+
continue;
|
|
49
|
+
seen.add(stub.ref_id);
|
|
50
|
+
const session = transcriptSessionFromObjectKey(stub.object_key);
|
|
51
|
+
if (!session) {
|
|
52
|
+
unkeyed += 1;
|
|
53
|
+
continue;
|
|
54
|
+
}
|
|
55
|
+
const key = stubSessionKey(session.source, session.session_id);
|
|
56
|
+
sessions.set(key, [...(sessions.get(key) ?? []), stub.raw_evidence_pointer_id]);
|
|
57
|
+
}
|
|
58
|
+
// Advance by what the server returned; a page that names no cursor is the last.
|
|
59
|
+
if (!page.nextAfter)
|
|
60
|
+
return finish("exhausted", "ok");
|
|
61
|
+
after = page.nextAfter;
|
|
62
|
+
}
|
|
63
|
+
function finish(status, reason) {
|
|
64
|
+
const result = {
|
|
65
|
+
status,
|
|
66
|
+
reason,
|
|
67
|
+
pages,
|
|
68
|
+
stub_count: stubCount,
|
|
69
|
+
unique_stub_count: seen.size,
|
|
70
|
+
unkeyed_stub_count: unkeyed,
|
|
71
|
+
sessions,
|
|
72
|
+
};
|
|
73
|
+
console.error("[collector stubs] redaction-crash stubs asked", JSON.stringify({
|
|
74
|
+
reason: status === "unavailable" ? `stub_door_unavailable:${reason}` : `stubs_${status}`,
|
|
75
|
+
pages,
|
|
76
|
+
stub_count: stubCount,
|
|
77
|
+
unique_stub_count: seen.size,
|
|
78
|
+
unkeyed_stub_count: unkeyed,
|
|
79
|
+
session_count: sessions.size,
|
|
80
|
+
...(status === "failed_partial" ? { failure: reason } : {}),
|
|
81
|
+
}));
|
|
82
|
+
return result;
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
async function requestPage(options) {
|
|
86
|
+
const controller = new AbortController();
|
|
87
|
+
const timeout = setTimeout(() => controller.abort(), STUBS_PAGE_TIMEOUT_MS);
|
|
88
|
+
try {
|
|
89
|
+
const response = await options.fetchImpl(`${options.dashboardUrl}/api/ambient/evidence/stubs`, {
|
|
90
|
+
method: "POST",
|
|
91
|
+
headers: {
|
|
92
|
+
"Content-Type": "application/json",
|
|
93
|
+
Authorization: `Bearer ${options.deviceToken}`,
|
|
94
|
+
},
|
|
95
|
+
body: JSON.stringify({
|
|
96
|
+
schema_version: EVIDENCE_STUBS_SCHEMA_VERSION,
|
|
97
|
+
after: options.after,
|
|
98
|
+
limit: EVIDENCE_STUBS_MAX_PAGE,
|
|
99
|
+
}),
|
|
100
|
+
signal: controller.signal,
|
|
101
|
+
});
|
|
102
|
+
if (!response.ok)
|
|
103
|
+
return { ok: false, reason: `http_${response.status}` };
|
|
104
|
+
const parsed = EvidenceStubsResponseSchema.safeParse(await response.json().catch(() => null));
|
|
105
|
+
if (!parsed.success)
|
|
106
|
+
return { ok: false, reason: "invalid_response" };
|
|
107
|
+
return { ok: true, stubs: parsed.data.stubs, nextAfter: parsed.data.next_after };
|
|
108
|
+
}
|
|
109
|
+
catch (error) {
|
|
110
|
+
return { ok: false, reason: describeError(error).error_name };
|
|
111
|
+
}
|
|
112
|
+
finally {
|
|
113
|
+
clearTimeout(timeout);
|
|
114
|
+
}
|
|
115
|
+
}
|
|
@@ -125,6 +125,7 @@ export async function buildLocalAmbientEnvelope(options = {}) {
|
|
|
125
125
|
operatorId: session.operator_id,
|
|
126
126
|
workContextId: uploadContext.work_context_id,
|
|
127
127
|
})),
|
|
128
|
+
rawEvidenceStubReoffers: options.rawEvidenceStubReoffers,
|
|
128
129
|
rawEvidenceByteBudget: options.rawEvidenceByteBudget,
|
|
129
130
|
rawEvidenceObjectBudget: options.rawEvidenceObjectBudget,
|
|
130
131
|
rawEvidenceBudget: options.rawEvidenceBudget,
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@bli-cockpit/cli",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.157",
|
|
4
4
|
"private": false,
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
@@ -27,8 +27,8 @@
|
|
|
27
27
|
"test": "node dist/cli.js --help && node ../../scripts/assert-public-cli-routing.mjs && node ../../scripts/assert-public-cli-verb-help.mjs && node ../../scripts/assert-public-cli-surface-doors.mjs && node ../../scripts/assert-public-cli-runtime-files.mjs && node ../../scripts/assert-public-cli-exit-contract.mjs && node ../../scripts/assert-public-cli-no-fleet-posts.mjs && node ../../scripts/assert-public-cli-doctor-json.mjs && node ../../scripts/assert-public-package-pack.mjs --workspace=@bli-cockpit/cli"
|
|
28
28
|
},
|
|
29
29
|
"dependencies": {
|
|
30
|
-
"@bli-cockpit/memory-mcp": "0.1.
|
|
31
|
-
"@bli-cockpit/mcp": "0.1.
|
|
32
|
-
"@bli-cockpit/telemetry-core": "0.1.
|
|
30
|
+
"@bli-cockpit/memory-mcp": "0.1.42",
|
|
31
|
+
"@bli-cockpit/mcp": "0.1.76",
|
|
32
|
+
"@bli-cockpit/telemetry-core": "0.1.54"
|
|
33
33
|
}
|
|
34
34
|
}
|