@bli-cockpit/cli 0.2.208 → 0.2.211
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/raw-evidence-sanitize.js +12 -2
- package/dist/commands/agent-session-report.js +1 -1
- package/dist/commands/doctor-access.js +7 -5
- package/dist/commands/install-receipts.js +27 -16
- package/dist/commands/local-help-commands.js +1 -1
- package/dist/commands/public-root.js +1 -1
- package/dist/commands/scout-render.js +2 -0
- package/dist/commands/session-sync-attribution.js +11 -3
- package/dist/commands/session-sync-health.js +1 -1
- package/dist/commands/session-sync-outside-roots.js +281 -0
- package/dist/commands/session-sync-plan.js +5 -2
- package/dist/commands/session-sync-scan.js +9 -4
- package/dist/commands/session-sync-upload.js +44 -25
- package/dist/commands/session-sync.js +29 -7
- package/dist/commands/settings-render.js +4 -0
- package/dist/commands/sources-check.js +15 -4
- package/dist/commands/sources-verdict.js +125 -14
- package/dist/commands/sync-report.js +1 -1
- package/dist/commands/sync.js +8 -1
- package/dist/image-description-session.js +7 -1
- package/dist/image-description-state.js +2 -2
- package/dist/image-mode-state.js +154 -0
- package/dist/local-state-files.js +20 -0
- package/dist/local-state-work-context.js +3 -1
- package/package.json +4 -4
|
@@ -1,6 +1,16 @@
|
|
|
1
1
|
import { containsSecretLikeContent, redactSecretLikeContent, scannedCleanRedactionMetadata, } from "@bli-cockpit/telemetry-core";
|
|
2
2
|
import { sha256 } from "./raw-evidence-keys.js";
|
|
3
3
|
import { describeError } from "../health-detail.js";
|
|
4
|
+
/**
|
|
5
|
+
* The one line a whole file becomes when masking its secret-bearing lines still leaves something
|
|
6
|
+
* secret-looking. It carries none of the file's content, so a reader that compares sizes or hashes
|
|
7
|
+
* against it (`cockpit sources check`) must treat it as "no redacted form", never as the file.
|
|
8
|
+
*/
|
|
9
|
+
export const WHOLE_FILE_MASK_LINE = "[REDACTED_LINE:secret_redaction_failed]\n";
|
|
10
|
+
/** True when these upload bytes are the whole-file mask, not a redacted copy of the content. */
|
|
11
|
+
export function isWholeFileMask(bytes) {
|
|
12
|
+
return Buffer.from(bytes).toString("utf8") === WHOLE_FILE_MASK_LINE;
|
|
13
|
+
}
|
|
4
14
|
/** BLI-3238: a crashed redactor has to say so; see the catch below. */
|
|
5
15
|
export function sanitizeTextEvidenceForUpload(options) {
|
|
6
16
|
const originalBytes = options.originalBytes ?? Buffer.from(options.text, "utf8");
|
|
@@ -75,7 +85,7 @@ function maskFileWithSecretLikeName(input) {
|
|
|
75
85
|
? input.redactionResult.text
|
|
76
86
|
: input.text;
|
|
77
87
|
const safeText = containsSecretLikeContent(sanitizedText)
|
|
78
|
-
?
|
|
88
|
+
? WHOLE_FILE_MASK_LINE
|
|
79
89
|
: sanitizedText;
|
|
80
90
|
const sanitizedBytes = Buffer.from(safeText, "utf8");
|
|
81
91
|
return {
|
|
@@ -101,7 +111,7 @@ function maskSecretBearingLinesOnly(input) {
|
|
|
101
111
|
});
|
|
102
112
|
let maskedText = maskSecretBearingLines(input.text, redaction);
|
|
103
113
|
if (containsSecretLikeContent(maskedText)) {
|
|
104
|
-
maskedText =
|
|
114
|
+
maskedText = WHOLE_FILE_MASK_LINE;
|
|
105
115
|
}
|
|
106
116
|
const sanitizedBytes = Buffer.from(maskedText, "utf8");
|
|
107
117
|
return {
|
|
@@ -246,7 +246,7 @@ export function buildAgentSessionReport(options) {
|
|
|
246
246
|
*/
|
|
247
247
|
export function buildAgentSessionSummary(options) {
|
|
248
248
|
const sidecarOutcomes = options.outcomes.flatMap((outcome) => outcome.sync.raw_evidence_outcomes.filter((upload) => upload.kind === "claude_jsonl_sidecar"));
|
|
249
|
-
const attributedClaude = options.claudeAttribution.results.filter((result) => isLiveRawEvidenceSyncAttribution(result.state, result.reason));
|
|
249
|
+
const attributedClaude = options.claudeAttribution.results.filter((result) => isLiveRawEvidenceSyncAttribution(result.state, result.reason, result.worktree !== null));
|
|
250
250
|
const sidecarsCollected = attributedClaude.reduce((total, result) => total +
|
|
251
251
|
result.sidecar_files.filter((sidecar) => !sidecar.skipped_reason).length, 0);
|
|
252
252
|
const sidecarsSkipped = options.claudeAttribution.results.reduce((total, result) => total +
|
|
@@ -160,10 +160,12 @@ function shellQuote(value) {
|
|
|
160
160
|
* a gate on it, so a session sitting outside one is not a defect doctor found
|
|
161
161
|
* and cannot repair. This row is therefore always informational (`ok`): it
|
|
162
162
|
* reports how many of this scan's sessions were inside vs. outside, never
|
|
163
|
-
* asks for approval, and attempts no fix.
|
|
164
|
-
*
|
|
165
|
-
*
|
|
166
|
-
*
|
|
163
|
+
* asks for approval, and attempts no fix. Since BLI-17916 the scheduled sync
|
|
164
|
+
* uploads the redacted transcript of an outside-root session too (under the
|
|
165
|
+
* collector's outside-roots workspace), so widening a root buys only
|
|
166
|
+
* attribution to a repo; the message names `cockpit roots add` as an OPTIONAL
|
|
167
|
+
* step an operator can choose, never something doctor, or any person, is
|
|
168
|
+
* waiting on to approve. The
|
|
167
169
|
* row only turns red the ordinary way every other doctor check does: the scan
|
|
168
170
|
* itself throwing (`work_directory_scan_incomplete` from unreadable session
|
|
169
171
|
* folders), which `runDoctorWithDeps`'s existing `check_threw` handling
|
|
@@ -180,7 +182,7 @@ export async function readRootsCoverWork(context) {
|
|
|
180
182
|
if (!outsideCount)
|
|
181
183
|
return ok("roots-cover-work", "work_covered", `${scan.observed} sessions seen; no working folders outside your saved roots. ${sessionCoverageLine(scan, outsideCount)}`);
|
|
182
184
|
return {
|
|
183
|
-
...ok("roots-cover-work", "work_outside_roots_labelled", `${insideCount} of ${scan.observed} sessions inside your saved roots, ${outsideCount} outside — every one already reached the server, the outside ones carrying an outside label. Nothing needs approving. Optional: add a folder as a root to
|
|
185
|
+
...ok("roots-cover-work", "work_outside_roots_labelled", `${insideCount} of ${scan.observed} sessions inside your saved roots, ${outsideCount} outside — every one already reached the server, the outside ones carrying an outside label, and their redacted transcripts upload too, a few per sync, newest first. Nothing needs approving. Optional: add a folder as a root to attribute its sessions to a repo (scan ${scan.generated_at}):\n${outside.map(entry => `${entry.count}: ${entry.directory}\n ${rootsAddCommand(entry.directory, process.platform, homeDir)}`).join("\n")}\n\n${sessionCoverageLine(scan, outsideCount)}`),
|
|
184
186
|
nextAction: rootsAddCommand(outside[0].directory, process.platform, homeDir),
|
|
185
187
|
};
|
|
186
188
|
}
|
|
@@ -136,11 +136,15 @@ async function deliverPendingInstallEvents(options, paths, deviceToken) {
|
|
|
136
136
|
let delivered = 0;
|
|
137
137
|
let observedMinCliVersion = null;
|
|
138
138
|
for (let offset = 0; offset < pending.length; offset += 5) {
|
|
139
|
-
await Promise.all(pending.slice(offset, offset + 5).map((entry) => deliverInstallEventEntry(options.io, paths, entry, deviceToken, failures).then((
|
|
140
|
-
if (
|
|
141
|
-
|
|
142
|
-
if (floor
|
|
143
|
-
|
|
139
|
+
await Promise.all(pending.slice(offset, offset + 5).map((entry) => deliverInstallEventEntry(options.io, paths, entry, deviceToken, failures).then((receipt) => {
|
|
140
|
+
if (receipt === undefined)
|
|
141
|
+
return;
|
|
142
|
+
if (receipt.floor != null)
|
|
143
|
+
observedMinCliVersion = receipt.floor;
|
|
144
|
+
if (receipt.imageHandlingMode !== undefined && options.observed) {
|
|
145
|
+
options.observed.imageHandlingMode = receipt.imageHandlingMode;
|
|
146
|
+
}
|
|
147
|
+
delivered += 1;
|
|
144
148
|
})));
|
|
145
149
|
}
|
|
146
150
|
if (delivered > 0) {
|
|
@@ -165,9 +169,9 @@ async function deliverInstallEventEntry(io, paths, entry, deviceToken, failures)
|
|
|
165
169
|
const timeout = setTimeout(() => controller.abort(), 5_000);
|
|
166
170
|
try {
|
|
167
171
|
const response = await postInstallEventEntry(io, entry, deviceToken, controller.signal);
|
|
168
|
-
const
|
|
172
|
+
const receipt = await readReceiptReply(response);
|
|
169
173
|
await removeInstallEventEntry(paths, entry.outbox_id);
|
|
170
|
-
return
|
|
174
|
+
return receipt;
|
|
171
175
|
}
|
|
172
176
|
catch (error) {
|
|
173
177
|
await keepFailedInstallEventEntry(paths, entry, error, failures);
|
|
@@ -193,18 +197,25 @@ async function postInstallEventEntry(io, entry, deviceToken, signal) {
|
|
|
193
197
|
throw new Error(`http_${response.status}`);
|
|
194
198
|
return response;
|
|
195
199
|
}
|
|
196
|
-
|
|
200
|
+
/**
|
|
201
|
+
* The reply carries two things the tick listens for: the server-published `min_cli_version` floor
|
|
202
|
+
* (BLI-2678) and this machine's `image_handling_mode` (BLI-17273; undefined when the reply names
|
|
203
|
+
* none, validated by the caller and never here).
|
|
204
|
+
*/
|
|
205
|
+
async function readReceiptReply(response) {
|
|
197
206
|
const receipt = (await response.json().catch((error) => {
|
|
198
|
-
//
|
|
199
|
-
// (
|
|
200
|
-
//
|
|
201
|
-
|
|
202
|
-
console.error("[install-receipts] receipt body unreadable; no min_cli_version observed", JSON.stringify({ reason: "receipt_body_unreadable", http_status: response.status, ...describeError(error) }));
|
|
207
|
+
// A body that will not parse means the floor and the image mode are not observed on this tick
|
|
208
|
+
// (the forced-update path silently does nothing, the machine keeps its cached mode) while the
|
|
209
|
+
// 2xx above says the receipt landed fine.
|
|
210
|
+
console.error("[install-receipts] receipt body unreadable; no min_cli_version or image_handling_mode observed", JSON.stringify({ reason: "receipt_body_unreadable", http_status: response.status, ...describeError(error) }));
|
|
203
211
|
return null;
|
|
204
212
|
}));
|
|
205
|
-
return
|
|
206
|
-
|
|
207
|
-
|
|
213
|
+
return {
|
|
214
|
+
floor: typeof receipt?.min_cli_version === "string" && receipt.min_cli_version.trim()
|
|
215
|
+
? receipt.min_cli_version.trim()
|
|
216
|
+
: null,
|
|
217
|
+
imageHandlingMode: receipt?.image_handling_mode,
|
|
218
|
+
};
|
|
208
219
|
}
|
|
209
220
|
async function keepFailedInstallEventEntry(paths, entry, error, failures) {
|
|
210
221
|
const failureReason = classifyInstallTelemetryError(error);
|
|
@@ -93,7 +93,7 @@ export function localSubcommandHelp(command) {
|
|
|
93
93
|
"",
|
|
94
94
|
"Diagnose, repair, and verify this machine.",
|
|
95
95
|
"Repairs CLI, background sync, sign-in, roots, memory and hooks, agent rules, catch-up, uploads, and cleanup.",
|
|
96
|
-
"roots-cover-work is informational: it names how many sessions are inside vs. outside saved roots (all already uploaded, outside ones labelled) and gives cockpit roots add commands for
|
|
96
|
+
"roots-cover-work is informational: it names how many sessions are inside vs. outside saved roots (all already uploaded, outside ones labelled) and gives cockpit roots add commands for attributing a folder's sessions to a repo (outside-root transcripts upload either way). It never needs a fix.",
|
|
97
97
|
"--no-usage-telemetry switches off Claude Code and Codex usage export, including later doctor runs.",
|
|
98
98
|
"--check or --no-repair diagnoses without repairs. --dry-run previews repairs.",
|
|
99
99
|
"--backfill-budget defaults to 900 seconds; catch-up repeats chunks until complete, with progress every 30 seconds.",
|
|
@@ -21,7 +21,7 @@ export async function runCockpitCli(argv, io) {
|
|
|
21
21
|
}
|
|
22
22
|
|
|
23
23
|
if (command === "--version" || command === "-V" || command === "version") {
|
|
24
|
-
writeLine(io?.stdout ?? process.stdout, "0.2.
|
|
24
|
+
writeLine(io?.stdout ?? process.stdout, "0.2.211");
|
|
25
25
|
return 0;
|
|
26
26
|
}
|
|
27
27
|
|
|
@@ -81,6 +81,8 @@ function renderExperiment(card, styled) {
|
|
|
81
81
|
for (const row of card.crossref ?? []) {
|
|
82
82
|
if (row.person) {
|
|
83
83
|
out.push(dim(` On your team · ${row.person} · ${row.observedPattern ?? ""}`, styled));
|
|
84
|
+
if (row.provenanceLine)
|
|
85
|
+
out.push(dim(` ${row.provenanceLine}`, styled));
|
|
84
86
|
}
|
|
85
87
|
}
|
|
86
88
|
if (card.expectedPayoff)
|
|
@@ -11,6 +11,7 @@
|
|
|
11
11
|
* re-derived anywhere else.
|
|
12
12
|
*/
|
|
13
13
|
import { isLiveRawEvidenceSyncAttribution } from "../raw-evidence-attribution-policy.js";
|
|
14
|
+
import { outsideRootEntryOwesUpload } from "./session-sync-outside-roots.js";
|
|
14
15
|
/**
|
|
15
16
|
* Live sync remains reason-allowlisted even though historical backfill accepts
|
|
16
17
|
* every deterministic fallback state.
|
|
@@ -18,9 +19,16 @@ import { isLiveRawEvidenceSyncAttribution } from "../raw-evidence-attribution-po
|
|
|
18
19
|
export function isLiveSyncCollectableAttributionState(state, reason, hasApprovedWorkspace = false) {
|
|
19
20
|
return isLiveRawEvidenceSyncAttribution(state, reason, hasApprovedWorkspace);
|
|
20
21
|
}
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
22
|
+
/**
|
|
23
|
+
* Does this cursor entry still owe a transcript upload, so the next scan window
|
|
24
|
+
* must reach back to it? An outside-root session owes one too (BLI-17916): the
|
|
25
|
+
* scheduled sync uploads it now, a few per tick, and the window stays open
|
|
26
|
+
* until the last of them is durable.
|
|
27
|
+
*/
|
|
28
|
+
export function liveSyncCursorEntryRequiresRetry(entry, now = new Date()) {
|
|
29
|
+
return ((!entry.uploaded_object_key &&
|
|
30
|
+
(isLiveSyncCollectableAttributionState(entry.state, entry.reason, Boolean(entry.worktree_fingerprint)) || entry.reason === "repo_not_on_disk")) ||
|
|
31
|
+
outsideRootEntryOwesUpload(entry, now));
|
|
24
32
|
}
|
|
25
33
|
export function matchesLiveSyncWorktree(result, target) {
|
|
26
34
|
return (isLiveRawEvidenceSyncAttribution(result.state, result.reason, result.worktree !== null) &&
|
|
@@ -118,6 +118,6 @@ function announceNothingInRoot(sessionsOutsideRoot, collectionRootCount) {
|
|
|
118
118
|
reason: "nothing_in_root",
|
|
119
119
|
sessions_outside_root: sessionsOutsideRoot,
|
|
120
120
|
collection_root_count: collectionRootCount,
|
|
121
|
-
next_action: "
|
|
121
|
+
next_action: "none needed: outside-root sessions upload under the outside-roots label (see the [session-sync] outside-root raw upload log line for why none did this tick); add a root only to attribute them to a repo",
|
|
122
122
|
}));
|
|
123
123
|
}
|
|
@@ -0,0 +1,281 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* BLI-17916: the scheduled sync uploads the transcript of a session that sits
|
|
3
|
+
* outside every approved root, the same way `cockpit backfill` does.
|
|
4
|
+
*
|
|
5
|
+
* Edward's ruling of 2026-09-17 (ticket 10734) made approved roots a LABEL on
|
|
6
|
+
* the upload, never a gate. The scheduled sync honoured it for the session
|
|
7
|
+
* REPORT only: the redacted transcript bytes of an outside-root session went up
|
|
8
|
+
* through `cockpit backfill` and nothing else, so on a machine where backfill
|
|
9
|
+
* never finished about a third of the last 30 days of sessions (2,301 of 7,815)
|
|
10
|
+
* could never be summarised.
|
|
11
|
+
*
|
|
12
|
+
* This is the one step that closes it. After the scan, before the upload, it
|
|
13
|
+
* picks which outside-root sessions ride this tick and hands each the SAME
|
|
14
|
+
* synthetic workspace backfill uses (`backfill-outside-roots.ts`), so the rest
|
|
15
|
+
* of the pass needs no special case: a session with a workspace is an ordinary
|
|
16
|
+
* sync target, goes through the ordinary envelope, and meets the same redaction.
|
|
17
|
+
* Attribution is left alone: the state stays `unattributed`, the reason stays
|
|
18
|
+
* `cwd_outside_scanned_worktrees`, the row stays `root_scope: "outside"`, and
|
|
19
|
+
* the workspace is transport identity only, never a repo the machine owns.
|
|
20
|
+
*
|
|
21
|
+
* What this step decides, and says out loud:
|
|
22
|
+
*
|
|
23
|
+
* - A session already durable with unchanged content rides for free. It
|
|
24
|
+
* uploads nothing (the object cursor reuses it) and keeps its row stable.
|
|
25
|
+
* - A session that is new, changed or not yet uploaded is "owed". The newest
|
|
26
|
+
* ones ride this tick up to a per-tick session cap. The rest wait, and their
|
|
27
|
+
* row says `deferred_object_budget`, the existing "next tick has room"
|
|
28
|
+
* label, instead of the old "not uploadable" one.
|
|
29
|
+
* - A transcript larger than the wire cap can never upload, so it does not
|
|
30
|
+
* spend a slot every tick; its row says `file_too_large`.
|
|
31
|
+
* - Nothing here runs for a source the operator switched off:
|
|
32
|
+
* `collect_claude_jsonl: false` means the Claude store is never read (the
|
|
33
|
+
* plan, not this step, decides that), so there are no Claude results to pick.
|
|
34
|
+
*
|
|
35
|
+
* The cap, the byte allowance and the object allowance are per tick, so a
|
|
36
|
+
* 1,800-session backlog drains across many ticks and never blows one tick's
|
|
37
|
+
* time or the machine's memory. Newest first: the sessions a person is most
|
|
38
|
+
* likely to be waiting on summarise first, and a backlog of old ones never
|
|
39
|
+
* starves today's.
|
|
40
|
+
*
|
|
41
|
+
* A new sibling must also join `scripts/build-public-cli.mjs` `runtimeFiles`.
|
|
42
|
+
*/
|
|
43
|
+
import fs from "node:fs/promises";
|
|
44
|
+
import path from "node:path";
|
|
45
|
+
import { RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES } from "@bli-cockpit/telemetry-core";
|
|
46
|
+
import { isRedactionStubObjectKey } from "../cursors/raw-evidence-stub-entry.js";
|
|
47
|
+
import { describeError } from "../health-detail.js";
|
|
48
|
+
import { writeJsonFile } from "../local-state-files.js";
|
|
49
|
+
import { rootScopeForReason } from "./agent-session-root-scope.js";
|
|
50
|
+
import { ensureOutsideRootsWorkspace, isOutsideRootsWorkspace, outsideRootsWorkspace, } from "./backfill-outside-roots.js";
|
|
51
|
+
/** Most outside-root sessions that upload bytes in one tick. 4 ticks an hour, so up to 3,840 a day. */
|
|
52
|
+
export const OUTSIDE_ROOT_SESSIONS_PER_TICK = 40;
|
|
53
|
+
/** Redacted bytes an outside-root pass may stage in one tick, kept apart from the in-root allowance. */
|
|
54
|
+
export const OUTSIDE_ROOT_BYTES_PER_TICK = 256 * 1024 * 1024;
|
|
55
|
+
/** Files (a Claude main plus its sidecars count one each) an outside-root pass may collect in one tick. */
|
|
56
|
+
export const OUTSIDE_ROOT_OBJECTS_PER_TICK = 120;
|
|
57
|
+
/**
|
|
58
|
+
* How far back the one-time catch-up scan reaches, and the oldest outside-root
|
|
59
|
+
* session that still widens a later tick's window. Longer history stays
|
|
60
|
+
* `cockpit backfill`'s job.
|
|
61
|
+
*/
|
|
62
|
+
export const OUTSIDE_ROOT_HORIZON_DAYS = 30;
|
|
63
|
+
export const OUTSIDE_ROOT_HORIZON_MINUTES = OUTSIDE_ROOT_HORIZON_DAYS * 24 * 60;
|
|
64
|
+
/** The upload_reason a session carries while it waits for a later tick. The existing transient label. */
|
|
65
|
+
export const OUTSIDE_ROOT_WAITING_REASON = "deferred_object_budget";
|
|
66
|
+
/** The upload_reason of a transcript over the wire cap. The existing deterministic label. */
|
|
67
|
+
export const OUTSIDE_ROOT_TOO_LARGE_REASON = "file_too_large";
|
|
68
|
+
const CATCH_UP_MARKER = "outside-root-catchup.json";
|
|
69
|
+
export function outsideRootCatchUpMarkerPath(paths) {
|
|
70
|
+
return path.join(paths.cursors_dir, CATCH_UP_MARKER);
|
|
71
|
+
}
|
|
72
|
+
/** Has the one-time 30-day look-back already run on this machine? */
|
|
73
|
+
export async function outsideRootCatchUpIsPending(paths) {
|
|
74
|
+
try {
|
|
75
|
+
await fs.access(outsideRootCatchUpMarkerPath(paths));
|
|
76
|
+
return false;
|
|
77
|
+
}
|
|
78
|
+
catch {
|
|
79
|
+
return true;
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
/**
|
|
83
|
+
* Record that the look-back ran, so later ticks go back to the normal window.
|
|
84
|
+
* Best effort: a marker that cannot be written only means the next tick looks
|
|
85
|
+
* back once more, and it says so.
|
|
86
|
+
*/
|
|
87
|
+
export async function markOutsideRootCatchUpDone(paths, facts) {
|
|
88
|
+
try {
|
|
89
|
+
await writeJsonFile(outsideRootCatchUpMarkerPath(paths), {
|
|
90
|
+
completed_at: facts.now.toISOString(),
|
|
91
|
+
window_days: OUTSIDE_ROOT_HORIZON_DAYS,
|
|
92
|
+
sessions_seen: facts.sessions_seen,
|
|
93
|
+
});
|
|
94
|
+
console.error("[session-sync] outside-root catch-up look-back done", JSON.stringify({ reason: "catch_up_marker_written", window_days: OUTSIDE_ROOT_HORIZON_DAYS, sessions_seen: facts.sessions_seen }));
|
|
95
|
+
}
|
|
96
|
+
catch (error) {
|
|
97
|
+
console.error("[session-sync] outside-root catch-up marker not written", JSON.stringify({ reason: "catch_up_marker_write_failed", ...describeError(error) }));
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
/** Widens a scan window to the catch-up horizon while the look-back is owed. */
|
|
101
|
+
export function widenForCatchUp(sinceMinutes, catchUpPending, normalMinutes) {
|
|
102
|
+
if (!catchUpPending)
|
|
103
|
+
return sinceMinutes;
|
|
104
|
+
return Math.max(sinceMinutes ?? normalMinutes, OUTSIDE_ROOT_HORIZON_MINUTES);
|
|
105
|
+
}
|
|
106
|
+
/**
|
|
107
|
+
* Does this cursor entry belong to an outside-root session that still owes its
|
|
108
|
+
* transcript? Such a session keeps the next scan window open back to its age,
|
|
109
|
+
* the same way an undurable in-root session does. A transcript over the wire
|
|
110
|
+
* cap never counts (it can never become durable), and neither does one older
|
|
111
|
+
* than the horizon (that is backfill's).
|
|
112
|
+
*/
|
|
113
|
+
export function outsideRootEntryOwesUpload(entry, now = new Date()) {
|
|
114
|
+
if (rootScopeForReason(entry.reason) !== "outside")
|
|
115
|
+
return false;
|
|
116
|
+
if (entry.uploaded_object_key && !isRedactionStubObjectKey(entry.uploaded_object_key))
|
|
117
|
+
return false;
|
|
118
|
+
if (entry.byte_size > RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES)
|
|
119
|
+
return false;
|
|
120
|
+
return now.getTime() - entry.file_mtime_ms <= OUTSIDE_ROOT_HORIZON_MINUTES * 60_000;
|
|
121
|
+
}
|
|
122
|
+
/** Is this scan result a session outside every approved root with no worktree of its own? */
|
|
123
|
+
function isUnclaimedOutsideRootResult(result) {
|
|
124
|
+
return !result.worktree && rootScopeForReason(result.reason) === "outside";
|
|
125
|
+
}
|
|
126
|
+
/** Already durable remotely with the bytes the scan just read: it needs no upload. */
|
|
127
|
+
function isDurableAndUnchanged(result, entry) {
|
|
128
|
+
return Boolean(entry &&
|
|
129
|
+
entry.uploaded_object_key &&
|
|
130
|
+
!isRedactionStubObjectKey(entry.uploaded_object_key) &&
|
|
131
|
+
!entry.stubbed &&
|
|
132
|
+
result.content_hash_sha256 &&
|
|
133
|
+
entry.file_hash_sha256 === result.content_hash_sha256);
|
|
134
|
+
}
|
|
135
|
+
function sightingsOf(scan, plan) {
|
|
136
|
+
const sightings = [];
|
|
137
|
+
for (const result of scan.codexAttribution.results) {
|
|
138
|
+
if (isUnclaimedOutsideRootResult(result)) {
|
|
139
|
+
sightings.push({ source: "codex", sessionId: result.codex_session_id, result });
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
// A machine that opted out of Claude has no Claude results to begin with
|
|
143
|
+
// (`session-sync-scan.ts` returns an empty scan); the guard keeps this step
|
|
144
|
+
// honest on its own as well.
|
|
145
|
+
if (plan.claudeEnabled) {
|
|
146
|
+
for (const result of scan.claudeAttribution.results) {
|
|
147
|
+
if (isUnclaimedOutsideRootResult(result)) {
|
|
148
|
+
sightings.push({ source: "claude_code", sessionId: result.claude_session_id, result });
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
return sightings;
|
|
153
|
+
}
|
|
154
|
+
export function outsideRootKey(source, sessionId) {
|
|
155
|
+
return `${source}:${sessionId}`;
|
|
156
|
+
}
|
|
157
|
+
/**
|
|
158
|
+
* Pick this tick's outside-root sessions and give them the synthetic workspace.
|
|
159
|
+
* Returns the scan to upload from (the input scan when nothing is admitted) with
|
|
160
|
+
* the admission facts the upload and the report steps read.
|
|
161
|
+
*/
|
|
162
|
+
export async function admitOutsideRootSessions(options) {
|
|
163
|
+
const { plan, scan } = options;
|
|
164
|
+
const sightings = sightingsOf(scan, plan);
|
|
165
|
+
if (sightings.length === 0) {
|
|
166
|
+
return { ...scan, outsideRoots: null };
|
|
167
|
+
}
|
|
168
|
+
const cursorFor = (source) => source === "codex" ? plan.codexCursorBefore : plan.claudeCursorBefore;
|
|
169
|
+
const free = [];
|
|
170
|
+
const owed = [];
|
|
171
|
+
const holds = new Map();
|
|
172
|
+
let tooLarge = 0;
|
|
173
|
+
for (const sighting of sightings) {
|
|
174
|
+
const entry = cursorFor(sighting.source).sessions[sighting.sessionId];
|
|
175
|
+
if (sighting.result.byte_size > RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES) {
|
|
176
|
+
holds.set(outsideRootKey(sighting.source, sighting.sessionId), OUTSIDE_ROOT_TOO_LARGE_REASON);
|
|
177
|
+
tooLarge += 1;
|
|
178
|
+
}
|
|
179
|
+
else if (isDurableAndUnchanged(sighting.result, entry)) {
|
|
180
|
+
free.push(sighting);
|
|
181
|
+
}
|
|
182
|
+
else {
|
|
183
|
+
owed.push(sighting);
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
// Newest first, with the session id as a tie-break so two ticks over the same
|
|
187
|
+
// files pick the same sessions.
|
|
188
|
+
owed.sort((left, right) => right.result.session_file_mtime_ms - left.result.session_file_mtime_ms ||
|
|
189
|
+
left.sessionId.localeCompare(right.sessionId));
|
|
190
|
+
const riding = owed.slice(0, OUTSIDE_ROOT_SESSIONS_PER_TICK);
|
|
191
|
+
const waiting = owed.slice(OUTSIDE_ROOT_SESSIONS_PER_TICK);
|
|
192
|
+
for (const sighting of waiting) {
|
|
193
|
+
holds.set(outsideRootKey(sighting.source, sighting.sessionId), OUTSIDE_ROOT_WAITING_REASON);
|
|
194
|
+
}
|
|
195
|
+
const admitted = [...riding, ...free];
|
|
196
|
+
const counts = {
|
|
197
|
+
outside_root_sessions: sightings.length,
|
|
198
|
+
riding_owed: riding.length,
|
|
199
|
+
riding_already_durable: free.length,
|
|
200
|
+
waiting_next_tick: waiting.length,
|
|
201
|
+
too_large: tooLarge,
|
|
202
|
+
codex_riding: admitted.filter((sighting) => sighting.source === "codex").length,
|
|
203
|
+
claude_riding: admitted.filter((sighting) => sighting.source === "claude_code").length,
|
|
204
|
+
};
|
|
205
|
+
const budget = {
|
|
206
|
+
sessions_per_tick: OUTSIDE_ROOT_SESSIONS_PER_TICK,
|
|
207
|
+
bytes_per_tick: OUTSIDE_ROOT_BYTES_PER_TICK,
|
|
208
|
+
objects_per_tick: OUTSIDE_ROOT_OBJECTS_PER_TICK,
|
|
209
|
+
};
|
|
210
|
+
if (admitted.length === 0) {
|
|
211
|
+
console.error("[session-sync] outside-root raw upload", JSON.stringify({ reason: "nothing_to_upload_outside_roots", ...counts, ...budget }));
|
|
212
|
+
return { ...scan, outsideRoots: { workspace: null, holds, counts } };
|
|
213
|
+
}
|
|
214
|
+
const workspace = outsideRootsWorkspace(options.homeDir);
|
|
215
|
+
try {
|
|
216
|
+
await ensureOutsideRootsWorkspace(workspace);
|
|
217
|
+
}
|
|
218
|
+
catch (error) {
|
|
219
|
+
// Not silent, and not a crash: every outside-root session keeps waiting and
|
|
220
|
+
// says why, and the in-root sessions of this tick are unaffected.
|
|
221
|
+
for (const sighting of admitted) {
|
|
222
|
+
holds.set(outsideRootKey(sighting.source, sighting.sessionId), "outside_root_workspace_unavailable");
|
|
223
|
+
}
|
|
224
|
+
console.error("[session-sync] outside-root raw upload", JSON.stringify({ reason: "workspace_unavailable", ...counts, ...budget, ...describeError(error) }));
|
|
225
|
+
return { ...scan, outsideRoots: { workspace: null, holds, counts } };
|
|
226
|
+
}
|
|
227
|
+
const admittedKeys = new Set(admitted.map((sighting) => outsideRootKey(sighting.source, sighting.sessionId)));
|
|
228
|
+
const labelled = {
|
|
229
|
+
...scan,
|
|
230
|
+
codexAttribution: {
|
|
231
|
+
...scan.codexAttribution,
|
|
232
|
+
results: scan.codexAttribution.results.map((result) => admittedKeys.has(outsideRootKey("codex", result.codex_session_id)) && isUnclaimedOutsideRootResult(result)
|
|
233
|
+
? { ...result, worktree: workspace }
|
|
234
|
+
: result),
|
|
235
|
+
},
|
|
236
|
+
claudeAttribution: {
|
|
237
|
+
...scan.claudeAttribution,
|
|
238
|
+
results: scan.claudeAttribution.results.map((result) => admittedKeys.has(outsideRootKey("claude_code", result.claude_session_id)) && isUnclaimedOutsideRootResult(result)
|
|
239
|
+
? { ...result, worktree: workspace }
|
|
240
|
+
: result),
|
|
241
|
+
},
|
|
242
|
+
outsideRoots: { workspace, holds, counts },
|
|
243
|
+
};
|
|
244
|
+
console.error("[session-sync] outside-root raw upload", JSON.stringify({ reason: "outside_root_sessions_admitted", ...counts, ...budget }));
|
|
245
|
+
return labelled;
|
|
246
|
+
}
|
|
247
|
+
/**
|
|
248
|
+
* Says why an outside-root session did not ride this tick, on its report row.
|
|
249
|
+
* A row that has a pointer, or a reason the upload itself recorded, is left
|
|
250
|
+
* alone; the rest were never offered an upload, so the step that decided
|
|
251
|
+
* ("waiting for a later tick", "over the wire cap") names it instead of the old
|
|
252
|
+
* "not uploadable" label, which stopped being true with this ticket.
|
|
253
|
+
*/
|
|
254
|
+
export function applyOutsideRootHolds(rows, holds) {
|
|
255
|
+
if (!holds || holds.size === 0)
|
|
256
|
+
return rows;
|
|
257
|
+
return rows.map((row) => {
|
|
258
|
+
const hold = holds.get(outsideRootKey(row.source, row.codex_session_id));
|
|
259
|
+
if (!hold || row.raw_evidence_pointer_id || row.upload_state !== "not_uploaded")
|
|
260
|
+
return row;
|
|
261
|
+
if (row.upload_reason && !row.upload_reason.startsWith("not_uploadable_attribution_state"))
|
|
262
|
+
return row;
|
|
263
|
+
return { ...row, upload_reason: hold };
|
|
264
|
+
});
|
|
265
|
+
}
|
|
266
|
+
/**
|
|
267
|
+
* The outside-root pass gets its own allowance, so a busy in-root tick cannot
|
|
268
|
+
* starve it and it cannot take the in-root room. The disk gate and the image
|
|
269
|
+
* description session stay shared (they live on the same budget object).
|
|
270
|
+
*/
|
|
271
|
+
export function giveOutsideRootsTheirOwnAllowance(budget) {
|
|
272
|
+
budget.remainingBytes = OUTSIDE_ROOT_BYTES_PER_TICK;
|
|
273
|
+
budget.remainingObjects = OUTSIDE_ROOT_OBJECTS_PER_TICK;
|
|
274
|
+
}
|
|
275
|
+
/** The sync target list with the outside-roots workspace moved last, so in-root evidence goes first. */
|
|
276
|
+
export function outsideRootsLast(worktrees) {
|
|
277
|
+
return [
|
|
278
|
+
...worktrees.filter((worktree) => !isOutsideRootsWorkspace(worktree)),
|
|
279
|
+
...worktrees.filter((worktree) => isOutsideRootsWorkspace(worktree)),
|
|
280
|
+
];
|
|
281
|
+
}
|
|
@@ -23,6 +23,7 @@ import { CLAUDE_CURSOR_FILENAME, emptyRawEvidenceCursorState, readRawEvidenceCur
|
|
|
23
23
|
import { normalizeCollectionRoots } from "../root-normalization.js";
|
|
24
24
|
import { readLocalUploadSpoolState, } from "../spool/local-spool.js";
|
|
25
25
|
import { liveSyncCursorEntryRequiresRetry } from "./session-sync-attribution.js";
|
|
26
|
+
import { outsideRootCatchUpIsPending } from "./session-sync-outside-roots.js";
|
|
26
27
|
/**
|
|
27
28
|
* The bound applied when a retry is pending but no cursor entry names the
|
|
28
29
|
* stuck file (e.g. a legacy spooled upload recorded before per-source retry
|
|
@@ -46,7 +47,7 @@ export async function planSyncFromLocalState(options) {
|
|
|
46
47
|
const config = await readLocalCollectorConfig(options.paths).catch(() => null);
|
|
47
48
|
const claudeEnabled = config?.collect_claude_jsonl !== false;
|
|
48
49
|
const collectionRoots = normalizeCollectionRoots(options.approvedCollectionRoots ?? config?.default_repo_paths ?? []);
|
|
49
|
-
const [codexCursorBefore, claudeCursorBefore, uploadSpool] = await Promise.all([
|
|
50
|
+
const [codexCursorBefore, claudeCursorBefore, uploadSpool, outsideRootCatchUpPending] = await Promise.all([
|
|
50
51
|
readRawEvidenceCursor(options.paths).catch(() => emptyRawEvidenceCursorState()),
|
|
51
52
|
claudeEnabled
|
|
52
53
|
? readRawEvidenceCursor(options.paths, {
|
|
@@ -54,6 +55,7 @@ export async function planSyncFromLocalState(options) {
|
|
|
54
55
|
}).catch(() => emptyRawEvidenceCursorState())
|
|
55
56
|
: Promise.resolve(emptyRawEvidenceCursorState()),
|
|
56
57
|
readLocalUploadSpoolState(options.paths).catch(() => null),
|
|
58
|
+
outsideRootCatchUpIsPending(options.paths),
|
|
57
59
|
]);
|
|
58
60
|
// A spooled upload from a CLI old enough not to have recorded its source
|
|
59
61
|
// could have come from either one, so both sources own it until it clears.
|
|
@@ -73,6 +75,7 @@ export async function planSyncFromLocalState(options) {
|
|
|
73
75
|
sourceHasPendingRetries("claude_code", uploadSpool, legacyRetryPending, claudeSourceRetryPending, claudeCursorBefore),
|
|
74
76
|
codexRetrySinceMinutes: boundedRetrySinceMinutes(codexCursorBefore, options.now),
|
|
75
77
|
claudeRetrySinceMinutes: boundedRetrySinceMinutes(claudeCursorBefore, options.now),
|
|
78
|
+
outsideRootCatchUpPending,
|
|
76
79
|
};
|
|
77
80
|
}
|
|
78
81
|
/**
|
|
@@ -85,7 +88,7 @@ export async function planSyncFromLocalState(options) {
|
|
|
85
88
|
export function boundedRetrySinceMinutes(cursor, now) {
|
|
86
89
|
let oldestUndurableMtimeMs = null;
|
|
87
90
|
for (const entry of Object.values(cursor.sessions)) {
|
|
88
|
-
if (!liveSyncCursorEntryRequiresRetry(entry))
|
|
91
|
+
if (!liveSyncCursorEntryRequiresRetry(entry, now))
|
|
89
92
|
continue;
|
|
90
93
|
if (oldestUndurableMtimeMs === null ||
|
|
91
94
|
entry.file_mtime_ms < oldestUndurableMtimeMs) {
|
|
@@ -14,7 +14,10 @@ import { scanAndAttributeClaudeSessions, } from "../adapters/claude-attribution.
|
|
|
14
14
|
import { CLAUDE_CURSOR_FILENAME } from "../cursors/raw-evidence-cursor.js";
|
|
15
15
|
import { clearSourceRetryFailure, recordSourceRetryFailure, } from "../spool/local-spool.js";
|
|
16
16
|
import { claudeAttributionReadFailureCount, codexAttributionReadFailureCount, } from "./agent-session-report.js";
|
|
17
|
+
import { widenForCatchUp } from "./session-sync-outside-roots.js";
|
|
17
18
|
const CLAUDE_FIRST_RUN_BACKFILL_MINUTES = 14 * 24 * 60;
|
|
19
|
+
/** The Claude scan's own default window when nothing widens it: the last 24 hours. */
|
|
20
|
+
const CLAUDE_DEFAULT_WINDOW_MINUTES = 24 * 60;
|
|
18
21
|
/**
|
|
19
22
|
* Scan and attribute both session sources, then settle each source's scan-retry
|
|
20
23
|
* bookkeeping against what the scan actually found.
|
|
@@ -31,6 +34,7 @@ export async function scanAndAttributeBothSources(options) {
|
|
|
31
34
|
collectionRoots: plan.collectionRoots,
|
|
32
35
|
retryPending: plan.codexRetryPending,
|
|
33
36
|
retrySinceMinutes: plan.codexRetrySinceMinutes,
|
|
37
|
+
outsideRootCatchUpPending: plan.outsideRootCatchUpPending,
|
|
34
38
|
});
|
|
35
39
|
// First run (D B.4 §8): without a Claude cursor yet, widen the window to 14
|
|
36
40
|
// days so the first sync captures retroactive history instead of only 24h.
|
|
@@ -45,6 +49,7 @@ export async function scanAndAttributeBothSources(options) {
|
|
|
45
49
|
retryPending: plan.claudeRetryPending,
|
|
46
50
|
retrySinceMinutes: plan.claudeRetrySinceMinutes,
|
|
47
51
|
firstRunBackfill,
|
|
52
|
+
outsideRootCatchUpPending: plan.outsideRootCatchUpPending,
|
|
48
53
|
});
|
|
49
54
|
await reconcileSourceScanRetry({
|
|
50
55
|
paths: options.paths,
|
|
@@ -72,9 +77,9 @@ export async function scanAndAttributeBothSources(options) {
|
|
|
72
77
|
* is always scanned, unlike Claude which can be disabled entirely.
|
|
73
78
|
*/
|
|
74
79
|
async function scanCodexSessionsForSync(options) {
|
|
75
|
-
const sinceMinutes = options.retryPending
|
|
80
|
+
const sinceMinutes = widenForCatchUp(options.retryPending
|
|
76
81
|
? options.retrySinceMinutes
|
|
77
|
-
: CODEX_ATTRIBUTION_SCAN_WINDOW_MINUTES;
|
|
82
|
+
: CODEX_ATTRIBUTION_SCAN_WINDOW_MINUTES, options.outsideRootCatchUpPending, CODEX_ATTRIBUTION_SCAN_WINDOW_MINUTES);
|
|
78
83
|
let scan = await scanAndAttributeCodexSessions({
|
|
79
84
|
sessionsDirs: defaultCodexSessionDirs(options.homeDir),
|
|
80
85
|
worktrees: options.worktrees,
|
|
@@ -107,11 +112,11 @@ async function scanCodexSessionsForSync(options) {
|
|
|
107
112
|
async function scanClaudeSessionsForSync(options) {
|
|
108
113
|
if (!options.claudeEnabled)
|
|
109
114
|
return emptyClaudeScan();
|
|
110
|
-
const sinceMinutes = options.retryPending
|
|
115
|
+
const sinceMinutes = widenForCatchUp(options.retryPending
|
|
111
116
|
? options.retrySinceMinutes
|
|
112
117
|
: options.firstRunBackfill
|
|
113
118
|
? CLAUDE_FIRST_RUN_BACKFILL_MINUTES
|
|
114
|
-
: undefined;
|
|
119
|
+
: undefined, options.outsideRootCatchUpPending, CLAUDE_DEFAULT_WINDOW_MINUTES);
|
|
115
120
|
const projectsDir = path.join(options.homeDir, ".claude", "projects");
|
|
116
121
|
let scan = await scanAndAttributeClaudeSessions({
|
|
117
122
|
projectsDir,
|