@bli-cockpit/cli 0.2.208 → 0.2.211
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/raw-evidence-sanitize.js +12 -2
- package/dist/commands/agent-session-report.js +1 -1
- package/dist/commands/doctor-access.js +7 -5
- package/dist/commands/install-receipts.js +27 -16
- package/dist/commands/local-help-commands.js +1 -1
- package/dist/commands/public-root.js +1 -1
- package/dist/commands/scout-render.js +2 -0
- package/dist/commands/session-sync-attribution.js +11 -3
- package/dist/commands/session-sync-health.js +1 -1
- package/dist/commands/session-sync-outside-roots.js +281 -0
- package/dist/commands/session-sync-plan.js +5 -2
- package/dist/commands/session-sync-scan.js +9 -4
- package/dist/commands/session-sync-upload.js +44 -25
- package/dist/commands/session-sync.js +29 -7
- package/dist/commands/settings-render.js +4 -0
- package/dist/commands/sources-check.js +15 -4
- package/dist/commands/sources-verdict.js +125 -14
- package/dist/commands/sync-report.js +1 -1
- package/dist/commands/sync.js +8 -1
- package/dist/image-description-session.js +7 -1
- package/dist/image-description-state.js +2 -2
- package/dist/image-mode-state.js +154 -0
- package/dist/local-state-files.js +20 -0
- package/dist/local-state-work-context.js +3 -1
- package/package.json +4 -4
|
@@ -10,6 +10,8 @@ import { startLocalWorkContext, startLocalWorkContextForAttributedTarget, } from
|
|
|
10
10
|
import { LocalUploadBlockedError, syncLocalAmbientEnvelope, } from "../upload.js";
|
|
11
11
|
import { RAW_EVIDENCE_DEFAULT_BYTE_BUDGET, RAW_EVIDENCE_DEFAULT_OBJECT_BUDGET, } from "../adapters/raw-evidence.js";
|
|
12
12
|
import { liveSyncTargetKey, liveSyncTargetWorktrees, matchesLiveSyncWorktree, } from "./session-sync-attribution.js";
|
|
13
|
+
import { isOutsideRootsWorkspace } from "./backfill-outside-roots.js";
|
|
14
|
+
import { giveOutsideRootsTheirOwnAllowance, outsideRootsLast, } from "./session-sync-outside-roots.js";
|
|
13
15
|
import { runYieldingAtBoundaries } from "./sync-yield.js";
|
|
14
16
|
const CLAUDE_DAMP_GROWTH_BYTES = 256 * 1024;
|
|
15
17
|
const CLAUDE_DAMP_MAX_AGE_MS = 6 * 60 * 60 * 1000;
|
|
@@ -22,10 +24,12 @@ const CLAUDE_DAMP_MAX_AGE_MS = 6 * 60 * 60 * 1000;
|
|
|
22
24
|
*/
|
|
23
25
|
export async function syncEveryTargetWorktree(options) {
|
|
24
26
|
const { run, scan } = options;
|
|
25
|
-
|
|
27
|
+
// BLI-17916: the outside-roots workspace goes last, so in-root evidence is
|
|
28
|
+
// never held up behind it.
|
|
29
|
+
const syncWorktrees = outsideRootsLast(liveSyncTargetWorktrees(run.worktrees, [
|
|
26
30
|
...scan.codexAttribution.results,
|
|
27
31
|
...scan.claudeAttribution.results,
|
|
28
|
-
]);
|
|
32
|
+
]));
|
|
29
33
|
const discoveredWorktreeKeys = new Set(run.worktrees.map(liveSyncTargetKey));
|
|
30
34
|
// sessionId -> prior durable pointer for damped sessions (drives the
|
|
31
35
|
// growth_damped count + the skip_main decision).
|
|
@@ -40,21 +44,28 @@ export async function syncEveryTargetWorktree(options) {
|
|
|
40
44
|
items: syncWorktrees,
|
|
41
45
|
shouldYield: run.yieldCheck,
|
|
42
46
|
describe: (outcome) => ({ status: outcome.sync.status }),
|
|
43
|
-
runOne: (worktree) =>
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
47
|
+
runOne: (worktree) => {
|
|
48
|
+
// The outside-root pass has its own per-tick allowance (bytes and files),
|
|
49
|
+
// so a busy in-root tick cannot starve it and it cannot take in-root room.
|
|
50
|
+
if (isOutsideRootsWorkspace(worktree)) {
|
|
51
|
+
giveOutsideRootsTheirOwnAllowance(rawEvidenceBudget);
|
|
52
|
+
}
|
|
53
|
+
return syncOneWorktree({
|
|
54
|
+
run,
|
|
55
|
+
now: options.now,
|
|
56
|
+
plan: options.plan,
|
|
57
|
+
scan,
|
|
58
|
+
worktree,
|
|
59
|
+
syncWorktrees,
|
|
60
|
+
rawEvidenceBudget,
|
|
61
|
+
// A target git discovery never produced is one the fallback attribution
|
|
62
|
+
// synthesized, and it syncs through its own attributed work context.
|
|
63
|
+
isDiscoveredWorktree: discoveredWorktreeKeys.has(liveSyncTargetKey(worktree)),
|
|
64
|
+
recordDampedClaudeSession: (sessionId, priorPointer) => {
|
|
65
|
+
dampedClaudePointers.set(sessionId, priorPointer);
|
|
66
|
+
},
|
|
67
|
+
});
|
|
68
|
+
},
|
|
58
69
|
});
|
|
59
70
|
const outcomes = loop.results;
|
|
60
71
|
const everyWorktreeUploaded = outcomes.every((outcome) => outcome.sync.status === "uploaded");
|
|
@@ -87,12 +98,14 @@ function claudeDurablePointersFromCursor(claudeCursorBefore) {
|
|
|
87
98
|
*/
|
|
88
99
|
async function syncOneWorktree(options) {
|
|
89
100
|
const { run, worktree } = options;
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
const
|
|
101
|
+
// The outside-roots workspace starts its work context the way `cockpit
|
|
102
|
+
// backfill` does (a plain folder, no attributed-target proof): it is a
|
|
103
|
+
// transport folder the collector owns, not a repo anyone attributed.
|
|
104
|
+
const outsideRoots = isOutsideRootsWorkspace(worktree);
|
|
105
|
+
const attributedSyntheticTarget = options.isDiscoveredWorktree || outsideRoots ? null : worktree;
|
|
106
|
+
const ensureContext = () => startWorktreeContext(options, attributedSyntheticTarget, outsideRoots);
|
|
94
107
|
let context = null;
|
|
95
|
-
if (run.startContexts || attributedSyntheticTarget) {
|
|
108
|
+
if (run.startContexts || attributedSyntheticTarget || outsideRoots) {
|
|
96
109
|
context = await ensureContext();
|
|
97
110
|
}
|
|
98
111
|
const syncOptions = ambientEnvelopeForWorktree(options);
|
|
@@ -112,15 +125,21 @@ async function syncOneWorktree(options) {
|
|
|
112
125
|
}
|
|
113
126
|
return { worktree, context, sync };
|
|
114
127
|
}
|
|
115
|
-
function startWorktreeContext(options, attributedSyntheticTarget) {
|
|
128
|
+
function startWorktreeContext(options, attributedSyntheticTarget, outsideRoots) {
|
|
116
129
|
const { run, worktree } = options;
|
|
117
130
|
const contextOptions = {
|
|
118
131
|
homeDir: run.homeDir,
|
|
119
132
|
repoRoot: worktree.repo_root,
|
|
120
|
-
|
|
133
|
+
// The caller's ticket and branch belong to the caller's repo, never to the
|
|
134
|
+
// outside-roots workspace.
|
|
135
|
+
...(outsideRoots ? {} : { activeTicketId: run.activeTicketId }),
|
|
121
136
|
operatorId: run.operatorId,
|
|
122
137
|
sessionId: run.sessionId,
|
|
123
|
-
...(
|
|
138
|
+
...(outsideRoots
|
|
139
|
+
? { branch: worktree.branch, leaveActiveContext: true }
|
|
140
|
+
: attributedSyntheticTarget
|
|
141
|
+
? {}
|
|
142
|
+
: { branch: run.branch }),
|
|
124
143
|
};
|
|
125
144
|
return attributedSyntheticTarget
|
|
126
145
|
? startLocalWorkContextForAttributedTarget(contextOptions, attributedSyntheticTarget)
|
|
@@ -43,7 +43,8 @@ import { getCollectorRuntimePaths } from "../local-state.js";
|
|
|
43
43
|
import { buildAgentSessionReport, buildAgentSessionSummary, } from "./agent-session-report.js";
|
|
44
44
|
import { collectSessionCommits, withSessionCommits } from "../session-commits.js";
|
|
45
45
|
import { planSyncFromLocalState } from "./session-sync-plan.js";
|
|
46
|
-
import { scanAndAttributeBothSources } from "./session-sync-scan.js";
|
|
46
|
+
import { scanAndAttributeBothSources, sourceScanFailureReason, } from "./session-sync-scan.js";
|
|
47
|
+
import { admitOutsideRootSessions, applyOutsideRootHolds, markOutsideRootCatchUpDone, } from "./session-sync-outside-roots.js";
|
|
47
48
|
import { syncEveryTargetWorktree } from "./session-sync-upload.js";
|
|
48
49
|
import { reportSessionsAndAdvanceCursors } from "./session-sync-record.js";
|
|
49
50
|
import { decideSyncHealth } from "./session-sync-health.js";
|
|
@@ -55,6 +56,11 @@ export { sourceScanFailureReason, sourceScanRetryReason, } from "./session-sync-
|
|
|
55
56
|
export { SYNC_FAILED_WITHOUT_REASON } from "./session-sync-failures.js";
|
|
56
57
|
export { nothingInRootCount } from "./session-sync-health.js";
|
|
57
58
|
export { isDeliveryHoldReason, summarizeDeliveryHold, } from "./session-sync-hold.js";
|
|
59
|
+
/** Did either session store fail to read this tick? A failed read means the scan saw less than exists. */
|
|
60
|
+
function scanHadReadFailure(scan) {
|
|
61
|
+
return (sourceScanFailureReason("codex", scan.codexAttribution) !== null ||
|
|
62
|
+
sourceScanFailureReason("claude_code", scan.claudeAttribution) !== null);
|
|
63
|
+
}
|
|
58
64
|
/**
|
|
59
65
|
* Shared dual-source sync orchestration for single-repo and parent-folder
|
|
60
66
|
* modes. Codex AND Claude Code sessions are scanned and attributed once across
|
|
@@ -62,7 +68,8 @@ export { isDeliveryHoldReason, summarizeDeliveryHold, } from "./session-sync-hol
|
|
|
62
68
|
* fallback-attributed transcripts (Codex + Claude main + sidecars). The
|
|
63
69
|
* ambiguous/unattributed/skipped sessions inside an approved root retain those
|
|
64
70
|
* labels while syncing once through that root's synthetic folder workspace.
|
|
65
|
-
* Outside-root sessions
|
|
71
|
+
* Outside-root sessions keep their labels and upload through the collector's
|
|
72
|
+
* own outside-roots workspace, a few per tick (BLI-17916). The session row's upload state
|
|
66
73
|
* maps ONLY from the main-file outcome (D3); sidecar outcomes aggregate into
|
|
67
74
|
* CLI counts.
|
|
68
75
|
*
|
|
@@ -80,13 +87,21 @@ export async function runAttributedWorktreeSync(options) {
|
|
|
80
87
|
now,
|
|
81
88
|
approvedCollectionRoots: options.collectionRoots,
|
|
82
89
|
});
|
|
83
|
-
const
|
|
90
|
+
const scanned = await scanAndAttributeBothSources({
|
|
84
91
|
paths,
|
|
85
92
|
homeDir,
|
|
86
93
|
now,
|
|
87
94
|
worktrees: options.worktrees,
|
|
88
95
|
plan,
|
|
89
96
|
});
|
|
97
|
+
// BLI-17916: roots are a label, never a gate. Outside-root sessions ride this
|
|
98
|
+
// tick's upload under the synthetic workspace, a few per tick, newest first.
|
|
99
|
+
const scan = await admitOutsideRootSessions({
|
|
100
|
+
homeDir: options.homeDir,
|
|
101
|
+
plan,
|
|
102
|
+
scan: scanned,
|
|
103
|
+
now,
|
|
104
|
+
});
|
|
90
105
|
const worktreePass = await syncEveryTargetWorktree({
|
|
91
106
|
run: options,
|
|
92
107
|
now,
|
|
@@ -102,18 +117,20 @@ export async function runAttributedWorktreeSync(options) {
|
|
|
102
117
|
}
|
|
103
118
|
console.error("[session-sync] bindings read", JSON.stringify({ bound_worktrees: ticketByWorktree.size, reason: "active_work_context" }));
|
|
104
119
|
// BLI-13028: which commits each session's working tree created while it ran.
|
|
120
|
+
// Read from the scan as it was BEFORE the outside-root step: the synthetic
|
|
121
|
+
// workspace is not a git tree, and no commit belongs to it.
|
|
105
122
|
const commitsBySession = await collectSessionCommits([
|
|
106
|
-
...
|
|
107
|
-
...
|
|
123
|
+
...scanned.codexAttribution.results.map((result) => ({ key: `codex:${result.codex_session_id}`, result })),
|
|
124
|
+
...scanned.claudeAttribution.results.map((result) => ({ key: `claude_code:${result.claude_session_id}`, result })),
|
|
108
125
|
]);
|
|
109
|
-
const sessions = withSessionCommits(buildAgentSessionReport({
|
|
126
|
+
const sessions = applyOutsideRootHolds(withSessionCommits(buildAgentSessionReport({
|
|
110
127
|
ticketByWorktree,
|
|
111
128
|
codexResults: scan.codexAttribution.results,
|
|
112
129
|
claudeResults: scan.claudeAttribution.results,
|
|
113
130
|
outcomes: worktreePass.outcomes,
|
|
114
131
|
now,
|
|
115
132
|
claudePriorDurablePointers: worktreePass.claudePriorDurablePointers,
|
|
116
|
-
}), commitsBySession);
|
|
133
|
+
}), commitsBySession), scan.outsideRoots?.holds);
|
|
117
134
|
// BLI-4341: every session should arrive counted. Say how many did, and say
|
|
118
135
|
// it on the success path too: a silent drop from "all" to "some" is exactly
|
|
119
136
|
// the kind of decay nobody notices.
|
|
@@ -133,6 +150,11 @@ export async function runAttributedWorktreeSync(options) {
|
|
|
133
150
|
sessions,
|
|
134
151
|
outcomes: worktreePass.outcomes,
|
|
135
152
|
});
|
|
153
|
+
// The one-time 30-day look-back is done once a tick read both stores without
|
|
154
|
+
// a failure and recorded what it saw in the cursors.
|
|
155
|
+
if (plan.outsideRootCatchUpPending && !scanHadReadFailure(scan)) {
|
|
156
|
+
await markOutsideRootCatchUpDone(paths, { now, sessions_seen: sessions.length });
|
|
157
|
+
}
|
|
136
158
|
const summary = buildAgentSessionSummary({
|
|
137
159
|
codexAttribution: scan.codexAttribution,
|
|
138
160
|
claudeAttribution: scan.claudeAttribution,
|
|
@@ -138,6 +138,10 @@ export function renderSwitches(body) {
|
|
|
138
138
|
if (typeof body.readFailureReason === "string") {
|
|
139
139
|
lines.push(`${INDENT}values could not be read: ${body.readFailureReason}`);
|
|
140
140
|
}
|
|
141
|
+
// "No machine is on slim" must never hide "could not ask" (BLI-17273).
|
|
142
|
+
if (typeof body.collectorImageModeReadFailureReason === "string") {
|
|
143
|
+
lines.push(`${INDENT}machine image modes could not be listed: ${body.collectorImageModeReadFailureReason}`);
|
|
144
|
+
}
|
|
141
145
|
return lines;
|
|
142
146
|
}
|
|
143
147
|
export function renderModelRouting(body) {
|
|
@@ -1,11 +1,12 @@
|
|
|
1
1
|
import { writeLine } from "./cli-io.js";
|
|
2
2
|
import { readDescribedImageKeys } from "../image-description-state.js";
|
|
3
|
+
import { resolveImageHandlingMode } from "../image-mode-state.js";
|
|
3
4
|
import { CLAUDE_CURSOR_FILENAME, readRawEvidenceCursor, } from "../cursors/raw-evidence-cursor.js";
|
|
4
5
|
import { fetchTranscriptObjects } from "../evidence-transcripts-client.js";
|
|
5
6
|
import { DEFAULT_DASHBOARD_URL, getCollectorRuntimePaths, readLocalCollectorConfig, readLocalCollectorSessionFile, } from "../local-state.js";
|
|
6
7
|
import { discoverLocalTranscripts } from "./sources-discovery.js";
|
|
7
8
|
import { fileFacts, judgeSource } from "./sources-verdict.js";
|
|
8
|
-
const VERDICTS = ["stored_full", "stored_partial", "not_stored"];
|
|
9
|
+
const VERDICTS = ["stored_full", "stored_slim", "stored_partial", "not_stored"];
|
|
9
10
|
const DAY_MS = 24 * 60 * 60 * 1000;
|
|
10
11
|
export async function runSourcesCheck(command, io, now = new Date()) {
|
|
11
12
|
const startedAt = Date.now();
|
|
@@ -30,13 +31,18 @@ export async function runSourcesCheck(command, io, now = new Date()) {
|
|
|
30
31
|
claude_code: await readRawEvidenceCursor(paths, { filename: CLAUDE_CURSOR_FILENAME, sessionsOnly: true }),
|
|
31
32
|
};
|
|
32
33
|
const describedKeys = new Set(await readDescribedImageKeys(paths.state_dir));
|
|
34
|
+
// BLI-17273: judge against THIS machine's mode (the cache the sync-start receipt wrote, else the
|
|
35
|
+
// fleet default), never the bare constant. A machine that runs or ever ran slim holds slim server
|
|
36
|
+
// copies, which must never read as safe to delete.
|
|
37
|
+
const imageMode = await resolveImageHandlingMode(paths.state_dir);
|
|
38
|
+
console.error("[collector sources] judging against this machine's image mode", JSON.stringify({ reason: `image_mode_from_${imageMode.source}`, mode: imageMode.mode, slim_aware: imageMode.slimAware }));
|
|
33
39
|
const report = emptyReport(command, remote, local.newer_count, local.file_read_failed + local.directory_read_failed);
|
|
34
40
|
for (const file of local.files) {
|
|
35
41
|
const judged = await judgeSource({
|
|
36
42
|
file,
|
|
37
43
|
remotes: remote.files.get(file.key) ?? [],
|
|
38
44
|
cursorEntry: file.kind === "claude_subagent" ? null : (cursors[file.source].sessions[file.session_id] ?? null),
|
|
39
|
-
facts: fileFacts(file, undefined, { describedKeys }),
|
|
45
|
+
facts: fileFacts(file, undefined, { mode: imageMode.mode, describedKeys, slimAware: imageMode.slimAware }),
|
|
40
46
|
}).catch((error) => {
|
|
41
47
|
report.local.unreadable += 1;
|
|
42
48
|
console.error("[collector sources] a transcript could not be judged; left out of the answer", JSON.stringify({ reason: "file_unreadable", kind: file.kind, byte_size: file.byte_size, error_name: error?.name ?? "Error" }));
|
|
@@ -71,6 +77,7 @@ export async function runSourcesCheck(command, io, now = new Date()) {
|
|
|
71
77
|
newer_skipped: report.local.newer_skipped,
|
|
72
78
|
unreadable: report.local.unreadable,
|
|
73
79
|
stored_full: report.counts.stored_full,
|
|
80
|
+
stored_slim: report.counts.stored_slim,
|
|
74
81
|
stored_partial: report.counts.stored_partial,
|
|
75
82
|
not_stored: report.counts.not_stored,
|
|
76
83
|
reasons: report.reasons,
|
|
@@ -93,6 +100,7 @@ function emptyReport(command, remote, newerSkipped, unreadable) {
|
|
|
93
100
|
local: { examined: 0, newer_skipped: newerSkipped, unreadable },
|
|
94
101
|
counts: {
|
|
95
102
|
stored_full: { files: 0, bytes: 0 },
|
|
103
|
+
stored_slim: { files: 0, bytes: 0 },
|
|
96
104
|
stored_partial: { files: 0, bytes: 0 },
|
|
97
105
|
not_stored: { files: 0, bytes: 0 },
|
|
98
106
|
},
|
|
@@ -115,7 +123,10 @@ export function sourcesCheckLines(report) {
|
|
|
115
123
|
const lines = [
|
|
116
124
|
`Tower sources check: ${scope} on this machine`,
|
|
117
125
|
` stored in full ${report.counts.stored_full.files} file(s), ${mb(report.counts.stored_full.bytes)} MB`,
|
|
118
|
-
|
|
126
|
+
...(report.counts.stored_slim.files > 0
|
|
127
|
+
? [` stored slim ${report.counts.stored_slim.files} file(s), ${mb(report.counts.stored_slim.bytes)} MB (KEEP: Tower holds the slim form, the image bytes are only here)`]
|
|
128
|
+
: []),
|
|
129
|
+
` stored in part ${report.counts.stored_partial.files} file(s), ${mb(report.counts.stored_partial.bytes)} MB`,
|
|
119
130
|
` not stored ${report.counts.not_stored.files} file(s), ${mb(report.counts.not_stored.bytes)} MB`,
|
|
120
131
|
];
|
|
121
132
|
const reasons = Object.entries(report.reasons).sort((a, b) => b[1] - a[1]);
|
|
@@ -129,7 +140,7 @@ export function sourcesCheckLines(report) {
|
|
|
129
140
|
if (report.status === "incomplete") {
|
|
130
141
|
lines.push(` INCOMPLETE Tower's list was cut short (${report.reason}); "not stored" and "stored in part" may be understated. Run it again.`);
|
|
131
142
|
}
|
|
132
|
-
lines.push("Tower never deletes these files. `--json` lists every file with its path and verdict. Only a \"stored in full\" file is safe to delete by your own hand; see docs/runbooks/freeing-disk-on-a-collector-machine.md.");
|
|
143
|
+
lines.push("Tower never deletes these files. `--json` lists every file with its path and verdict. Only a \"stored in full\" file is safe to delete by your own hand (never \"stored slim\"); see docs/runbooks/freeing-disk-on-a-collector-machine.md.");
|
|
133
144
|
return lines;
|
|
134
145
|
}
|
|
135
146
|
function mb(bytes) {
|
|
@@ -18,6 +18,14 @@
|
|
|
18
18
|
* grow after its upload, else `remote_smaller`), or its size
|
|
19
19
|
* is not recorded (`remote_size_unknown`).
|
|
20
20
|
*
|
|
21
|
+
* stored_slim (BLI-17273) the copy Tower holds is the SLIM form of the file: images
|
|
22
|
+
* described and replaced by a marker, duplicate copies and bookkeeping
|
|
23
|
+
* stripped. Complete for every reader, but the local file holds the only
|
|
24
|
+
* copy of the image bytes, so a slim copy is NEVER `stored_full`: keep the
|
|
25
|
+
* file. Only judged for a machine that runs, or ever ran, slim
|
|
26
|
+
* (`image-mode-state.ts`); every other machine reads exactly as before.
|
|
27
|
+
* The rules are on `judgeWithSlimCopy` below.
|
|
28
|
+
*
|
|
21
29
|
* A file too large to redact in memory is compared by its raw size
|
|
22
30
|
* (`remote_at_least_file_size`). A file whose redaction would crash to the stub
|
|
23
31
|
* is compared the same way, never by the stub's 42 bytes.
|
|
@@ -28,7 +36,7 @@ import crypto from "node:crypto";
|
|
|
28
36
|
import fs from "node:fs/promises";
|
|
29
37
|
import path from "node:path";
|
|
30
38
|
import { RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES, REDACTION_STUB_SHA256, } from "@bli-cockpit/telemetry-core";
|
|
31
|
-
import { sanitizeTextEvidenceForUpload } from "../adapters/raw-evidence-sanitize.js";
|
|
39
|
+
import { isWholeFileMask, sanitizeTextEvidenceForUpload } from "../adapters/raw-evidence-sanitize.js";
|
|
32
40
|
import { isSecretLikePath } from "../adapters/raw-evidence-keys.js";
|
|
33
41
|
import { IMAGE_HANDLING_MODE } from "@bli-cockpit/telemetry-core";
|
|
34
42
|
import { uploadFormOfTranscript } from "../image-description-session.js";
|
|
@@ -47,7 +55,15 @@ export async function judgeSource(input) {
|
|
|
47
55
|
reason,
|
|
48
56
|
remote_byte_size: best,
|
|
49
57
|
});
|
|
50
|
-
|
|
58
|
+
const split = input.facts.slimSplit ? await input.facts.slimSplit() : null;
|
|
59
|
+
if (split)
|
|
60
|
+
return judgeWithSlimCopy({ file, cursorEntry, facts: input.facts, held, best, split });
|
|
61
|
+
// A slim-aware machine with a null split may have lost its described-image list: the server copy
|
|
62
|
+
// can be slim while this side sees nothing to slim. The cursor cannot tell, so judge only against
|
|
63
|
+
// the whole form below (a slim copy then reads stored_partial/remote_smaller: keep the file).
|
|
64
|
+
const trustCursor = input.facts.slimAware !== true;
|
|
65
|
+
if (trustCursor &&
|
|
66
|
+
cursorEntry &&
|
|
51
67
|
!cursorEntry.stubbed &&
|
|
52
68
|
cursorEntry.uploaded_object_key &&
|
|
53
69
|
held.some((remote) => remote.object_key === cursorEntry.uploaded_object_key) &&
|
|
@@ -56,7 +72,12 @@ export async function judgeSource(input) {
|
|
|
56
72
|
cursorEntry.file_hash_sha256 === (await input.facts.rawSha256())) {
|
|
57
73
|
return full("unchanged_since_upload");
|
|
58
74
|
}
|
|
59
|
-
|
|
75
|
+
if (!trustCursor) {
|
|
76
|
+
console.error("[sources-verdict] slim-aware machine, no slim split: cursor shortcut skipped", JSON.stringify({ reason: "slim_aware_split_null", has_cursor: cursorEntry !== null }));
|
|
77
|
+
}
|
|
78
|
+
// Slim-aware with no split: only the WHOLE form is evidence of a whole server copy. When it
|
|
79
|
+
// stubs (null) the raw size below answers, so a slim-only copy is never stored_full.
|
|
80
|
+
const sanitized = !trustCursor && input.facts.wholeForm ? await input.facts.wholeForm() : await input.facts.sanitized();
|
|
60
81
|
if (sanitized) {
|
|
61
82
|
if (held.some((remote) => remote.content_hash_sha256 === sanitized.sha256))
|
|
62
83
|
return full("content_hash_match");
|
|
@@ -76,16 +97,91 @@ export async function judgeSource(input) {
|
|
|
76
97
|
cursorEntry.uploaded_byte_size < file.byte_size;
|
|
77
98
|
return partial(grewAfterUpload ? "changed_since_upload" : "remote_smaller");
|
|
78
99
|
}
|
|
100
|
+
/**
|
|
101
|
+
* BLI-17273. The file has two redacted forms (whole and slim) and this machine has run slim, so a
|
|
102
|
+
* copy Tower holds may be either. The rule that keeps local image bytes safe: only evidence that the
|
|
103
|
+
* WHOLE form is on the server makes `stored_full`; a copy that is the slim form, or cannot be told
|
|
104
|
+
* apart from it, is `stored_slim` (keep the file).
|
|
105
|
+
*
|
|
106
|
+
* hash equals the whole form stored_full content_hash_match
|
|
107
|
+
* hash equals the slim form stored_slim content_hash_match
|
|
108
|
+
* largest copy is at least the whole form's size stored_full remote_at_least_sanitized_size
|
|
109
|
+
* cursor says its upload is current, file unchanged stored_slim unchanged_since_upload
|
|
110
|
+
* (the upload may be either form and its hash matches neither, so it is not proven whole)
|
|
111
|
+
* largest copy is at least the slim form's size stored_slim remote_at_least_slim_size
|
|
112
|
+
* size never recorded stored_partial remote_size_unknown
|
|
113
|
+
* otherwise stored_partial changed_since_upload / remote_smaller
|
|
114
|
+
*
|
|
115
|
+
* A stale whole copy cannot outrank the slim size: a copy smaller than the whole form is never
|
|
116
|
+
* `stored_full` here, however it compares to the slim form.
|
|
117
|
+
*/
|
|
118
|
+
async function judgeWithSlimCopy(input) {
|
|
119
|
+
const { file, cursorEntry, held, best, split } = input;
|
|
120
|
+
const result = (verdict, reason) => ({
|
|
121
|
+
verdict,
|
|
122
|
+
reason,
|
|
123
|
+
remote_byte_size: best,
|
|
124
|
+
});
|
|
125
|
+
if (held.some((remote) => remote.content_hash_sha256 === split.full.sha256))
|
|
126
|
+
return result("stored_full", "content_hash_match");
|
|
127
|
+
if (held.some((remote) => remote.content_hash_sha256 === split.slim.sha256))
|
|
128
|
+
return result("stored_slim", "content_hash_match");
|
|
129
|
+
if (best !== null && best >= split.full.byte_size)
|
|
130
|
+
return result("stored_full", "remote_at_least_sanitized_size");
|
|
131
|
+
if (cursorEntry &&
|
|
132
|
+
!cursorEntry.stubbed &&
|
|
133
|
+
cursorEntry.uploaded_object_key &&
|
|
134
|
+
held.some((remote) => remote.object_key === cursorEntry.uploaded_object_key) &&
|
|
135
|
+
cursorEntry.uploaded_byte_size === file.byte_size &&
|
|
136
|
+
cursorEntry.file_hash_sha256 &&
|
|
137
|
+
cursorEntry.file_hash_sha256 === (await input.facts.rawSha256())) {
|
|
138
|
+
return result("stored_slim", "unchanged_since_upload");
|
|
139
|
+
}
|
|
140
|
+
if (best === null)
|
|
141
|
+
return result("stored_partial", "remote_size_unknown");
|
|
142
|
+
if (best >= split.slim.byte_size)
|
|
143
|
+
return result("stored_slim", "remote_at_least_slim_size");
|
|
144
|
+
const grewAfterUpload = cursorEntry?.uploaded_object_key != null &&
|
|
145
|
+
cursorEntry.uploaded_byte_size != null &&
|
|
146
|
+
cursorEntry.uploaded_byte_size < file.byte_size;
|
|
147
|
+
return result("stored_partial", grewAfterUpload ? "changed_since_upload" : "remote_smaller");
|
|
148
|
+
}
|
|
79
149
|
/**
|
|
80
150
|
* Reads the file at most once, on first ask, and redacts it exactly as the
|
|
81
151
|
* upload does (`raw-evidence-transcript-file.ts`): same sanitizer, same
|
|
82
152
|
* secret-like-name rule, same byte cap.
|
|
83
153
|
*/
|
|
84
154
|
export function fileFacts(file, maxBytes = RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES,
|
|
85
|
-
/**
|
|
155
|
+
/**
|
|
156
|
+
* `mode` is this machine's mode (`image-mode-state.ts`; absent means the fleet default
|
|
157
|
+
* `IMAGE_HANDLING_MODE`). `describedKeys` is the collector's described-image state. `slimAware`
|
|
158
|
+
* (BLI-17273) is true for a machine that runs or ever ran slim: its server copies may be slim, so
|
|
159
|
+
* `slimSplit()` answers; absent it defaults to "the mode is drop".
|
|
160
|
+
*/
|
|
86
161
|
upload = {}) {
|
|
87
162
|
const mode = upload.mode ?? IMAGE_HANDLING_MODE;
|
|
88
163
|
const describedKeys = upload.describedKeys ?? new Set();
|
|
164
|
+
const slimAware = upload.slimAware ?? mode === "describe_and_drop";
|
|
165
|
+
const redactForUpload = (text, original) => {
|
|
166
|
+
const result = sanitizeTextEvidenceForUpload({
|
|
167
|
+
text,
|
|
168
|
+
originalBytes: text === original.toString("utf8") ? original : Buffer.from(text, "utf8"),
|
|
169
|
+
redactedFields: [`${file.kind === "codex_rollout" ? "codex_jsonl" : file.kind === "claude_main" ? "claude_jsonl" : "claude_jsonl_sidecar"}.body`],
|
|
170
|
+
secretLikeFileName: isSecretLikePath(path.basename(file.file_path)),
|
|
171
|
+
});
|
|
172
|
+
const hash = sha256(result.bytes);
|
|
173
|
+
if (hash === REDACTION_STUB_SHA256)
|
|
174
|
+
return null;
|
|
175
|
+
// A whole-file mask is one 40-byte line, not the content: every copy Tower holds would look
|
|
176
|
+
// "at least as large" and read stored_full. No redacted form, so the raw size answers instead.
|
|
177
|
+
if (isWholeFileMask(result.bytes)) {
|
|
178
|
+
console.error("[sources-verdict] redacted form is the whole-file mask: judged by raw size", JSON.stringify({ reason: "whole_file_mask", source: file.source, file_bytes: file.byte_size }));
|
|
179
|
+
return null;
|
|
180
|
+
}
|
|
181
|
+
return { sha256: hash, byte_size: result.bytes.byteLength };
|
|
182
|
+
};
|
|
183
|
+
let split = null;
|
|
184
|
+
let whole = null;
|
|
89
185
|
let bytes = null;
|
|
90
186
|
const read = () => {
|
|
91
187
|
bytes ??= file.byte_size > maxBytes ? Promise.resolve(null) : fs.readFile(file.file_path);
|
|
@@ -94,6 +190,7 @@ upload = {}) {
|
|
|
94
190
|
let rawHash = null;
|
|
95
191
|
let sanitized = null;
|
|
96
192
|
return {
|
|
193
|
+
slimAware,
|
|
97
194
|
rawSha256() {
|
|
98
195
|
rawHash ??= read().then(async (buffer) => (buffer ? sha256(buffer) : await streamSha256(file.file_path)));
|
|
99
196
|
return rawHash;
|
|
@@ -104,19 +201,33 @@ upload = {}) {
|
|
|
104
201
|
return null;
|
|
105
202
|
// Compare against what the uploader sends: in drop mode that is the slim form, not the disk bytes.
|
|
106
203
|
const uploadText = uploadFormOfTranscript(buffer.toString("utf8"), file.source === "codex" ? "codex" : "claude_code", mode, describedKeys);
|
|
107
|
-
|
|
108
|
-
text: uploadText,
|
|
109
|
-
originalBytes: uploadText === buffer.toString("utf8") ? buffer : Buffer.from(uploadText, "utf8"),
|
|
110
|
-
redactedFields: [`${file.kind === "codex_rollout" ? "codex_jsonl" : file.kind === "claude_main" ? "claude_jsonl" : "claude_jsonl_sidecar"}.body`],
|
|
111
|
-
secretLikeFileName: isSecretLikePath(path.basename(file.file_path)),
|
|
112
|
-
});
|
|
113
|
-
const hash = sha256(result.bytes);
|
|
114
|
-
if (hash === REDACTION_STUB_SHA256)
|
|
115
|
-
return null;
|
|
116
|
-
return { sha256: hash, byte_size: result.bytes.byteLength };
|
|
204
|
+
return redactForUpload(uploadText, buffer);
|
|
117
205
|
});
|
|
118
206
|
return sanitized;
|
|
119
207
|
},
|
|
208
|
+
wholeForm() {
|
|
209
|
+
whole ??= read().then((buffer) => (buffer ? redactForUpload(buffer.toString("utf8"), buffer) : null));
|
|
210
|
+
return whole;
|
|
211
|
+
},
|
|
212
|
+
async slimSplit() {
|
|
213
|
+
if (!slimAware)
|
|
214
|
+
return null;
|
|
215
|
+
split ??= read().then((buffer) => {
|
|
216
|
+
if (!buffer)
|
|
217
|
+
return null;
|
|
218
|
+
const text = buffer.toString("utf8");
|
|
219
|
+
const slimText = uploadFormOfTranscript(text, file.source === "codex" ? "codex" : "claude_code", "describe_and_drop", describedKeys);
|
|
220
|
+
// Slimming changes nothing: one form, and `sanitized()` is the answer.
|
|
221
|
+
if (slimText === text)
|
|
222
|
+
return null;
|
|
223
|
+
const whole = redactForUpload(text, buffer);
|
|
224
|
+
const slim = redactForUpload(slimText, buffer);
|
|
225
|
+
if (!whole || !slim || whole.sha256 === slim.sha256)
|
|
226
|
+
return null;
|
|
227
|
+
return { full: whole, slim };
|
|
228
|
+
});
|
|
229
|
+
return split;
|
|
230
|
+
},
|
|
120
231
|
};
|
|
121
232
|
}
|
|
122
233
|
function sha256(buffer) {
|
|
@@ -63,7 +63,7 @@ export async function reportNoWorktreeSync(command, io, run, dedup) {
|
|
|
63
63
|
if (run.notice?.startsWith("nothing_in_root")) {
|
|
64
64
|
// Says out loud what the receipt now says to the dashboard: the sessions
|
|
65
65
|
// this machine ran were all outside the folders it is allowed to look at.
|
|
66
|
-
writeLine(io.stdout, `Every session seen this run was outside your approved folders (${run.notice}).
|
|
66
|
+
writeLine(io.stdout, `Every session seen this run was outside your approved folders (${run.notice}). No transcript was uploaded for them this run (the log line "outside-root raw upload" says why), and nothing is broken.`);
|
|
67
67
|
}
|
|
68
68
|
writeAgentSessionSummary(io, run.summary);
|
|
69
69
|
if (gc && !gc.skipped)
|
package/dist/commands/sync.js
CHANGED
|
@@ -45,6 +45,7 @@ import { sendSyncHeartbeat } from "./sync-heartbeat.js";
|
|
|
45
45
|
import { runSyncWithHealthReceipt } from "./sync-receipt.js";
|
|
46
46
|
import { runAutostartSelfHealAfterSync, runMemoryInstallAfterSync, runScheduledSelfUpdateAfterSync, runStagingPruneAfterSync, } from "./sync-followups.js";
|
|
47
47
|
import { rotateCollectorLogsBestEffort } from "../log-rotation.js";
|
|
48
|
+
import { settleTickImageModeBestEffort } from "../image-mode-state.js";
|
|
48
49
|
import { stampSyncTick } from "../sync-tick-stamp.js";
|
|
49
50
|
import { startTickTimer } from "../sync-tick-timing.js";
|
|
50
51
|
import { DEFAULT_DASHBOARD_URL, getCollectorRuntimePaths, readLocalCollectorConfig, } from "../local-state.js";
|
|
@@ -137,14 +138,20 @@ async function tickDashboardUrl(command, paths) {
|
|
|
137
138
|
* one to fall back on (BLI-2678).
|
|
138
139
|
*/
|
|
139
140
|
async function reportTickStarted(command, io, dashboardUrl) {
|
|
140
|
-
|
|
141
|
+
const observed = {};
|
|
142
|
+
const minCliVersion = await reportInstallEventsBestEffort({
|
|
141
143
|
homeDir: command.homeDir,
|
|
142
144
|
dashboardUrl,
|
|
143
145
|
command: "sync",
|
|
144
146
|
events: [{ step: "sync_started", status: "ok" }],
|
|
145
147
|
json: command.json,
|
|
146
148
|
io,
|
|
149
|
+
observed,
|
|
147
150
|
});
|
|
151
|
+
// BLI-17273: the reply also names this machine's image-handling mode. Put it on file BEFORE
|
|
152
|
+
// collection starts (the uploader reads the file); an offline tick keeps the last value on file.
|
|
153
|
+
await settleTickImageModeBestEffort(getCollectorRuntimePaths(command.homeDir).state_dir, observed.imageHandlingMode);
|
|
154
|
+
return minCliVersion;
|
|
148
155
|
}
|
|
149
156
|
/** Post the receipt the run earned, whatever it says. */
|
|
150
157
|
async function reportTickOutcome(command, io, dashboardUrl, completion) {
|
|
@@ -9,7 +9,8 @@
|
|
|
9
9
|
* the describe door, a few at a time, inside a per-sync count and time budget. The state file
|
|
10
10
|
* remembers what was sent, so a screenshot that appears in 40 turns, or in 3 sessions, is sent
|
|
11
11
|
* once. A failure never fails the transcript.
|
|
12
|
-
* 3. REPLACE, only when
|
|
12
|
+
* 3. REPLACE, only when this machine's mode (set per machine from Tower, kept on file by
|
|
13
|
+
* `image-mode-state.ts`; the fleet default is `IMAGE_HANDLING_MODE` in telemetry-core) says
|
|
13
14
|
* `describe_and_drop`: every image whose description is stored becomes its short marker. An
|
|
14
15
|
* image with no stored description is KEPT. In `describe_and_keep` (the default) the text
|
|
15
16
|
* comes back byte for byte as it went in.
|
|
@@ -22,6 +23,7 @@
|
|
|
22
23
|
*/
|
|
23
24
|
import { IMAGE_DESCRIBE_MAX_BYTES, IMAGE_HANDLING_MODE, base64Of, findEmbeddedImages, replaceImagesWithMarkers, } from "@bli-cockpit/telemetry-core";
|
|
24
25
|
import { postImageToDescribeDoor } from "./image-describe-client.js";
|
|
26
|
+
import { resolveImageHandlingMode } from "./image-mode-state.js";
|
|
25
27
|
import { slimOrKeep, slimTranscript } from "./slim/slim-transcript.js";
|
|
26
28
|
import { imageStateKey, imageRetryDelayMs, readDescribedImageKeys, readFailedImages, writeImageDescriptionState, } from "./image-description-state.js";
|
|
27
29
|
/** New images sent to the door in one sync (all worktrees together). */
|
|
@@ -204,11 +206,15 @@ export async function openImageStepForSync(options) {
|
|
|
204
206
|
if (options.budget?.images)
|
|
205
207
|
return options.budget.images;
|
|
206
208
|
try {
|
|
209
|
+
// BLI-17273: the mode is PER MACHINE: the last value Tower sent (kept on file so an offline
|
|
210
|
+
// tick does not flip a slim machine back), else the fleet default `IMAGE_HANDLING_MODE`.
|
|
211
|
+
const mode = options.mode ?? (await resolveImageHandlingMode(options.stateDir)).mode;
|
|
207
212
|
const session = await openImageDescriptions({
|
|
208
213
|
stateDir: options.stateDir,
|
|
209
214
|
dashboardUrl: options.dashboardUrl,
|
|
210
215
|
deviceToken: options.deviceToken,
|
|
211
216
|
fetchImpl: options.fetchImpl,
|
|
217
|
+
mode,
|
|
212
218
|
});
|
|
213
219
|
if (options.budget)
|
|
214
220
|
options.budget.images = session;
|
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
*/
|
|
10
10
|
import path from "node:path";
|
|
11
11
|
import { isMissingFileFailure } from "./health-detail.js";
|
|
12
|
-
import { readJsonFile,
|
|
12
|
+
import { readJsonFile, writeJsonFileAtomic } from "./local-state-files.js";
|
|
13
13
|
export const IMAGE_DESCRIPTION_STATE_FILENAME = "image-descriptions.json";
|
|
14
14
|
/** Oldest keys leave first past this many (about 5 MB on disk). */
|
|
15
15
|
export const IMAGE_DESCRIPTION_STATE_MAX_KEYS = 150_000;
|
|
@@ -76,7 +76,7 @@ export async function writeImageDescriptionState(stateDir, state) {
|
|
|
76
76
|
const kept = keys.length > IMAGE_DESCRIPTION_STATE_MAX_KEYS ? keys.slice(keys.length - IMAGE_DESCRIPTION_STATE_MAX_KEYS) : keys;
|
|
77
77
|
const failedEntries = Object.entries(state.failed).sort((a, b) => a[1].retry_after - b[1].retry_after);
|
|
78
78
|
const failed = Object.fromEntries(failedEntries.slice(Math.max(0, failedEntries.length - IMAGE_FAILED_STATE_MAX_KEYS)));
|
|
79
|
-
await
|
|
79
|
+
await writeJsonFileAtomic(imageDescriptionStatePath(stateDir), {
|
|
80
80
|
schema_version: "cockpit-image-descriptions.v1",
|
|
81
81
|
described: kept,
|
|
82
82
|
failed,
|