@bli-cockpit/cli 0.2.205 → 0.2.207
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/raw-evidence-transcript-file.js +9 -5
- package/dist/commands/public-root.js +1 -1
- package/dist/commands/sources-check.js +3 -1
- package/dist/commands/sources-verdict.js +11 -3
- package/dist/image-description-session.js +96 -72
- package/dist/slim/claude-attachments.js +96 -0
- package/dist/slim/claude-rules.js +48 -0
- package/dist/slim/claude-tool-results.js +178 -0
- package/dist/slim/codex-rules.js +179 -0
- package/dist/slim/slim-support.js +90 -0
- package/dist/slim/slim-transcript.js +133 -0
- package/package.json +4 -4
|
@@ -106,13 +106,17 @@ export async function collectOneEvidenceFile(collection, options) {
|
|
|
106
106
|
}
|
|
107
107
|
/**
|
|
108
108
|
* The transcript bytes after the image step. Unchanged (the very same buffer) unless this sync has
|
|
109
|
-
* an image session AND the switch is `describe_and_drop`
|
|
110
|
-
*
|
|
109
|
+
* an image session AND the switch is `describe_and_drop` (images become markers where a description
|
|
110
|
+
* is stored, then the slim rules run, see src/slim/). In `describe_and_keep` the bytes always come
|
|
111
|
+
* back untouched. Never throws: an image step that breaks hands back the original bytes and says so.
|
|
111
112
|
*/
|
|
112
113
|
async function prepareEmbeddedImages(collection, file) {
|
|
113
114
|
const session = collection.budget.images;
|
|
114
115
|
const source = file.kind.startsWith("codex") ? "codex" : file.kind.startsWith("claude") ? "claude_code" : null;
|
|
115
|
-
if (!session || !source
|
|
116
|
+
if (!session || !source)
|
|
117
|
+
return file.bytes;
|
|
118
|
+
// Drop mode slims every transcript, image-bearing or not; keep mode only describes the ones that can hold an image.
|
|
119
|
+
if (session.mode !== "describe_and_drop" && !mightHoldEmbeddedImages(file.bytes))
|
|
116
120
|
return file.bytes;
|
|
117
121
|
try {
|
|
118
122
|
const prepared = await session.prepareTranscript({
|
|
@@ -120,12 +124,12 @@ async function prepareEmbeddedImages(collection, file) {
|
|
|
120
124
|
source,
|
|
121
125
|
sessionId: file.sessionId,
|
|
122
126
|
});
|
|
123
|
-
if (prepared.
|
|
127
|
+
if (!prepared.changed)
|
|
124
128
|
return file.bytes;
|
|
125
129
|
return Buffer.from(prepared.text, "utf8");
|
|
126
130
|
}
|
|
127
131
|
catch (error) {
|
|
128
|
-
console.error("[raw-evidence] image step failed, uploading the transcript
|
|
132
|
+
console.error("[raw-evidence] image or slim step failed, uploading the transcript as it was", JSON.stringify({ reason: "image_step_failed", kind: file.kind, ...describeError(error) }));
|
|
129
133
|
return file.bytes;
|
|
130
134
|
}
|
|
131
135
|
}
|
|
@@ -21,7 +21,7 @@ export async function runCockpitCli(argv, io) {
|
|
|
21
21
|
}
|
|
22
22
|
|
|
23
23
|
if (command === "--version" || command === "-V" || command === "version") {
|
|
24
|
-
writeLine(io?.stdout ?? process.stdout, "0.2.
|
|
24
|
+
writeLine(io?.stdout ?? process.stdout, "0.2.207");
|
|
25
25
|
return 0;
|
|
26
26
|
}
|
|
27
27
|
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { writeLine } from "./cli-io.js";
|
|
2
|
+
import { readDescribedImageKeys } from "../image-description-state.js";
|
|
2
3
|
import { CLAUDE_CURSOR_FILENAME, readRawEvidenceCursor, } from "../cursors/raw-evidence-cursor.js";
|
|
3
4
|
import { fetchTranscriptObjects } from "../evidence-transcripts-client.js";
|
|
4
5
|
import { DEFAULT_DASHBOARD_URL, getCollectorRuntimePaths, readLocalCollectorConfig, readLocalCollectorSessionFile, } from "../local-state.js";
|
|
@@ -28,13 +29,14 @@ export async function runSourcesCheck(command, io, now = new Date()) {
|
|
|
28
29
|
codex: await readRawEvidenceCursor(paths),
|
|
29
30
|
claude_code: await readRawEvidenceCursor(paths, { filename: CLAUDE_CURSOR_FILENAME, sessionsOnly: true }),
|
|
30
31
|
};
|
|
32
|
+
const describedKeys = new Set(await readDescribedImageKeys(paths.state_dir));
|
|
31
33
|
const report = emptyReport(command, remote, local.newer_count, local.file_read_failed + local.directory_read_failed);
|
|
32
34
|
for (const file of local.files) {
|
|
33
35
|
const judged = await judgeSource({
|
|
34
36
|
file,
|
|
35
37
|
remotes: remote.files.get(file.key) ?? [],
|
|
36
38
|
cursorEntry: file.kind === "claude_subagent" ? null : (cursors[file.source].sessions[file.session_id] ?? null),
|
|
37
|
-
facts: fileFacts(file),
|
|
39
|
+
facts: fileFacts(file, undefined, { describedKeys }),
|
|
38
40
|
}).catch((error) => {
|
|
39
41
|
report.local.unreadable += 1;
|
|
40
42
|
console.error("[collector sources] a transcript could not be judged; left out of the answer", JSON.stringify({ reason: "file_unreadable", kind: file.kind, byte_size: file.byte_size, error_name: error?.name ?? "Error" }));
|
|
@@ -30,6 +30,8 @@ import path from "node:path";
|
|
|
30
30
|
import { RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES, REDACTION_STUB_SHA256, } from "@bli-cockpit/telemetry-core";
|
|
31
31
|
import { sanitizeTextEvidenceForUpload } from "../adapters/raw-evidence-sanitize.js";
|
|
32
32
|
import { isSecretLikePath } from "../adapters/raw-evidence-keys.js";
|
|
33
|
+
import { IMAGE_HANDLING_MODE } from "@bli-cockpit/telemetry-core";
|
|
34
|
+
import { uploadFormOfTranscript } from "../image-description-session.js";
|
|
33
35
|
export async function judgeSource(input) {
|
|
34
36
|
const { file, remotes, cursorEntry } = input;
|
|
35
37
|
if (remotes.length === 0)
|
|
@@ -79,7 +81,11 @@ export async function judgeSource(input) {
|
|
|
79
81
|
* upload does (`raw-evidence-transcript-file.ts`): same sanitizer, same
|
|
80
82
|
* secret-like-name rule, same byte cap.
|
|
81
83
|
*/
|
|
82
|
-
export function fileFacts(file, maxBytes = RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES
|
|
84
|
+
export function fileFacts(file, maxBytes = RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES,
|
|
85
|
+
/** Tests only set `mode`. Production reads the one switch. `describedKeys` is the collector's described-image state. */
|
|
86
|
+
upload = {}) {
|
|
87
|
+
const mode = upload.mode ?? IMAGE_HANDLING_MODE;
|
|
88
|
+
const describedKeys = upload.describedKeys ?? new Set();
|
|
83
89
|
let bytes = null;
|
|
84
90
|
const read = () => {
|
|
85
91
|
bytes ??= file.byte_size > maxBytes ? Promise.resolve(null) : fs.readFile(file.file_path);
|
|
@@ -96,9 +102,11 @@ export function fileFacts(file, maxBytes = RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES) {
|
|
|
96
102
|
sanitized ??= read().then((buffer) => {
|
|
97
103
|
if (!buffer)
|
|
98
104
|
return null;
|
|
105
|
+
// Compare against what the uploader sends: in drop mode that is the slim form, not the disk bytes.
|
|
106
|
+
const uploadText = uploadFormOfTranscript(buffer.toString("utf8"), file.source === "codex" ? "codex" : "claude_code", mode, describedKeys);
|
|
99
107
|
const result = sanitizeTextEvidenceForUpload({
|
|
100
|
-
text:
|
|
101
|
-
originalBytes: buffer,
|
|
108
|
+
text: uploadText,
|
|
109
|
+
originalBytes: uploadText === buffer.toString("utf8") ? buffer : Buffer.from(uploadText, "utf8"),
|
|
102
110
|
redactedFields: [`${file.kind === "codex_rollout" ? "codex_jsonl" : file.kind === "claude_main" ? "claude_jsonl" : "claude_jsonl_sidecar"}.body`],
|
|
103
111
|
secretLikeFileName: isSecretLikePath(path.basename(file.file_path)),
|
|
104
112
|
});
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* One sync's handling of the images inside transcripts (BLI-16977).
|
|
3
3
|
*
|
|
4
|
-
* `prepareTranscript` takes a transcript's text just before it is masked and staged and does
|
|
4
|
+
* `prepareTranscript` takes a transcript's text just before it is masked and staged and does four
|
|
5
5
|
* things, in this order:
|
|
6
6
|
*
|
|
7
7
|
* 1. FIND every embedded image (`transcript-images.ts` in telemetry-core) and group them by sha256.
|
|
@@ -13,12 +13,16 @@
|
|
|
13
13
|
* `describe_and_drop`: every image whose description is stored becomes its short marker. An
|
|
14
14
|
* image with no stored description is KEPT. In `describe_and_keep` (the default) the text
|
|
15
15
|
* comes back byte for byte as it went in.
|
|
16
|
+
* 4. SLIM, only in `describe_and_drop`: the slim transcript (src/slim/): the duplicate copies and
|
|
17
|
+
* the bookkeeping go, every line stays. In `describe_and_keep` the slim form is not computed
|
|
18
|
+
* and not uploaded.
|
|
16
19
|
*
|
|
17
20
|
* Metadata only in the logs: counts, sizes, hashes, reason labels. Never image bytes, never a
|
|
18
21
|
* description.
|
|
19
22
|
*/
|
|
20
23
|
import { IMAGE_DESCRIBE_MAX_BYTES, IMAGE_HANDLING_MODE, base64Of, findEmbeddedImages, replaceImagesWithMarkers, } from "@bli-cockpit/telemetry-core";
|
|
21
24
|
import { postImageToDescribeDoor } from "./image-describe-client.js";
|
|
25
|
+
import { slimOrKeep, slimTranscript } from "./slim/slim-transcript.js";
|
|
22
26
|
import { imageStateKey, readDescribedImageKeys, writeDescribedImageKeys, } from "./image-description-state.js";
|
|
23
27
|
/** New images sent to the door in one sync (all worktrees together). */
|
|
24
28
|
export const IMAGE_DESCRIBE_MAX_PER_RUN = 100;
|
|
@@ -62,84 +66,104 @@ export async function openImageDescriptions(options) {
|
|
|
62
66
|
}
|
|
63
67
|
settledThisRun.set(image.sha256, `failed:${outcome.reason}`);
|
|
64
68
|
}
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
69
|
+
async function describeAndReplace(input) {
|
|
70
|
+
const images = findEmbeddedImages(input.text);
|
|
71
|
+
if (images.length === 0)
|
|
72
|
+
return { text: input.text, imageCount: 0, uniqueCount: 0, replacedCount: 0, charsSaved: 0 };
|
|
73
|
+
const unique = new Map();
|
|
74
|
+
for (const image of images)
|
|
75
|
+
if (!unique.has(image.sha256))
|
|
76
|
+
unique.set(image.sha256, image);
|
|
77
|
+
const counts = { already: 0, described_now: 0, known: 0, too_large: 0, over_budget: 0, failed: 0 };
|
|
78
|
+
const failedReasons = {};
|
|
79
|
+
const toSend = [];
|
|
80
|
+
for (const image of unique.values()) {
|
|
81
|
+
if (described.has(imageStateKey(image.sha256))) {
|
|
82
|
+
counts.already += 1;
|
|
83
|
+
continue;
|
|
84
|
+
}
|
|
85
|
+
const earlier = settledThisRun.get(image.sha256);
|
|
86
|
+
if (earlier)
|
|
87
|
+
continue;
|
|
88
|
+
if (image.byteSize > IMAGE_DESCRIBE_MAX_BYTES) {
|
|
89
|
+
settledThisRun.set(image.sha256, "too_large");
|
|
90
|
+
counts.too_large += 1;
|
|
91
|
+
continue;
|
|
92
|
+
}
|
|
93
|
+
toSend.push(image);
|
|
94
|
+
}
|
|
95
|
+
let next = 0;
|
|
96
|
+
const worker = async () => {
|
|
97
|
+
while (next < toSend.length) {
|
|
98
|
+
const image = toSend[next++];
|
|
99
|
+
if (sentThisRun >= maxPerRun || now() - startedAt > maxRunMs) {
|
|
100
|
+
counts.over_budget += 1;
|
|
89
101
|
continue;
|
|
90
102
|
}
|
|
91
|
-
|
|
103
|
+
await sendOne(image, input.text, input);
|
|
92
104
|
}
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
for (const image of toSend) {
|
|
106
|
-
const settled = settledThisRun.get(image.sha256);
|
|
107
|
-
if (settled === "described")
|
|
108
|
-
counts.described_now += 1;
|
|
109
|
-
else if (settled === "known")
|
|
110
|
-
counts.known += 1;
|
|
111
|
-
else if (settled?.startsWith("failed:")) {
|
|
112
|
-
counts.failed += 1;
|
|
113
|
-
const reason = settled.slice("failed:".length);
|
|
114
|
-
failedReasons[reason] = (failedReasons[reason] ?? 0) + 1;
|
|
115
|
-
}
|
|
105
|
+
};
|
|
106
|
+
await Promise.all(Array.from({ length: Math.min(CONCURRENCY, toSend.length) }, worker));
|
|
107
|
+
for (const image of toSend) {
|
|
108
|
+
const settled = settledThisRun.get(image.sha256);
|
|
109
|
+
if (settled === "described")
|
|
110
|
+
counts.described_now += 1;
|
|
111
|
+
else if (settled === "known")
|
|
112
|
+
counts.known += 1;
|
|
113
|
+
else if (settled?.startsWith("failed:")) {
|
|
114
|
+
counts.failed += 1;
|
|
115
|
+
const reason = settled.slice("failed:".length);
|
|
116
|
+
failedReasons[reason] = (failedReasons[reason] ?? 0) + 1;
|
|
116
117
|
}
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
118
|
+
}
|
|
119
|
+
if (counts.described_now + counts.known > 0)
|
|
120
|
+
await writeDescribedImageKeys(options.stateDir, orderedKeys);
|
|
121
|
+
const replaced = mode === "describe_and_drop"
|
|
122
|
+
? replaceImagesWithMarkers(input.text, images, (image) => described.has(imageStateKey(image.sha256)))
|
|
123
|
+
: { text: input.text, replacedCount: 0, charsSaved: 0 };
|
|
124
|
+
console.error(TAG + " transcript prepared", JSON.stringify({
|
|
125
|
+
mode,
|
|
126
|
+
source: input.source,
|
|
127
|
+
images: images.length,
|
|
128
|
+
unique: unique.size,
|
|
129
|
+
...counts,
|
|
130
|
+
failed_reasons: failedReasons,
|
|
131
|
+
replaced: replaced.replacedCount,
|
|
132
|
+
chars_saved: replaced.charsSaved,
|
|
133
|
+
kept_without_description: mode === "describe_and_drop" ? images.length - replaced.replacedCount : 0,
|
|
134
|
+
}));
|
|
135
|
+
return {
|
|
136
|
+
text: replaced.text,
|
|
137
|
+
imageCount: images.length,
|
|
138
|
+
uniqueCount: unique.size,
|
|
139
|
+
replacedCount: replaced.replacedCount,
|
|
140
|
+
charsSaved: replaced.charsSaved,
|
|
141
|
+
};
|
|
142
|
+
}
|
|
143
|
+
return {
|
|
144
|
+
mode,
|
|
145
|
+
async prepareTranscript(input) {
|
|
146
|
+
const afterImages = await describeAndReplace(input);
|
|
147
|
+
if (mode !== "describe_and_drop")
|
|
148
|
+
return { ...afterImages, text: input.text, changed: false, slim: null };
|
|
149
|
+
const slim = slimOrKeep(afterImages.text, input.source, { sessionId: input.sessionId });
|
|
150
|
+
return { ...afterImages, text: slim.text, changed: slim.text !== input.text, slim: slim.stats };
|
|
140
151
|
},
|
|
141
152
|
};
|
|
142
153
|
}
|
|
154
|
+
/**
|
|
155
|
+
* What the uploader sends for this transcript text, worked out offline (BLI-17013). Keep mode: the
|
|
156
|
+
* text as it is. Drop mode: every image whose key is in `describedKeys` becomes its marker, then
|
|
157
|
+
* the slim rules run, the same two steps as `prepareTranscript` above, with no describe-door call.
|
|
158
|
+
* `cockpit sources check` uses it so a complete upload is judged against the bytes actually uploaded.
|
|
159
|
+
*/
|
|
160
|
+
export function uploadFormOfTranscript(text, source, mode, describedKeys) {
|
|
161
|
+
if (mode !== "describe_and_drop")
|
|
162
|
+
return text;
|
|
163
|
+
const images = findEmbeddedImages(text);
|
|
164
|
+
const replaced = replaceImagesWithMarkers(text, images, (image) => describedKeys.has(imageStateKey(image.sha256)));
|
|
165
|
+
return slimTranscript(replaced.text, source).text;
|
|
166
|
+
}
|
|
143
167
|
/**
|
|
144
168
|
* The image step for one build of an envelope. When a sync-wide budget is shared across worktrees
|
|
145
169
|
* the session lives on it (one count budget and one state file for the whole sync); otherwise this
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Claude Code `attachment` lines: the harness's bookkeeping, re-saved every few turns (BLI-16977).
|
|
3
|
+
* On one laptop attachment lines are 30% of Claude transcript bytes and almost none of it is
|
|
4
|
+
* conversation. The line always stays (Tower counts lines and reads every timestamp); what goes is
|
|
5
|
+
* the payload that is a second copy, an exact repeat of an earlier block, or text nobody reads.
|
|
6
|
+
*
|
|
7
|
+
* Three kinds of change, by attachment type:
|
|
8
|
+
*
|
|
9
|
+
* - PAYLOAD dropped: `hook_success` (except SessionStart, whose output is context the model saw),
|
|
10
|
+
* `total_tokens_reminder`, `deferred_tools_record`.
|
|
11
|
+
* - REPEATS become a reference after the first copy: the system prompt and tool list
|
|
12
|
+
* (`prompt_snapshot`), CLAUDE.md files (`instructions`), the skill listing, session context,
|
|
13
|
+
* nested memory, MCP and agent listings, the deferred-tools delta's description lines, invoked
|
|
14
|
+
* skills, hook-injected context.
|
|
15
|
+
* - The `rendered` and `renderedInHumanTurn` copies (the same attachment as the model saw it)
|
|
16
|
+
* are dropped for every type measured to be a template around fields that stay.
|
|
17
|
+
*
|
|
18
|
+
* Kept whole: `queued_command` (what the person typed while the agent was busy, often found
|
|
19
|
+
* nowhere else), `edited_text_file` snippets, SessionStart hook output, `hook_additional_context`
|
|
20
|
+
* (once per distinct text), `deferred_tools_delta` names (added AND removed), and every attachment
|
|
21
|
+
* type this file does not name.
|
|
22
|
+
*/
|
|
23
|
+
import { isJsonObject, stripKey } from "./slim-support.js";
|
|
24
|
+
/** Fields dropped from the attachment, by type. */
|
|
25
|
+
const PAYLOAD_FIELDS_DROPPED = {
|
|
26
|
+
total_tokens_reminder: ["text"],
|
|
27
|
+
deferred_tools_record: ["entries", "toolInputCopies"],
|
|
28
|
+
};
|
|
29
|
+
/** Fields kept once per distinct text, by type. */
|
|
30
|
+
const REPEATED_FIELDS = {
|
|
31
|
+
prompt_snapshot: ["systemPrompt", "tools"],
|
|
32
|
+
skill_listing: ["content"],
|
|
33
|
+
session_context: ["context"],
|
|
34
|
+
nested_memory: ["content"],
|
|
35
|
+
mcp_instructions_delta: ["addedBlocks"],
|
|
36
|
+
agent_listing_delta: ["addedLines"],
|
|
37
|
+
deferred_tools_delta: ["addedLines", "surfacedDefinitions"],
|
|
38
|
+
invoked_skills: ["skills"],
|
|
39
|
+
hook_additional_context: ["content"],
|
|
40
|
+
};
|
|
41
|
+
/**
|
|
42
|
+
* Types whose `rendered` copy is a template around fields that stay, or around nothing that
|
|
43
|
+
* matters. `environment` is deliberately absent: its rendered text had 622 distinct versions in the
|
|
44
|
+
* survey and nobody proved they hold nothing else.
|
|
45
|
+
*/
|
|
46
|
+
const RENDERED_IS_A_COPY = new Set([
|
|
47
|
+
"hook_success", "hook_additional_context", "hook_system_message", "instructions", "skill_listing",
|
|
48
|
+
"queued_command", "total_tokens_reminder", "mcp_instructions_delta", "agent_listing_delta",
|
|
49
|
+
"deferred_tools_delta", "deferred_tools_record", "session_context", "nested_memory", "invoked_skills",
|
|
50
|
+
"edited_text_file", "file", "bash_output_audience_note", "remote_session_change", "model",
|
|
51
|
+
"batching_reminder_sent", "date", "auto_mode", "credential_org", "silent_turn_reminder",
|
|
52
|
+
"thinking_drop", "read_truncation_notice", "structured_output",
|
|
53
|
+
]);
|
|
54
|
+
export function slimAttachmentLine(record, seen, edits) {
|
|
55
|
+
const attachment = record["attachment"];
|
|
56
|
+
if (!isJsonObject(attachment))
|
|
57
|
+
return;
|
|
58
|
+
const type = typeof attachment["type"] === "string" ? attachment["type"] : "";
|
|
59
|
+
if (type === "hook_success")
|
|
60
|
+
dropHookOutput(attachment, edits);
|
|
61
|
+
for (const field of PAYLOAD_FIELDS_DROPPED[type] ?? [])
|
|
62
|
+
stripKey(attachment, field, `attachment.${type}.${field}`, edits);
|
|
63
|
+
for (const field of REPEATED_FIELDS[type] ?? [])
|
|
64
|
+
seen.dedupeKey(attachment, field, `attachment.${type}.repeat`, edits);
|
|
65
|
+
if (type === "instructions" && Array.isArray(attachment["files"])) {
|
|
66
|
+
for (const file of attachment["files"]) {
|
|
67
|
+
if (isJsonObject(file))
|
|
68
|
+
seen.dedupeKey(file, "content", "attachment.instructions.repeat", edits);
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
if (RENDERED_IS_A_COPY.has(type)) {
|
|
72
|
+
for (const copy of ["rendered", "renderedInHumanTurn"])
|
|
73
|
+
stripKey(record, copy, "attachment.rendered", edits);
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
/** A hook that ran fine and printed something: its output is bookkeeping unless it was the SessionStart hook. */
|
|
77
|
+
function dropHookOutput(attachment, edits) {
|
|
78
|
+
if (attachment["hookEvent"] === "SessionStart")
|
|
79
|
+
return;
|
|
80
|
+
const exitCode = attachment["exitCode"];
|
|
81
|
+
if (exitCode !== undefined && exitCode !== 0)
|
|
82
|
+
return;
|
|
83
|
+
for (const field of ["command", "content", "stderr", "stdout"]) {
|
|
84
|
+
stripKey(attachment, field, "attachment.hook_success.output", edits);
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
/** A `stop_hook_summary` system line with no error and no stop request: the per-hook list is bookkeeping. */
|
|
88
|
+
export function slimStopHookSummary(record, edits) {
|
|
89
|
+
if (record["subtype"] !== "stop_hook_summary")
|
|
90
|
+
return;
|
|
91
|
+
const errors = record["hookErrors"];
|
|
92
|
+
const hadErrors = Array.isArray(errors) ? errors.length > 0 : errors !== undefined && errors !== null && errors !== false;
|
|
93
|
+
if (hadErrors || record["preventedContinuation"] === true || record["stopReason"])
|
|
94
|
+
return;
|
|
95
|
+
stripKey(record, "hookInfos", "system.stop_hook_summary.hookInfos", edits);
|
|
96
|
+
}
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The slim rules for one Claude Code transcript line (BLI-16977). Entry point: `slimClaudeRecord`.
|
|
3
|
+
*
|
|
4
|
+
* The line's skeleton is never touched: type, timestamp, uuid, parentUuid, sessionId, cwd,
|
|
5
|
+
* gitBranch, isSidechain, requestId, version and the rest of the envelope stay, because Tower reads
|
|
6
|
+
* the timestamp and the working folder of every line. `message.usage` stays whole too (the 1-hour
|
|
7
|
+
* versus 5-minute cache-write split is what the cost is priced on). Only payload goes.
|
|
8
|
+
*
|
|
9
|
+
* What goes, and why it is safe (measured on 160 local files; the per-rule table is in
|
|
10
|
+
* docs/architecture/image-descriptions.md):
|
|
11
|
+
*
|
|
12
|
+
* - `wireToolInputs`: a second copy of each tool call's input, keyed by tool_use id.
|
|
13
|
+
* - `message.input_transformations`: a list of "thinking was dropped here" pointers, re-listed on
|
|
14
|
+
* every assistant line, holding no content.
|
|
15
|
+
* - `toolUseResult` fields that repeat the tool_result or the tool input (claude-tool-results.ts).
|
|
16
|
+
* - Attachment bookkeeping (claude-attachments.ts).
|
|
17
|
+
*
|
|
18
|
+
* Unknown shapes pass through untouched.
|
|
19
|
+
*/
|
|
20
|
+
import { slimAttachmentLine, slimStopHookSummary } from "./claude-attachments.js";
|
|
21
|
+
import { rememberToolInputs, slimToolUseResult, } from "./claude-tool-results.js";
|
|
22
|
+
import { isJsonObject, stripKey } from "./slim-support.js";
|
|
23
|
+
export function newClaudeSlimState() {
|
|
24
|
+
return { toolInputs: new Map() };
|
|
25
|
+
}
|
|
26
|
+
export function slimClaudeRecord(record, state, seen, edits) {
|
|
27
|
+
stripKey(record, "wireToolInputs", "wireToolInputs", edits);
|
|
28
|
+
switch (record["type"]) {
|
|
29
|
+
case "assistant": {
|
|
30
|
+
rememberToolInputs(record, state.toolInputs);
|
|
31
|
+
const message = record["message"];
|
|
32
|
+
if (isJsonObject(message))
|
|
33
|
+
stripKey(message, "input_transformations", "message.input_transformations", edits);
|
|
34
|
+
return;
|
|
35
|
+
}
|
|
36
|
+
case "user":
|
|
37
|
+
slimToolUseResult(record, state.toolInputs, edits);
|
|
38
|
+
return;
|
|
39
|
+
case "attachment":
|
|
40
|
+
slimAttachmentLine(record, seen, edits);
|
|
41
|
+
return;
|
|
42
|
+
case "system":
|
|
43
|
+
slimStopHookSummary(record, edits);
|
|
44
|
+
return;
|
|
45
|
+
default:
|
|
46
|
+
return;
|
|
47
|
+
}
|
|
48
|
+
}
|
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Claude Code `toolUseResult`: the structured copy of what a tool returned, written next to the
|
|
3
|
+
* `tool_result` block that holds the same thing for the model (BLI-16977).
|
|
4
|
+
*
|
|
5
|
+
* Measured on one laptop (80 largest plus 80 random files), 92% of `toolUseResult` bytes repeat
|
|
6
|
+
* something already on the same line (the tool_result text) or on the tool call's own input. This
|
|
7
|
+
* file strips exactly those repeats, and only after PROVING each one is a repeat:
|
|
8
|
+
*
|
|
9
|
+
* - Read `file.content`: the first and last non-blank lines of the file text appear in the
|
|
10
|
+
* tool_result. (A truncated tool_result fails the check and the field stays.)
|
|
11
|
+
* - Bash `stdout` and `stderr`: the text appears in the tool_result, and no `persistedOutputPath`
|
|
12
|
+
* says the tool_result is only a preview of a longer output.
|
|
13
|
+
* - Write `content`, Edit `oldString` and `newString`, Agent and Task `prompt` and `description`:
|
|
14
|
+
* the same sha256 as the tool call's input, which an earlier assistant line carried.
|
|
15
|
+
* - A string or list result (MCP tools): every long string in it appears in the tool_result.
|
|
16
|
+
*
|
|
17
|
+
* Kept on purpose, because each holds something found nowhere else: Bash `bashEditDiff`, the
|
|
18
|
+
* `originalFile` and `structuredPatch` of an Edit or a Write, full output behind a
|
|
19
|
+
* `persistedOutputPath`, Read `totalLines`, `originalSize` and `dimensions`, and every flag
|
|
20
|
+
* (`interrupted`, `isImage`, `userModified`, ...). The harvest extractor reads
|
|
21
|
+
* `toolUseResult.interrupted`; it is never touched.
|
|
22
|
+
*/
|
|
23
|
+
import { canonicalText, contentHash, isJsonObject, stripKey, } from "./slim-support.js";
|
|
24
|
+
/** Tools whose result repeats a field of the tool call's input, and which input fields. */
|
|
25
|
+
const COPIED_INPUT_FIELDS = {
|
|
26
|
+
Write: ["content"],
|
|
27
|
+
Edit: ["old_string", "new_string"],
|
|
28
|
+
Agent: ["prompt", "description"],
|
|
29
|
+
Task: ["prompt", "description"],
|
|
30
|
+
};
|
|
31
|
+
/** Which result field repeats which input field. */
|
|
32
|
+
const RESULT_FIELD_FOR_INPUT = {
|
|
33
|
+
Write: { content: "content" },
|
|
34
|
+
Edit: { oldString: "old_string", newString: "new_string" },
|
|
35
|
+
Agent: { prompt: "prompt", description: "description" },
|
|
36
|
+
Task: { prompt: "prompt", description: "description" },
|
|
37
|
+
};
|
|
38
|
+
/** Tool calls remembered so a result can be matched to its input. Oldest dropped first. */
|
|
39
|
+
const MAX_REMEMBERED_TOOL_CALLS = 50_000;
|
|
40
|
+
/** A string shorter than this is not checked as a copy (it costs nothing to keep). */
|
|
41
|
+
const MIN_COPY_CHARS = 40;
|
|
42
|
+
/** Called for each assistant line: remember what the Write, Edit and Agent calls carried. */
|
|
43
|
+
export function rememberToolInputs(record, inputs) {
|
|
44
|
+
const message = record["message"];
|
|
45
|
+
const content = isJsonObject(message) ? message["content"] : undefined;
|
|
46
|
+
if (!Array.isArray(content))
|
|
47
|
+
return;
|
|
48
|
+
for (const block of content) {
|
|
49
|
+
if (!isJsonObject(block) || block["type"] !== "tool_use")
|
|
50
|
+
continue;
|
|
51
|
+
const id = block["id"];
|
|
52
|
+
const tool = block["name"];
|
|
53
|
+
const input = block["input"];
|
|
54
|
+
if (typeof id !== "string" || typeof tool !== "string" || !isJsonObject(input))
|
|
55
|
+
continue;
|
|
56
|
+
const fields = COPIED_INPUT_FIELDS[tool];
|
|
57
|
+
if (!fields)
|
|
58
|
+
continue;
|
|
59
|
+
const hashes = {};
|
|
60
|
+
for (const field of fields) {
|
|
61
|
+
const text = input[field];
|
|
62
|
+
if (typeof text === "string")
|
|
63
|
+
hashes[field] = contentHash(text);
|
|
64
|
+
}
|
|
65
|
+
inputs.set(id, { tool, hashes });
|
|
66
|
+
if (inputs.size > MAX_REMEMBERED_TOOL_CALLS) {
|
|
67
|
+
const oldest = inputs.keys().next().value;
|
|
68
|
+
if (oldest !== undefined)
|
|
69
|
+
inputs.delete(oldest);
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
/** Called for each user line that carries a `toolUseResult`. */
|
|
74
|
+
export function slimToolUseResult(record, inputs, edits) {
|
|
75
|
+
const result = record["toolUseResult"];
|
|
76
|
+
if (result === undefined)
|
|
77
|
+
return;
|
|
78
|
+
const blocks = toolResultBlocks(record);
|
|
79
|
+
// Several results on one line is not a shape we have seen; leave the line alone rather than guess which is which.
|
|
80
|
+
if (blocks.length !== 1)
|
|
81
|
+
return;
|
|
82
|
+
const block = blocks[0];
|
|
83
|
+
const toolUseId = block["tool_use_id"];
|
|
84
|
+
const digest = typeof toolUseId === "string" ? inputs.get(toolUseId) : undefined;
|
|
85
|
+
if (typeof toolUseId === "string")
|
|
86
|
+
inputs.delete(toolUseId);
|
|
87
|
+
if (isJsonObject(result)) {
|
|
88
|
+
stripCopiedFields(result, digest, toolResultText(block), edits);
|
|
89
|
+
return;
|
|
90
|
+
}
|
|
91
|
+
if (typeof result === "string" || Array.isArray(result)) {
|
|
92
|
+
if (everyLongStringAppearsIn(result, block["content"])) {
|
|
93
|
+
edits.strip("toolUseResult.copy_of_tool_result", canonicalText(result).length);
|
|
94
|
+
delete record["toolUseResult"];
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
function stripCopiedFields(result, digest, resultText, edits) {
|
|
99
|
+
const file = result["file"];
|
|
100
|
+
if (isJsonObject(file) && typeof file["content"] === "string" && fileLinesAppearIn(file["content"], resultText)) {
|
|
101
|
+
stripKey(file, "content", "toolUseResult.file.content", edits);
|
|
102
|
+
}
|
|
103
|
+
if (typeof result["stdout"] === "string" && result["persistedOutputPath"] === undefined && result["isImage"] !== true) {
|
|
104
|
+
for (const field of ["stdout", "stderr"]) {
|
|
105
|
+
const text = result[field];
|
|
106
|
+
if (typeof text === "string" && text.length >= MIN_COPY_CHARS && resultText.includes(text)) {
|
|
107
|
+
stripKey(result, field, `toolUseResult.${field}`, edits);
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
const fieldMap = digest ? RESULT_FIELD_FOR_INPUT[digest.tool] : undefined;
|
|
112
|
+
if (!digest || !fieldMap)
|
|
113
|
+
return;
|
|
114
|
+
for (const [resultField, inputField] of Object.entries(fieldMap)) {
|
|
115
|
+
const text = result[resultField];
|
|
116
|
+
// Same hash as the call's input is an exact repeat at any length, so no minimum here.
|
|
117
|
+
if (typeof text !== "string" || text.length === 0)
|
|
118
|
+
continue;
|
|
119
|
+
if (digest.hashes[inputField] === contentHash(text)) {
|
|
120
|
+
stripKey(result, resultField, `toolUseResult.${resultField}`, edits);
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
function toolResultBlocks(record) {
|
|
125
|
+
const message = record["message"];
|
|
126
|
+
const content = isJsonObject(message) ? message["content"] : undefined;
|
|
127
|
+
if (!Array.isArray(content))
|
|
128
|
+
return [];
|
|
129
|
+
return content.filter((block) => isJsonObject(block) && block["type"] === "tool_result");
|
|
130
|
+
}
|
|
131
|
+
/** The text of a tool_result block: a string as it is, a list of blocks as their text parts joined. */
|
|
132
|
+
function toolResultText(block) {
|
|
133
|
+
const content = block["content"];
|
|
134
|
+
if (typeof content === "string")
|
|
135
|
+
return content;
|
|
136
|
+
if (!Array.isArray(content))
|
|
137
|
+
return "";
|
|
138
|
+
const parts = [];
|
|
139
|
+
for (const part of content) {
|
|
140
|
+
if (isJsonObject(part) && typeof part["text"] === "string")
|
|
141
|
+
parts.push(part["text"]);
|
|
142
|
+
}
|
|
143
|
+
return parts.join("\n");
|
|
144
|
+
}
|
|
145
|
+
/** A Read puts `cat -n` numbering in front of each line, so compare by the first and last non-blank line. */
|
|
146
|
+
function fileLinesAppearIn(fileText, resultText) {
|
|
147
|
+
if (resultText.length === 0)
|
|
148
|
+
return false;
|
|
149
|
+
const lines = fileText.split("\n").map((line) => line.trim()).filter((line) => line.length > 0);
|
|
150
|
+
if (lines.length === 0)
|
|
151
|
+
return false;
|
|
152
|
+
return resultText.includes(lines[0]) && resultText.includes(lines[lines.length - 1]);
|
|
153
|
+
}
|
|
154
|
+
/** Every string of 40 or more characters in `value` appears (JSON-escaped) inside the tool_result content. */
|
|
155
|
+
function everyLongStringAppearsIn(value, toolResultContent) {
|
|
156
|
+
const haystack = JSON.stringify(toolResultContent) ?? "";
|
|
157
|
+
const longStrings = [];
|
|
158
|
+
collectLongStrings(value, longStrings, 0);
|
|
159
|
+
if (longStrings.length === 0)
|
|
160
|
+
return false;
|
|
161
|
+
return longStrings.every((text) => haystack.includes(JSON.stringify(text).slice(1, -1)));
|
|
162
|
+
}
|
|
163
|
+
function collectLongStrings(value, into, depth) {
|
|
164
|
+
if (depth > 6 || into.length > 200)
|
|
165
|
+
return;
|
|
166
|
+
if (typeof value === "string") {
|
|
167
|
+
if (value.length >= MIN_COPY_CHARS)
|
|
168
|
+
into.push(value);
|
|
169
|
+
}
|
|
170
|
+
else if (Array.isArray(value)) {
|
|
171
|
+
for (const item of value)
|
|
172
|
+
collectLongStrings(item, into, depth + 1);
|
|
173
|
+
}
|
|
174
|
+
else if (isJsonObject(value)) {
|
|
175
|
+
for (const item of Object.values(value))
|
|
176
|
+
collectLongStrings(item, into, depth + 1);
|
|
177
|
+
}
|
|
178
|
+
}
|
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The slim rules for one Codex rollout line (BLI-16977). Entry point: `slimCodexRecord`.
|
|
3
|
+
*
|
|
4
|
+
* A Codex line is `{timestamp, type, payload}`. The envelope never changes; only payload goes.
|
|
5
|
+
* Measured on 80 local rollouts, almost all the bulk is images (turned into markers before this
|
|
6
|
+
* runs) and a second copy of each finished item (`event_msg` `item_completed`) next to the
|
|
7
|
+
* `response_item` the model saw. The survey and its skeptic agree on what is a copy:
|
|
8
|
+
*
|
|
9
|
+
* - `item_completed` CommandExecution: `formatted_output` and `parsed_cmd` go, and `stdout` goes
|
|
10
|
+
* when it equals `aggregated_output`. `aggregated_output` and `command` STAY: they are the only
|
|
11
|
+
* full copy of what the shell printed.
|
|
12
|
+
* - `item_completed` UserMessage, AgentMessage, Reasoning: any long string identical to one in an
|
|
13
|
+
* EARLIER `response_item` becomes a reference. A copy that came first in the file is kept.
|
|
14
|
+
* - McpToolCall results, FileChange contents, `compacted.replacement_history`, image generations:
|
|
15
|
+
* kept (their images are markers by now). `compacted.guardian_history` keeps only what an
|
|
16
|
+
* earlier `response_item` did not already hold.
|
|
17
|
+
* - `token_count`: `last_token_usage` and `rate_limits` go; `total_token_usage` and
|
|
18
|
+
* `model_context_window` stay (the session facts read them).
|
|
19
|
+
* - `token_usage_record`: payload emptied (17,964 of 18,109 were byte-equal to a `token_count`).
|
|
20
|
+
* - `thread_settings_applied`: reduced to `service_tier`.
|
|
21
|
+
* - `response_item`: `internal_chat_message_metadata_passthrough` goes; a developer message, the
|
|
22
|
+
* `session_meta` `base_instructions` and each `world_state` block are kept once per distinct
|
|
23
|
+
* text and become references after.
|
|
24
|
+
* - `task_complete.last_agent_message`: a reference when an earlier message holds the same text.
|
|
25
|
+
*
|
|
26
|
+
* `task_started`/`task_complete` durations, `session_meta.forked_from_id` and every unnamed shape
|
|
27
|
+
* pass through untouched.
|
|
28
|
+
*/
|
|
29
|
+
import { SLIM_COPY_MIN_CHARS, canonicalText, contentHash, isJsonObject, isSlimRef, slimRefFor, stripKey, } from "./slim-support.js";
|
|
30
|
+
/** Strings from `response_item` lines remembered for comparison. Past this, new strings are not added. */
|
|
31
|
+
const MAX_REMEMBERED_STRINGS = 300_000;
|
|
32
|
+
/** Item types whose event copy is a pure repeat of a `response_item`. */
|
|
33
|
+
const ITEM_TYPES_THAT_REPEAT = new Set(["UserMessage", "AgentMessage", "Reasoning"]);
|
|
34
|
+
export function newCodexSlimState() {
|
|
35
|
+
return { earlierStrings: new Set() };
|
|
36
|
+
}
|
|
37
|
+
export function slimCodexRecord(record, state, seen, edits) {
|
|
38
|
+
const payload = record["payload"];
|
|
39
|
+
if (!isJsonObject(payload))
|
|
40
|
+
return;
|
|
41
|
+
switch (record["type"]) {
|
|
42
|
+
case "response_item":
|
|
43
|
+
rememberLongStrings(payload, state, 0);
|
|
44
|
+
stripKey(payload, "internal_chat_message_metadata_passthrough", "codex.response_item.passthrough", edits);
|
|
45
|
+
if (payload["type"] === "message" && payload["role"] === "developer") {
|
|
46
|
+
seen.dedupeKey(payload, "content", "codex.developer_message.repeat", edits);
|
|
47
|
+
}
|
|
48
|
+
return;
|
|
49
|
+
case "session_meta":
|
|
50
|
+
seen.dedupeKey(payload, "base_instructions", "codex.session_meta.base_instructions.repeat", edits);
|
|
51
|
+
return;
|
|
52
|
+
case "world_state": {
|
|
53
|
+
const blocks = payload["state"];
|
|
54
|
+
if (isJsonObject(blocks))
|
|
55
|
+
for (const key of Object.keys(blocks))
|
|
56
|
+
seen.dedupeKey(blocks, key, "codex.world_state.repeat", edits);
|
|
57
|
+
return;
|
|
58
|
+
}
|
|
59
|
+
case "compacted": {
|
|
60
|
+
// guardian_history re-saves earlier turns; replacement_history is kept whole (part of it is found nowhere else).
|
|
61
|
+
const guardian = payload["guardian_history"];
|
|
62
|
+
if (Array.isArray(guardian))
|
|
63
|
+
refEarlierCopiesIn(guardian, state, "codex.compacted.guardian_history.copy_of_response_item", edits, 0);
|
|
64
|
+
return;
|
|
65
|
+
}
|
|
66
|
+
case "token_usage_record":
|
|
67
|
+
emptyPayload(record, payload, "codex.token_usage_record", edits);
|
|
68
|
+
return;
|
|
69
|
+
case "event_msg":
|
|
70
|
+
slimEvent(payload, state, edits);
|
|
71
|
+
return;
|
|
72
|
+
default:
|
|
73
|
+
return;
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
function slimEvent(payload, state, edits) {
|
|
77
|
+
switch (payload["type"]) {
|
|
78
|
+
case "token_count": {
|
|
79
|
+
const info = payload["info"];
|
|
80
|
+
if (isJsonObject(info))
|
|
81
|
+
stripKey(info, "last_token_usage", "codex.token_count.last_token_usage", edits);
|
|
82
|
+
stripKey(payload, "rate_limits", "codex.token_count.rate_limits", edits);
|
|
83
|
+
return;
|
|
84
|
+
}
|
|
85
|
+
case "thread_settings_applied":
|
|
86
|
+
keepOnlyServiceTier(payload, edits);
|
|
87
|
+
return;
|
|
88
|
+
case "task_complete":
|
|
89
|
+
refIfEarlierCopy(payload, "last_agent_message", state, "codex.task_complete.last_agent_message", edits);
|
|
90
|
+
return;
|
|
91
|
+
case "item_completed":
|
|
92
|
+
slimFinishedItem(payload["item"], state, edits);
|
|
93
|
+
return;
|
|
94
|
+
default:
|
|
95
|
+
return;
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
function slimFinishedItem(item, state, edits) {
|
|
99
|
+
if (!isJsonObject(item))
|
|
100
|
+
return;
|
|
101
|
+
const type = item["type"];
|
|
102
|
+
if (type === "CommandExecution") {
|
|
103
|
+
stripKey(item, "formatted_output", "codex.item.command.formatted_output", edits);
|
|
104
|
+
stripKey(item, "parsed_cmd", "codex.item.command.parsed_cmd", edits);
|
|
105
|
+
const stdout = item["stdout"];
|
|
106
|
+
if (typeof stdout === "string" && stdout.length >= SLIM_COPY_MIN_CHARS && stdout === item["aggregated_output"]) {
|
|
107
|
+
stripKey(item, "stdout", "codex.item.command.stdout", edits);
|
|
108
|
+
}
|
|
109
|
+
return;
|
|
110
|
+
}
|
|
111
|
+
if (typeof type === "string" && ITEM_TYPES_THAT_REPEAT.has(type)) {
|
|
112
|
+
refEarlierCopiesIn(item, state, `codex.item.${type}.copy_of_response_item`, edits, 0);
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
/** The payload emptied, once. */
|
|
116
|
+
function emptyPayload(record, payload, rule, edits) {
|
|
117
|
+
if (Object.keys(payload).length === 0)
|
|
118
|
+
return;
|
|
119
|
+
edits.strip(rule, canonicalText(payload).length);
|
|
120
|
+
record["payload"] = {};
|
|
121
|
+
}
|
|
122
|
+
function keepOnlyServiceTier(payload, edits) {
|
|
123
|
+
const settings = payload["thread_settings"];
|
|
124
|
+
if (!isJsonObject(settings))
|
|
125
|
+
return;
|
|
126
|
+
const others = Object.keys(settings).filter((key) => key !== "service_tier");
|
|
127
|
+
if (others.length === 0)
|
|
128
|
+
return;
|
|
129
|
+
edits.strip("codex.thread_settings_applied", canonicalText(settings).length);
|
|
130
|
+
payload["thread_settings"] = settings["service_tier"] === undefined ? {} : { service_tier: settings["service_tier"] };
|
|
131
|
+
}
|
|
132
|
+
/** Walks a `response_item` payload and remembers each long string (reasoning ciphertext excluded). */
|
|
133
|
+
function rememberLongStrings(value, state, depth) {
|
|
134
|
+
if (depth > 8 || state.earlierStrings.size >= MAX_REMEMBERED_STRINGS)
|
|
135
|
+
return;
|
|
136
|
+
if (typeof value === "string") {
|
|
137
|
+
if (value.length >= SLIM_COPY_MIN_CHARS)
|
|
138
|
+
state.earlierStrings.add(contentHash(value));
|
|
139
|
+
}
|
|
140
|
+
else if (Array.isArray(value)) {
|
|
141
|
+
for (const item of value)
|
|
142
|
+
rememberLongStrings(item, state, depth + 1);
|
|
143
|
+
}
|
|
144
|
+
else if (isJsonObject(value)) {
|
|
145
|
+
for (const [key, item] of Object.entries(value)) {
|
|
146
|
+
if (key !== "encrypted_content")
|
|
147
|
+
rememberLongStrings(item, state, depth + 1);
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
function refIfEarlierCopy(holder, key, state, rule, edits) {
|
|
152
|
+
const value = holder[key];
|
|
153
|
+
if (typeof value !== "string" || value.length < SLIM_COPY_MIN_CHARS)
|
|
154
|
+
return;
|
|
155
|
+
const hash = contentHash(value);
|
|
156
|
+
if (!state.earlierStrings.has(hash))
|
|
157
|
+
return;
|
|
158
|
+
holder[key] = slimRefFor(hash);
|
|
159
|
+
edits.strip(rule, value.length);
|
|
160
|
+
}
|
|
161
|
+
/** Every long string in `node` that an earlier `response_item` already held becomes a reference. */
|
|
162
|
+
function refEarlierCopiesIn(node, state, rule, edits, depth) {
|
|
163
|
+
if (depth > 8)
|
|
164
|
+
return;
|
|
165
|
+
const keys = Array.isArray(node) ? node.map((_, index) => index) : Object.keys(node);
|
|
166
|
+
for (const key of keys) {
|
|
167
|
+
const holder = node;
|
|
168
|
+
const child = holder[key];
|
|
169
|
+
if (typeof child === "string") {
|
|
170
|
+
if (child.length >= SLIM_COPY_MIN_CHARS && state.earlierStrings.has(contentHash(child))) {
|
|
171
|
+
holder[key] = slimRefFor(contentHash(child));
|
|
172
|
+
edits.strip(rule, child.length);
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
else if ((Array.isArray(child) || isJsonObject(child)) && !isSlimRef(child)) {
|
|
176
|
+
refEarlierCopiesIn(child, state, rule, edits, depth + 1);
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
}
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shared pieces of the slim transcript (BLI-16977): what a rule records, how a repeated block is
|
|
3
|
+
* swapped for a short reference, and the few JSON helpers every rule uses.
|
|
4
|
+
*
|
|
5
|
+
* The slim transcript is built one line at a time, in order, and each line may depend only on the
|
|
6
|
+
* lines before it and itself. That is the whole contract: it is why a growing transcript's slim
|
|
7
|
+
* form extends its earlier slim form byte for byte (the BLI-11088 prefix rule), and why the same
|
|
8
|
+
* code can slim a 700 MB file without holding it in memory.
|
|
9
|
+
*
|
|
10
|
+
* Nothing here reads a file, the network or the environment. Rules record sizes and rule names,
|
|
11
|
+
* never content.
|
|
12
|
+
*/
|
|
13
|
+
import { createHash } from "node:crypto";
|
|
14
|
+
/** A block shorter than this is never worth swapping for a reference (the reference is about 40 characters). */
|
|
15
|
+
export const SLIM_REF_MIN_CHARS = 300;
|
|
16
|
+
/** A string shorter than this is not looked up against earlier copies. */
|
|
17
|
+
export const SLIM_COPY_MIN_CHARS = 80;
|
|
18
|
+
export function isJsonObject(value) {
|
|
19
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
20
|
+
}
|
|
21
|
+
export function isSlimRef(value) {
|
|
22
|
+
return isJsonObject(value) && typeof value["slim_ref"] === "string" && Object.keys(value).length === 1;
|
|
23
|
+
}
|
|
24
|
+
/** `sha256:` plus the first 16 hex characters of the sha256 of the value's text (a string as it is, anything else as compact JSON). */
|
|
25
|
+
export function contentHash(canonicalText) {
|
|
26
|
+
return createHash("sha256").update(canonicalText).digest("hex").slice(0, 16);
|
|
27
|
+
}
|
|
28
|
+
export function slimRefFor(hash) {
|
|
29
|
+
return { slim_ref: `sha256:${hash}` };
|
|
30
|
+
}
|
|
31
|
+
/** The text a value is hashed and measured by. */
|
|
32
|
+
export function canonicalText(value) {
|
|
33
|
+
if (typeof value === "string")
|
|
34
|
+
return value;
|
|
35
|
+
return JSON.stringify(value) ?? "";
|
|
36
|
+
}
|
|
37
|
+
/** What one line lost, by rule, in characters. A line with no entries was not touched. */
|
|
38
|
+
export class LineEdits {
|
|
39
|
+
charsByRule = new Map();
|
|
40
|
+
strip(rule, chars) {
|
|
41
|
+
this.charsByRule.set(rule, (this.charsByRule.get(rule) ?? 0) + chars);
|
|
42
|
+
}
|
|
43
|
+
get rules() {
|
|
44
|
+
return [...this.charsByRule.keys()];
|
|
45
|
+
}
|
|
46
|
+
get entries() {
|
|
47
|
+
return this.charsByRule.entries();
|
|
48
|
+
}
|
|
49
|
+
get isEmpty() {
|
|
50
|
+
return this.charsByRule.size === 0;
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
/** Remove `key` from `holder`, recording the rule and the characters it held. */
|
|
54
|
+
export function stripKey(holder, key, rule, edits) {
|
|
55
|
+
if (!(key in holder))
|
|
56
|
+
return false;
|
|
57
|
+
edits.strip(rule, canonicalText(holder[key]).length);
|
|
58
|
+
delete holder[key];
|
|
59
|
+
return true;
|
|
60
|
+
}
|
|
61
|
+
/**
|
|
62
|
+
* The one copy per transcript. The first time a block is seen it stays in the line; every time
|
|
63
|
+
* after, the block is replaced by `{"slim_ref":"sha256:<16 hex>"}` naming the earlier copy. The
|
|
64
|
+
* hash is of the block's text (a string as it is, anything else as compact JSON).
|
|
65
|
+
*/
|
|
66
|
+
export class SeenBlocks {
|
|
67
|
+
hashes = new Set();
|
|
68
|
+
/** True when this block was seen before in the transcript; remembers it otherwise. */
|
|
69
|
+
seenBefore(hash) {
|
|
70
|
+
if (this.hashes.has(hash))
|
|
71
|
+
return true;
|
|
72
|
+
this.hashes.add(hash);
|
|
73
|
+
return false;
|
|
74
|
+
}
|
|
75
|
+
/** Swap `holder[key]` for a reference when an identical block came earlier. */
|
|
76
|
+
dedupeKey(holder, key, rule, edits) {
|
|
77
|
+
const value = holder[key];
|
|
78
|
+
if (value === undefined || isSlimRef(value))
|
|
79
|
+
return;
|
|
80
|
+
const text = canonicalText(value);
|
|
81
|
+
if (text.length < SLIM_REF_MIN_CHARS)
|
|
82
|
+
return;
|
|
83
|
+
const hash = contentHash(text);
|
|
84
|
+
if (!this.seenBefore(hash))
|
|
85
|
+
return;
|
|
86
|
+
const ref = slimRefFor(hash);
|
|
87
|
+
holder[key] = ref;
|
|
88
|
+
edits.strip(rule, Math.max(0, text.length - JSON.stringify(ref).length));
|
|
89
|
+
}
|
|
90
|
+
}
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The slim transcript (BLI-16977): the transcript the collector uploads when
|
|
3
|
+
* `IMAGE_HANDLING_MODE` in telemetry-core is `describe_and_drop`. Images are already markers by the
|
|
4
|
+
* time this runs; this strips the duplicate copies and the bookkeeping that remain.
|
|
5
|
+
*
|
|
6
|
+
* The contract, in the order Edward and the skeptic pass set it:
|
|
7
|
+
*
|
|
8
|
+
* 1. ONE OUTPUT LINE PER INPUT LINE, same order. Tower counts lines and reads every line's
|
|
9
|
+
* timestamp, so a line is never dropped; its payload is. A blank line, a line that is not JSON,
|
|
10
|
+
* and a JSON value that is not an object are emitted as they came.
|
|
11
|
+
* 2. A LINE NO RULE TOUCHES IS EMITTED AS ITS ORIGINAL TEXT, never re-serialised. Only a modified
|
|
12
|
+
* line is re-written, and it gains a compact top-level `"_slim":["rule", ...]` naming what was
|
|
13
|
+
* stripped, so a reader can tell.
|
|
14
|
+
* 3. PREFIX STABLE. A line's slim form depends only on that line and the lines before it (no
|
|
15
|
+
* look-ahead, no global pass). So the slim form of a growing transcript extends the slim form
|
|
16
|
+
* of the shorter one byte for byte, and the BLI-11088 prefix supersede rule keeps working.
|
|
17
|
+
* The one honest exception is a last line still being written: a half-written JSON line is
|
|
18
|
+
* emitted as it is and becomes a slim line once it is whole.
|
|
19
|
+
* 4. NEVER FAIL A TRANSCRIPT. A line whose rules throw is emitted as it came and counted; any other
|
|
20
|
+
* failure hands back the text it was given (`slimOrKeep`) and says why.
|
|
21
|
+
*
|
|
22
|
+
* What each rule keeps and drops: claude-rules.ts and codex-rules.ts. The table for people:
|
|
23
|
+
* docs/architecture/image-descriptions.md.
|
|
24
|
+
*/
|
|
25
|
+
import { newClaudeSlimState, slimClaudeRecord } from "./claude-rules.js";
|
|
26
|
+
import { newCodexSlimState, slimCodexRecord } from "./codex-rules.js";
|
|
27
|
+
import { LineEdits, SeenBlocks, isJsonObject } from "./slim-support.js";
|
|
28
|
+
/** The text an image marker starts with; a line carrying one is a modified line (an image became a marker). */
|
|
29
|
+
const IMAGE_MARKER_START = "[image sha256:";
|
|
30
|
+
export function createSlimmer(kind, rulesForTests) {
|
|
31
|
+
const seen = new SeenBlocks();
|
|
32
|
+
const claude = newClaudeSlimState();
|
|
33
|
+
const codex = newCodexSlimState();
|
|
34
|
+
const stats = {
|
|
35
|
+
lines: 0, linesModified: 0, linesFailed: 0, bytesBefore: 0, bytesAfter: 0, charsByRule: {}, linesByRule: {},
|
|
36
|
+
};
|
|
37
|
+
const applyRules = rulesForTests ?? ((record, edits) => {
|
|
38
|
+
if (kind === "claude_code")
|
|
39
|
+
slimClaudeRecord(record, claude, seen, edits);
|
|
40
|
+
else
|
|
41
|
+
slimCodexRecord(record, codex, seen, edits);
|
|
42
|
+
});
|
|
43
|
+
return {
|
|
44
|
+
slimLine(line) {
|
|
45
|
+
stats.lines += 1;
|
|
46
|
+
const before = Buffer.byteLength(line);
|
|
47
|
+
stats.bytesBefore += before;
|
|
48
|
+
let out = line;
|
|
49
|
+
try {
|
|
50
|
+
out = slimOneLine(line, applyRules, stats);
|
|
51
|
+
}
|
|
52
|
+
catch {
|
|
53
|
+
stats.linesFailed += 1;
|
|
54
|
+
out = line;
|
|
55
|
+
}
|
|
56
|
+
stats.bytesAfter += out === line ? before : Buffer.byteLength(out);
|
|
57
|
+
return out;
|
|
58
|
+
},
|
|
59
|
+
stats: () => stats,
|
|
60
|
+
};
|
|
61
|
+
}
|
|
62
|
+
function slimOneLine(line, applyRules, stats) {
|
|
63
|
+
const body = line.endsWith("\r") ? line.slice(0, -1) : line;
|
|
64
|
+
const first = body.trimStart().charCodeAt(0);
|
|
65
|
+
if (first !== 0x7b /* { */)
|
|
66
|
+
return line;
|
|
67
|
+
let record;
|
|
68
|
+
try {
|
|
69
|
+
record = JSON.parse(body);
|
|
70
|
+
}
|
|
71
|
+
catch {
|
|
72
|
+
return line;
|
|
73
|
+
}
|
|
74
|
+
if (!isJsonObject(record))
|
|
75
|
+
return line;
|
|
76
|
+
const edits = new LineEdits();
|
|
77
|
+
applyRules(record, edits);
|
|
78
|
+
const rules = edits.rules;
|
|
79
|
+
if (line.includes(IMAGE_MARKER_START))
|
|
80
|
+
rules.push("images");
|
|
81
|
+
const already = Array.isArray(record["_slim"]) ? record["_slim"] : [];
|
|
82
|
+
const fresh = rules.filter((rule) => !already.includes(rule));
|
|
83
|
+
if (fresh.length === 0)
|
|
84
|
+
return line;
|
|
85
|
+
record["_slim"] = [...already, ...fresh];
|
|
86
|
+
for (const [rule, chars] of edits.entries) {
|
|
87
|
+
stats.charsByRule[rule] = (stats.charsByRule[rule] ?? 0) + chars;
|
|
88
|
+
stats.linesByRule[rule] = (stats.linesByRule[rule] ?? 0) + 1;
|
|
89
|
+
}
|
|
90
|
+
if (fresh.includes("images"))
|
|
91
|
+
stats.linesByRule["images"] = (stats.linesByRule["images"] ?? 0) + 1;
|
|
92
|
+
stats.linesModified += 1;
|
|
93
|
+
return JSON.stringify(record) + (line.endsWith("\r") ? "\r" : "");
|
|
94
|
+
}
|
|
95
|
+
/** The slim form of a whole transcript's text, line for line. */
|
|
96
|
+
export function slimTranscript(text, kind) {
|
|
97
|
+
const slimmer = createSlimmer(kind);
|
|
98
|
+
const lines = text.split("\n");
|
|
99
|
+
const out = new Array(lines.length);
|
|
100
|
+
for (let index = 0; index < lines.length; index += 1)
|
|
101
|
+
out[index] = slimmer.slimLine(lines[index]);
|
|
102
|
+
const stats = slimmer.stats();
|
|
103
|
+
return { text: stats.linesModified === 0 ? text : out.join("\n"), stats };
|
|
104
|
+
}
|
|
105
|
+
const TAG = "[slim-transcript]";
|
|
106
|
+
/**
|
|
107
|
+
* `slimTranscript` that can never fail a transcript: any exception returns the text it was given and
|
|
108
|
+
* logs the reason. Metadata only: counts, sizes, rule names. Never a line, never a field value.
|
|
109
|
+
*/
|
|
110
|
+
export function slimOrKeep(text, kind, label) {
|
|
111
|
+
const startedAt = Date.now();
|
|
112
|
+
try {
|
|
113
|
+
const slim = slimTranscript(text, kind);
|
|
114
|
+
const stats = slim.stats;
|
|
115
|
+
console.error(TAG + (stats.linesModified === 0 ? " nothing to strip" : " transcript slimmed"), JSON.stringify({
|
|
116
|
+
source: kind,
|
|
117
|
+
session_id: label.sessionId.slice(0, 80),
|
|
118
|
+
lines: stats.lines,
|
|
119
|
+
lines_modified: stats.linesModified,
|
|
120
|
+
lines_failed: stats.linesFailed,
|
|
121
|
+
bytes_before: stats.bytesBefore,
|
|
122
|
+
bytes_after: stats.bytesAfter,
|
|
123
|
+
chars_by_rule: stats.charsByRule,
|
|
124
|
+
lines_by_rule: stats.linesByRule,
|
|
125
|
+
elapsed_ms: Date.now() - startedAt,
|
|
126
|
+
}));
|
|
127
|
+
return { text: slim.text, stats };
|
|
128
|
+
}
|
|
129
|
+
catch (error) {
|
|
130
|
+
console.error(TAG + " failed, uploading the transcript as it was", JSON.stringify({ reason: "slim_failed", source: kind, error_name: error instanceof Error ? error.name : typeof error, bytes: typeof text === "string" ? text.length : 0 }));
|
|
131
|
+
return { text, stats: null };
|
|
132
|
+
}
|
|
133
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@bli-cockpit/cli",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.207",
|
|
4
4
|
"private": false,
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
@@ -27,9 +27,9 @@
|
|
|
27
27
|
"test": "node dist/cli.js --help && node ../../scripts/assert-public-cli-routing.mjs && node ../../scripts/assert-public-cli-verb-help.mjs && node ../../scripts/assert-public-cli-surface-doors.mjs && node ../../scripts/assert-public-cli-runtime-files.mjs && node ../../scripts/assert-public-cli-exit-contract.mjs && node ../../scripts/assert-public-cli-no-fleet-posts.mjs && node ../../scripts/assert-public-cli-doctor-json.mjs && node ../../scripts/assert-public-package-pack.mjs --workspace=@bli-cockpit/cli"
|
|
28
28
|
},
|
|
29
29
|
"dependencies": {
|
|
30
|
-
"@bli-cockpit/memory-mcp": "0.1.
|
|
31
|
-
"@bli-cockpit/mcp": "0.1.
|
|
32
|
-
"@bli-cockpit/telemetry-core": "0.1.
|
|
30
|
+
"@bli-cockpit/memory-mcp": "0.1.63",
|
|
31
|
+
"@bli-cockpit/mcp": "0.1.118",
|
|
32
|
+
"@bli-cockpit/telemetry-core": "0.1.72"
|
|
33
33
|
},
|
|
34
34
|
"repository": {
|
|
35
35
|
"type": "git",
|