opencode-vcc-compaction 0.0.0-stage → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +335 -2
- package/adapter.mjs +264 -0
- package/engine/compile.mjs +962 -0
- package/engine/recall.mjs +277 -0
- package/engine/search.mjs +124 -0
- package/engine/session-compat.mjs +98 -0
- package/engine/tokenize.mjs +202 -0
- package/index.ts +286 -0
- package/package.json +33 -4
- package/snapshot.mjs +515 -0
|
@@ -0,0 +1,962 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Region compiler, ported from dsh-compaction-vcc@0.7.4 `src/compiler.js`
|
|
3
|
+
* (MIT: Copyright (c) 2026 TsFreddie, Copyright (c) 2026 Tingfeng) —
|
|
4
|
+
* everything from the noise-stripping section through checkpoint framing.
|
|
5
|
+
* The tokenizer half lives in `./tokenize.mjs` and is imported below,
|
|
6
|
+
* never re-defined.
|
|
7
|
+
*
|
|
8
|
+
* Faithful port with three documented deviations:
|
|
9
|
+
* - Budget primitives (`sanitize`, `countTokens`, `truncateTokens`,
|
|
10
|
+
* `tailTokens`, `estimateEntryTokens`) are imported from `./tokenize.mjs`
|
|
11
|
+
* (module-private upstream; exported there so this sibling stage can
|
|
12
|
+
* import them across the file split).
|
|
13
|
+
* - DSH harness vocabulary is carried verbatim as configurable defaults, not
|
|
14
|
+
* forked: `DEFAULT_NOISE_PATTERNS`, `DEFAULT_TOOL_KEY_FIELDS`,
|
|
15
|
+
* `DEFAULT_ARG_TOOLS`, `ELIDABLE_SOURCE_LABELS`, the memory-snapshot /
|
|
16
|
+
* system-reminder span finders, and `isCheckpointSource` kinds all describe
|
|
17
|
+
* the DSH host. The OpenCode mapping lives in config (`noisePatterns`,
|
|
18
|
+
* `toolKeyFields`, `toolArgTools`, `hideTools`, `elideSourceKinds`) and in
|
|
19
|
+
* the node-projection adapter (hook `Message` + `SessionMessage.ID` ->
|
|
20
|
+
* `{ seq, message }`), which is a later stage per `archive/plans/vcc-compiler.md`.
|
|
21
|
+
* - `elideCheckpointInjectedBodies` is retained verbatim but stays inert
|
|
22
|
+
* unless `config.elideCheckpointInjectedBodies === true`; the OpenCode
|
|
23
|
+
* consumer leaves it off (REUSE.md rejects it — the preamble here is
|
|
24
|
+
* per-request transient, never persisted history).
|
|
25
|
+
*
|
|
26
|
+
* Note on style: this file keeps upstream's semicolons verbatim for
|
|
27
|
+
* line-for-line comparability with the MIT source (present and future
|
|
28
|
+
* re-sync). This coincides with the repo's own `.mjs` style
|
|
29
|
+
* (`compiler/selftest.mjs`, `scripts/check-engine-sync.mjs` both use
|
|
30
|
+
* semicolons), so there is no style conflict in practice.
|
|
31
|
+
*
|
|
32
|
+
* Everything below is deterministic and offline: same nodes + config always
|
|
33
|
+
* yield byte-identical entries, with no model call and no dependencies.
|
|
34
|
+
*/
|
|
35
|
+
import {
|
|
36
|
+
countTokens,
|
|
37
|
+
estimateEntryTokens,
|
|
38
|
+
sanitize,
|
|
39
|
+
tailTokens,
|
|
40
|
+
truncateTokens,
|
|
41
|
+
} from "./tokenize.mjs";
|
|
42
|
+
|
|
43
|
+
// ── noise stripping (VCC `_BRIEF_STRIP_RE` family) ─────────────────────────
|
|
44
|
+
|
|
45
|
+
const NOISE_UNWRAP_RE = /<\/?(?:command-name|command-args)>/g;
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* The VCC brief-mode noise XML patterns, adapted to harness markup. Applied
|
|
49
|
+
* with the `s` flag; the replacement set is configurable via
|
|
50
|
+
* `InstantCompactionConfig.noisePatterns`.
|
|
51
|
+
*/
|
|
52
|
+
export const DEFAULT_NOISE_PATTERNS = [
|
|
53
|
+
"<ide_opened_file>.*?</ide_opened_file>",
|
|
54
|
+
"<ide_selection>.*?</ide_selection>",
|
|
55
|
+
"<system-reminder>.*?</system-reminder>",
|
|
56
|
+
"<command-message>.*?</command-message>",
|
|
57
|
+
"<task-notification>.*?</task-notification>"
|
|
58
|
+
];
|
|
59
|
+
|
|
60
|
+
/** Compile untrusted noise-pattern sources, throwing on invalid regex. */
|
|
61
|
+
export function compileNoisePatterns(sources) {
|
|
62
|
+
return sources.map((source, index) => {
|
|
63
|
+
try {
|
|
64
|
+
return new RegExp(source, "s");
|
|
65
|
+
} catch (error) {
|
|
66
|
+
throw new Error(`noisePatterns[${index}] (${JSON.stringify(source)}) is not a valid regular expression: ${error instanceof Error ? error.message : String(error)}`);
|
|
67
|
+
}
|
|
68
|
+
});
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* Strip configured noise XML blocks from user text and unwrap command-name /
|
|
73
|
+
* command-args tags (VCC `_strip_noise_xml` + `_BRIEF_UNWRAP_RE`).
|
|
74
|
+
* @param text - user text.
|
|
75
|
+
* @param patterns - compiled noise regexes.
|
|
76
|
+
* @returns the stripped text, trimmed.
|
|
77
|
+
*/
|
|
78
|
+
export function stripNoiseXml(text, patterns) {
|
|
79
|
+
let out = text;
|
|
80
|
+
for (const pattern of patterns) out = out.replace(pattern, "");
|
|
81
|
+
out = out.replace(NOISE_UNWRAP_RE, "");
|
|
82
|
+
return out.trim();
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
// ── tool-call one-liner (VCC `_tool_summary`) ──────────────────────────────
|
|
86
|
+
|
|
87
|
+
/** VCC's `_TOOL_SUMMARY_FIELDS`, extended for the DSH tool catalog. */
|
|
88
|
+
export const DEFAULT_TOOL_KEY_FIELDS = Object.freeze({
|
|
89
|
+
read: "file_path",
|
|
90
|
+
edit: "file_path",
|
|
91
|
+
write: "file_path",
|
|
92
|
+
glob: "pattern",
|
|
93
|
+
grep: "pattern",
|
|
94
|
+
bash: "command",
|
|
95
|
+
web_search: "query",
|
|
96
|
+
skill: "name",
|
|
97
|
+
ralph: "objective",
|
|
98
|
+
subagent: "description",
|
|
99
|
+
subagent_fork: "description",
|
|
100
|
+
create_goal: "objective",
|
|
101
|
+
update_goal: "objective",
|
|
102
|
+
workflow: "name",
|
|
103
|
+
interrupt_agent: "agent_id",
|
|
104
|
+
job_kill: "job_id",
|
|
105
|
+
job_output: "job_id",
|
|
106
|
+
send_message: "message"
|
|
107
|
+
});
|
|
108
|
+
|
|
109
|
+
/**
|
|
110
|
+
* Tools whose key argument is kept in the one-liner. Everything else renders
|
|
111
|
+
* name-only (`* tool (seq N)`) — bookkeeping tools carry no semantic payload
|
|
112
|
+
* worth the tokens (VCC's brief mode keeps `Read`/`Edit`/… args and drops the
|
|
113
|
+
* rest; `_BRIEF_HIDE_TOOLS` goes further and hides TodoWrite/ToolSearch
|
|
114
|
+
* entirely, which `hideTools` config covers here).
|
|
115
|
+
*/
|
|
116
|
+
export const DEFAULT_ARG_TOOLS = Object.freeze([
|
|
117
|
+
"read",
|
|
118
|
+
"write",
|
|
119
|
+
"edit",
|
|
120
|
+
"glob",
|
|
121
|
+
"grep",
|
|
122
|
+
"bash",
|
|
123
|
+
"shell",
|
|
124
|
+
"web_search",
|
|
125
|
+
"skill",
|
|
126
|
+
"subagent",
|
|
127
|
+
"subagent_fork",
|
|
128
|
+
"ralph",
|
|
129
|
+
"workflow"
|
|
130
|
+
]);
|
|
131
|
+
|
|
132
|
+
/**
|
|
133
|
+
* First-line projection for a tool-call one-liner: multiline arguments
|
|
134
|
+
* collapse to the first physical line plus a `[+ N lines]` marker, so
|
|
135
|
+
* scripted commands (bash heredocs, multi-line pipelines) stay one compact
|
|
136
|
+
* line instead of echoing the whole payload.
|
|
137
|
+
* @param value - the key-argument string.
|
|
138
|
+
* @returns the display value.
|
|
139
|
+
*/
|
|
140
|
+
export function oneLineArg(value) {
|
|
141
|
+
const lines = value.split(/\r?\n/u);
|
|
142
|
+
if (lines.length === 1) return value;
|
|
143
|
+
const first = lines[0] !== "" || lines.every((line) => line.trim() === "")
|
|
144
|
+
? lines[0]
|
|
145
|
+
: lines.find((line) => line.trim() !== "");
|
|
146
|
+
return `${first} [+ ${lines.length - 1} lines]`;
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
/**
|
|
150
|
+
* Pick the key argument for a tool-call one-liner: the tool-specific field
|
|
151
|
+
* first, then the first string-valued argument, else nothing.
|
|
152
|
+
* @param name - tool name.
|
|
153
|
+
* @param input - parsed arguments object (or null when unparseable).
|
|
154
|
+
* @param keyFields - merged tool-name → preferred-field map.
|
|
155
|
+
* @returns the display value, or undefined.
|
|
156
|
+
*/
|
|
157
|
+
export function pickToolKeyArg(name, input, keyFields) {
|
|
158
|
+
if (input === null || typeof input !== "object" || Array.isArray(input)) return undefined;
|
|
159
|
+
const preferred = keyFields[name];
|
|
160
|
+
if (preferred !== undefined) {
|
|
161
|
+
const value = input[preferred];
|
|
162
|
+
if (typeof value === "string" && value.length > 0) return value;
|
|
163
|
+
}
|
|
164
|
+
for (const value of Object.values(input)) {
|
|
165
|
+
if (typeof value === "string" && value.length > 0) return value;
|
|
166
|
+
}
|
|
167
|
+
return undefined;
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
/**
|
|
171
|
+
* Safely parse a tool-call's raw JSON arguments string.
|
|
172
|
+
* @param arguments_ - raw JSON as produced by the model.
|
|
173
|
+
* @returns the parsed value, or null when it is not valid JSON.
|
|
174
|
+
*/
|
|
175
|
+
export function parseToolArguments(arguments_) {
|
|
176
|
+
try {
|
|
177
|
+
return JSON.parse(arguments_);
|
|
178
|
+
} catch {
|
|
179
|
+
return null;
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
// ── content projection helpers ─────────────────────────────────────────────
|
|
184
|
+
|
|
185
|
+
/**
|
|
186
|
+
* Project nested tool-result blocks into one plain-text string: text blocks
|
|
187
|
+
* are joined, images and documents render as VCC-style labels.
|
|
188
|
+
* @param blocks - nested result content.
|
|
189
|
+
* @returns projected text.
|
|
190
|
+
*/
|
|
191
|
+
export function projectToolResultText(blocks) {
|
|
192
|
+
const parts = [];
|
|
193
|
+
// ⚠️ 2026-11-07 — tolerate a missing/non-array projection input and null
|
|
194
|
+
// blocks inside it. Both used to throw (`block.type` on null), and this helper
|
|
195
|
+
// sits on the tool-result, memory-recall and search paths alike.
|
|
196
|
+
if (!Array.isArray(blocks)) return "";
|
|
197
|
+
for (const block of blocks) {
|
|
198
|
+
if (block === null || block === undefined) continue;
|
|
199
|
+
if (block.type === "text") {
|
|
200
|
+
const text = sanitize(block.text ?? "");
|
|
201
|
+
if (text.length > 0) parts.push(text);
|
|
202
|
+
} else if (block.type === "image") parts.push("[image]");
|
|
203
|
+
else if (block.type === "document") parts.push("[document]");
|
|
204
|
+
else parts.push(`[${String(block.type)}]`);
|
|
205
|
+
}
|
|
206
|
+
return parts.join("\n");
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
/** Whether a message source identifies a landed compaction checkpoint node. */
|
|
210
|
+
export function isCheckpointSource(source) {
|
|
211
|
+
if (source === undefined || source === null) return false;
|
|
212
|
+
// Host 0.2.x builds the marker with `compactCheckpointSource()`, which yields
|
|
213
|
+
// `{ kind: "compact-checkpoint", compactionId, sourceCommandId? }`. The legacy
|
|
214
|
+
// `{ kind: "plugin", plugin: "compact" }` shape from earlier hosts is still
|
|
215
|
+
// accepted so checkpoints written by an older host stay recallable.
|
|
216
|
+
if (source.kind === "compact-checkpoint") return true;
|
|
217
|
+
return source.kind === "plugin" && source.plugin === "compact";
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
/**
|
|
221
|
+
* Whether a message source is one of the harness-injected instruction blocks
|
|
222
|
+
* the caller asked to elide (see `elideSourceKinds`). Matching is on the
|
|
223
|
+
* structured `source.kind` field the harness records, never on message text:
|
|
224
|
+
* a user quoting "Instructions from:" must not be mistaken for the real thing.
|
|
225
|
+
*
|
|
226
|
+
* The default (`undefined`/empty) matches nothing, so the open-source default
|
|
227
|
+
* behavior is byte-identical to before this option existed.
|
|
228
|
+
* @param source - message source descriptor, possibly absent.
|
|
229
|
+
* @param kinds - configured exact `source.kind` values to elide.
|
|
230
|
+
* @returns true when this node should collapse to a recall pointer.
|
|
231
|
+
*/
|
|
232
|
+
export function isElidableSource(source, kinds) {
|
|
233
|
+
if (kinds === undefined || kinds.length === 0) return false;
|
|
234
|
+
const kind = source?.kind;
|
|
235
|
+
return typeof kind === "string" && kinds.includes(kind);
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
/**
|
|
239
|
+
* Display labels for elidable instruction sources. Anything not listed falls
|
|
240
|
+
* back to its raw `source.kind`, so a new host source still gets a readable
|
|
241
|
+
* pointer without a code change.
|
|
242
|
+
*/
|
|
243
|
+
const ELIDABLE_SOURCE_LABELS = Object.freeze({
|
|
244
|
+
"dsh-mnemon": "记忆快照",
|
|
245
|
+
"agent-instructions": "规则与冷库索引",
|
|
246
|
+
"skill-catalog": "技能清单",
|
|
247
|
+
"runtime-context": "运行时上下文",
|
|
248
|
+
"time-context": "时间上下文"
|
|
249
|
+
});
|
|
250
|
+
|
|
251
|
+
/**
|
|
252
|
+
* Markers that open a hot-memory snapshot as our memory plugin renders it.
|
|
253
|
+
* `[MNEMON]` is the routing hint, `MNEMON RUNTIME MEMORY SNAPSHOT` the payload.
|
|
254
|
+
*/
|
|
255
|
+
const MEMORY_SNAPSHOT_MARKERS = Object.freeze(["MNEMON RUNTIME MEMORY SNAPSHOT", "[MNEMON]"]);
|
|
256
|
+
/** Closing tag of one memory file block inside a snapshot. */
|
|
257
|
+
const MEMORY_FILE_CLOSE = "</runtime-memory-file>";
|
|
258
|
+
/**
|
|
259
|
+
* Text that may separate two consecutive memory file blocks of ONE snapshot
|
|
260
|
+
* (`USER.md` then `MEMORY.md`), so the two are treated as a single span.
|
|
261
|
+
*/
|
|
262
|
+
const MEMORY_BLOCK_JOINER = /^\s*(?:MEMORY\.md \(working reference\)|USER\.md \(user profile\)|<runtime-memory-file )/;
|
|
263
|
+
/**
|
|
264
|
+
* One role header line of this compiler's own checkpoint format. Memory snapshots
|
|
265
|
+
* never contain one, so finding it between a file block's opening and its close
|
|
266
|
+
* means the close belongs to a LATER block and the span must stop before this.
|
|
267
|
+
* Guards the truncation case: a block cut mid-way loses its closing tag, and
|
|
268
|
+
* without this the span would swallow the conversation that follows.
|
|
269
|
+
*/
|
|
270
|
+
const MEMORY_ROLE_HEAD = /\n\[(?:user|assistant|tool)\]\s*\n/;
|
|
271
|
+
|
|
272
|
+
/**
|
|
273
|
+
* Marker text the host puts inside a REAL rules/index wrapper. Prose that only
|
|
274
|
+
* mentions `<system-reminder>` as a string must not qualify, so a candidate
|
|
275
|
+
* span has to prove itself by carrying one of these phrases early on.
|
|
276
|
+
*/
|
|
277
|
+
const SYSTEM_REMINDER_BODY_MARKERS = Object.freeze([
|
|
278
|
+
"The following workspace instructions may be relevant",
|
|
279
|
+
"Instructions from:"
|
|
280
|
+
]);
|
|
281
|
+
/** How far into a candidate wrapper the proof phrase must appear. */
|
|
282
|
+
const SYSTEM_REMINDER_PROOF_WINDOW = 600;
|
|
283
|
+
/** How close a memory marker must sit to the first file block to count. */
|
|
284
|
+
const MEMORY_MARKER_WINDOW = 800;
|
|
285
|
+
|
|
286
|
+
/**
|
|
287
|
+
* Locate injected memory snapshots embedded in a PRIOR checkpoint body.
|
|
288
|
+
*
|
|
289
|
+
* A snapshot is a run of consecutive `<runtime-memory-file …>…</…>` blocks.
|
|
290
|
+
* The span starts at the nearest preceding snapshot marker so the routing hint
|
|
291
|
+
* and the "Entry annotations default…" preamble go with it. The marker must sit
|
|
292
|
+
* close to the first file block, so prose that merely quotes the marker text
|
|
293
|
+
* (e.g. a discussion about this very feature) is not mistaken for a snapshot.
|
|
294
|
+
* @param text - prior checkpoint body.
|
|
295
|
+
* @returns `[start, end)` character spans, left to right.
|
|
296
|
+
*/
|
|
297
|
+
function findMemorySnapshotSpans(text) {
|
|
298
|
+
const spans = [];
|
|
299
|
+
let searchFrom = 0;
|
|
300
|
+
while (searchFrom < text.length) {
|
|
301
|
+
const markerAt = MEMORY_SNAPSHOT_MARKERS
|
|
302
|
+
.map((marker) => text.indexOf(marker, searchFrom))
|
|
303
|
+
.filter((index) => index !== -1)
|
|
304
|
+
.reduce((lowest, index) => (lowest === undefined || index < lowest ? index : lowest), undefined);
|
|
305
|
+
if (markerAt === undefined) break;
|
|
306
|
+
const firstFile = text.indexOf("<runtime-memory-file ", markerAt);
|
|
307
|
+
if (firstFile === -1) break;
|
|
308
|
+
if (firstFile - markerAt > MEMORY_MARKER_WINDOW) {
|
|
309
|
+
// Just prose quoting the marker — skip past it and keep looking.
|
|
310
|
+
searchFrom = markerAt + 1;
|
|
311
|
+
continue;
|
|
312
|
+
}
|
|
313
|
+
let end;
|
|
314
|
+
let cursor = firstFile;
|
|
315
|
+
for (;;) {
|
|
316
|
+
const close = text.indexOf(MEMORY_FILE_CLOSE, cursor);
|
|
317
|
+
if (close === -1) break;
|
|
318
|
+
// A memory block whose closing tag was truncated away has no close of its
|
|
319
|
+
// own, so the search below would run on to the NEXT block's close and
|
|
320
|
+
// swallow every real turn in between. Role headers are this compiler's own
|
|
321
|
+
// output and never appear inside a memory snapshot, so a role header before
|
|
322
|
+
// the close proves the span has left the block: stop there and keep the
|
|
323
|
+
// dialogue. Erring toward eliding less is safe; eliding a turn is not.
|
|
324
|
+
const roleHead = MEMORY_ROLE_HEAD.exec(text.slice(cursor, close));
|
|
325
|
+
if (roleHead !== null) {
|
|
326
|
+
end = cursor + roleHead.index;
|
|
327
|
+
break;
|
|
328
|
+
}
|
|
329
|
+
const after = close + MEMORY_FILE_CLOSE.length;
|
|
330
|
+
end = after;
|
|
331
|
+
cursor = after;
|
|
332
|
+
if (!MEMORY_BLOCK_JOINER.test(text.slice(after, after + 240))) break;
|
|
333
|
+
}
|
|
334
|
+
if (end === undefined) break;
|
|
335
|
+
spans.push([markerAt, end]);
|
|
336
|
+
searchFrom = end;
|
|
337
|
+
}
|
|
338
|
+
return spans;
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
/**
|
|
342
|
+
* Locate `<system-reminder>…</system-reminder>` wrappers that really frame
|
|
343
|
+
* workspace rules, the cold-store index or runtime context.
|
|
344
|
+
*
|
|
345
|
+
* The wrapper must open at a line start, be followed by the block body marker
|
|
346
|
+
* within a short window, and close before the body repeats — the host emits one
|
|
347
|
+
* wrapper per injection, never nested. A sentence that merely mentions the tag
|
|
348
|
+
* name fails all three checks, which is what keeps live prose intact.
|
|
349
|
+
* @param text - prior checkpoint body.
|
|
350
|
+
* @returns `[start, end)` character spans, left to right.
|
|
351
|
+
*/
|
|
352
|
+
function findSystemReminderSpans(text) {
|
|
353
|
+
const spans = [];
|
|
354
|
+
const pattern = /(?:^|\n)<system-reminder>\n/g;
|
|
355
|
+
let match;
|
|
356
|
+
while ((match = pattern.exec(text)) !== null) {
|
|
357
|
+
const open = text.indexOf("<system-reminder>", match.index);
|
|
358
|
+
const head = text.slice(open, open + SYSTEM_REMINDER_PROOF_WINDOW);
|
|
359
|
+
if (!SYSTEM_REMINDER_BODY_MARKERS.some((marker) => head.includes(marker))) continue;
|
|
360
|
+
const close = text.indexOf("</system-reminder>", open);
|
|
361
|
+
if (close === -1) continue;
|
|
362
|
+
spans.push([open, close + "</system-reminder>".length]);
|
|
363
|
+
}
|
|
364
|
+
return spans;
|
|
365
|
+
}
|
|
366
|
+
|
|
367
|
+
/**
|
|
368
|
+
* Collapse harness-injected bodies embedded in a PRIOR checkpoint to one-line
|
|
369
|
+
* pointers.
|
|
370
|
+
*
|
|
371
|
+
* Why this exists: those bodies are re-sent in full after every compaction, so
|
|
372
|
+
* any copy kept inside a checkpoint body is a duplicate — the checkpoint grows
|
|
373
|
+
* by one snapshot per compaction even though nothing new was said. The originals
|
|
374
|
+
* stay in the append-only session log, so `recall` still restores them verbatim.
|
|
375
|
+
*
|
|
376
|
+
* Only checkpoint prose is touched; live conversation text is never scanned.
|
|
377
|
+
* @param text - prior checkpoint body.
|
|
378
|
+
* @returns the rewritten body plus how many spans and characters were removed.
|
|
379
|
+
*/
|
|
380
|
+
export function elideCheckpointInjectedBodies(text) {
|
|
381
|
+
const spans = [...findMemorySnapshotSpans(text), ...findSystemReminderSpans(text)]
|
|
382
|
+
.sort((left, right) => left[0] - right[0]);
|
|
383
|
+
if (spans.length === 0) return { text, spans: 0, chars: 0 };
|
|
384
|
+
let out = "";
|
|
385
|
+
let cursor = 0;
|
|
386
|
+
let chars = 0;
|
|
387
|
+
let removed = 0;
|
|
388
|
+
for (const [start, end] of spans) {
|
|
389
|
+
// Overlapping spans would otherwise duplicate text; keep the outer one.
|
|
390
|
+
if (start < cursor) continue;
|
|
391
|
+
out += text.slice(cursor, start);
|
|
392
|
+
const length = end - start;
|
|
393
|
+
chars += length;
|
|
394
|
+
removed += 1;
|
|
395
|
+
out += `[注入内容已压缩为指针(原文 ${length} 字符)— 压缩后会重新注入,原文可用 recall 调回]`;
|
|
396
|
+
cursor = end;
|
|
397
|
+
}
|
|
398
|
+
out += text.slice(cursor);
|
|
399
|
+
return { text: out, spans: removed, chars };
|
|
400
|
+
}
|
|
401
|
+
|
|
402
|
+
/**
|
|
403
|
+
* One-line pointer standing in for an elided instruction block. Carries the
|
|
404
|
+
* seq so the agent can `recall` the original, plus what it was and how big it
|
|
405
|
+
* was — a bare `(seq N)` would read as content loss rather than a substitution.
|
|
406
|
+
* @param label - human name of the block ("记忆快照" / "规则与索引").
|
|
407
|
+
* @param seq - source sequence of the elided message.
|
|
408
|
+
* @param length - original character count.
|
|
409
|
+
* @returns the marker line.
|
|
410
|
+
*/
|
|
411
|
+
function elidedSourceMarker(label, seq, length) {
|
|
412
|
+
return `[${label} 已压缩为指针(${seqRef(seq)},原文 ${length} 字符)— 原文可用 recall 调回,且下一次压缩后会重新注入]`;
|
|
413
|
+
}
|
|
414
|
+
|
|
415
|
+
/**
|
|
416
|
+
* Head/tail excerpt of a tool result under one shared token budget. The tail
|
|
417
|
+
* is anchored to the end of the text at character granularity, so a result
|
|
418
|
+
* whose tail matters more than its head (file listings, logs) keeps its end.
|
|
419
|
+
* @param text - projected result text.
|
|
420
|
+
* @param budget - combined non-whitespace token budget for head + tail.
|
|
421
|
+
* @param ref - provenance for the elision marker.
|
|
422
|
+
* @returns the excerpt, or the full text when it already fits.
|
|
423
|
+
*/
|
|
424
|
+
export function excerptToolResult(text, budget, ref) {
|
|
425
|
+
const overBudget = countTokens(text) > budget || text.length > budget * 4;
|
|
426
|
+
if (budget <= 0 || !overBudget) return text;
|
|
427
|
+
const half = Math.floor(budget / 2);
|
|
428
|
+
const headLimit = Math.max(8, half);
|
|
429
|
+
const tailLimit = Math.max(8, budget - headLimit);
|
|
430
|
+
const head = truncateTokens(text, headLimit, ref);
|
|
431
|
+
const tail = tailTokens(text, tailLimit, ref);
|
|
432
|
+
return `${head.text}\n...(elided from ${ref})\n${tail.text}`;
|
|
433
|
+
}
|
|
434
|
+
|
|
435
|
+
// ── the region compiler ────────────────────────────────────────────────────
|
|
436
|
+
|
|
437
|
+
const CHECKPOINT_PREAMBLE = "This is an automatically generated checkpoint condensing an earlier span of the conversation to free up context. Treat the captured context as established background and build on it without restating it. Continue the task directly from the messages that follow, without acknowledging this checkpoint.";
|
|
438
|
+
const CHECKPOINT_OPEN_TAG = "<compacted-checkpoint>";
|
|
439
|
+
const CHECKPOINT_CLOSE_TAG = "</compacted-checkpoint>";
|
|
440
|
+
|
|
441
|
+
/**
|
|
442
|
+
* Model-facing guide framed into the head of every checkpoint: tells the
|
|
443
|
+
* agent how to recover content the compiler elided or truncated, using the
|
|
444
|
+
* two recall tools that read the append-only durable log.
|
|
445
|
+
*/
|
|
446
|
+
export const RECALL_GUIDE = "RECALL: append-only log — nothing is lost. `recall` restores original content: type \"seq\", id \"3-7\"; \"result\", id \"3\"; \"checkpoint\", id \"1\" (`[checkpoint N]` = dropped). `search` finds by keyword.";
|
|
447
|
+
|
|
448
|
+
/** Per-node reference marker: the durable seq is the lossless pointer. */
|
|
449
|
+
function seqRef(seq) {
|
|
450
|
+
return `seq ${seq}`;
|
|
451
|
+
}
|
|
452
|
+
|
|
453
|
+
/**
|
|
454
|
+
* Build marker for this compiler revision. Bumped whenever the compiled
|
|
455
|
+
* output format changes, so debug logs can prove which code version ran.
|
|
456
|
+
*/
|
|
457
|
+
export const COMPILER_REV = "vcc-compiler-debug-2";
|
|
458
|
+
|
|
459
|
+
/**
|
|
460
|
+
* Emit one debug line through the configured sink (and stderr). The sink is
|
|
461
|
+
* installed by the engine (`config.debugSink`) when `debug` is enabled; the
|
|
462
|
+
* compiler itself stays dependency-free.
|
|
463
|
+
* @param config - resolved compiler configuration.
|
|
464
|
+
* @param tag - short component tag for the line.
|
|
465
|
+
* @param line - the diagnostic text.
|
|
466
|
+
*/
|
|
467
|
+
function debugLog(config, tag, line) {
|
|
468
|
+
if (config.debug !== true) return;
|
|
469
|
+
const text = `[dsh-compaction-vcc] ${tag}: ${line}`;
|
|
470
|
+
try {
|
|
471
|
+
if (typeof config.debugSink === "function") config.debugSink(text);
|
|
472
|
+
} catch {
|
|
473
|
+
/* a failing debug sink must never break a compaction */
|
|
474
|
+
}
|
|
475
|
+
try {
|
|
476
|
+
console.error(text);
|
|
477
|
+
} catch {
|
|
478
|
+
/* stderr may be unavailable in embedded contexts */
|
|
479
|
+
}
|
|
480
|
+
}
|
|
481
|
+
|
|
482
|
+
/**
|
|
483
|
+
* Compile one ordered region into compact entries.
|
|
484
|
+
* @param nodes - ordered `{ seq, message }` projections of the shadowed
|
|
485
|
+
* surface nodes; `message` may be null (non-projecting node).
|
|
486
|
+
* @param config - resolved compiler-relevant configuration (see index.js).
|
|
487
|
+
* @param budgets - optional per-block budgets; defaults to the config values
|
|
488
|
+
* (the budget-cap loop passes scaled budgets on retry).
|
|
489
|
+
* @returns `{ entries, stats }` where each entry is one line-group string.
|
|
490
|
+
*/
|
|
491
|
+
export function compileNodes(nodes, config, budgets) {
|
|
492
|
+
const effective = budgets ?? {
|
|
493
|
+
textTokens: config.textTokens,
|
|
494
|
+
userTextTokens: config.userTextTokens,
|
|
495
|
+
toolCallTokens: config.toolCallTokens,
|
|
496
|
+
toolResultExcerptTokens: config.toolResultExcerptTokens
|
|
497
|
+
};
|
|
498
|
+
const keyFields = { ...DEFAULT_TOOL_KEY_FIELDS, ...(config.toolKeyFields ?? {}) };
|
|
499
|
+
const patterns = config.noisePatterns ?? [];
|
|
500
|
+
// Empty lists mean "unset": the cordis config pipeline injects `[]` for
|
|
501
|
+
// absent array keys, so fall back to the defaults instead of rendering
|
|
502
|
+
// every tool call name-only.
|
|
503
|
+
const argTools = config.toolArgTools?.length > 0 ? config.toolArgTools : DEFAULT_ARG_TOOLS;
|
|
504
|
+
const hiddenTools = new Set(config.hideTools ?? []);
|
|
505
|
+
debugLog(config, "compile", `rev=${COMPILER_REV} nodes=${nodes.length} argTools=[${argTools.join(",")}] hideTools=[${[...hiddenTools].join(",")}] budgets=${JSON.stringify(effective)}`);
|
|
506
|
+
const entries = [];
|
|
507
|
+
const stats = {
|
|
508
|
+
nodes: nodes.length,
|
|
509
|
+
entries: 0,
|
|
510
|
+
toolCalls: 0,
|
|
511
|
+
toolResults: 0,
|
|
512
|
+
images: 0,
|
|
513
|
+
documents: 0,
|
|
514
|
+
reasoningElided: 0,
|
|
515
|
+
noiseElided: 0,
|
|
516
|
+
elidedSourceRows: 0,
|
|
517
|
+
elidedSourceChars: 0,
|
|
518
|
+
elidedCheckpointSpans: 0,
|
|
519
|
+
elidedCheckpointChars: 0,
|
|
520
|
+
checkpoints: 0,
|
|
521
|
+
tokens: 0,
|
|
522
|
+
elidedToolRows: 0,
|
|
523
|
+
elidedRows: 0
|
|
524
|
+
};
|
|
525
|
+
// Pre-pass over the ordered nodes: map each tool-call id to the seq of its
|
|
526
|
+
// result node, so every call one-liner can carry a VCC-style result pointer
|
|
527
|
+
// (`(seq N -> result M)`) even though results no longer occupy entries.
|
|
528
|
+
//
|
|
529
|
+
// ⚠️ 2026-11-06 (this fork) — host 0.2.2 compatibility fix.
|
|
530
|
+
// Up to host ~0.1.x a derived tool-result message exposed `toolCallId` on its
|
|
531
|
+
// FIRST CONTENT BLOCK (`message.content[0].toolCallId`), and that is the only
|
|
532
|
+
// shape this pre-pass used to look at. Host 0.2.2 derives a tool message the
|
|
533
|
+
// way the model APIs do: `role: "tool"` with `toolCallId` on the MESSAGE
|
|
534
|
+
// (`message.toolCallId`), while `content[0]` is a plain text block. The old
|
|
535
|
+
// check therefore never matched, `resultSeqByCallId` stayed empty, and every
|
|
536
|
+
// `* read "a.js" (seq 2)` one-liner silently lost its `-> result 3` pointer —
|
|
537
|
+
// the hook `recall(type:"result")` relies on. Nothing threw; the checkpoint
|
|
538
|
+
// just quietly stopped being lossless. Read the message-level field first and
|
|
539
|
+
// keep the legacy `content[0]` shape as a fallback so one build serves both
|
|
540
|
+
// host generations.
|
|
541
|
+
const resultSeqByCallId = new Map();
|
|
542
|
+
for (const node of nodes) {
|
|
543
|
+
const message = node.message;
|
|
544
|
+
if (message === null || message === undefined) continue;
|
|
545
|
+
const first = message.content?.[0];
|
|
546
|
+
// ⚠️ 2026-11-07 — `first !== undefined` misses null, which threw on
|
|
547
|
+
// `first.type`. Guard the null block explicitly.
|
|
548
|
+
const toolResultBlock = first !== undefined && first !== null && first.type === "tool-result" ? first : undefined;
|
|
549
|
+
const callId = typeof message.toolCallId === "string"
|
|
550
|
+
? message.toolCallId
|
|
551
|
+
: typeof toolResultBlock?.toolCallId === "string"
|
|
552
|
+
? toolResultBlock.toolCallId
|
|
553
|
+
: undefined;
|
|
554
|
+
if (callId !== undefined) resultSeqByCallId.set(callId, node.seq);
|
|
555
|
+
}
|
|
556
|
+
let lastRole;
|
|
557
|
+
let open = false;
|
|
558
|
+
|
|
559
|
+
/**
|
|
560
|
+
* Append one compiled entry carrying a deletion-priority kind. Kinds order
|
|
561
|
+
* the cap-elision passes in compileRegion: `result`/`tool`/`media`/`note`
|
|
562
|
+
* rows are dropped before `text`/`reasoning`/`checkpoint` rows, so when a
|
|
563
|
+
* checkpoint must shrink, the conversation's words survive its logs.
|
|
564
|
+
* @param seq - durable seq the entry derives from.
|
|
565
|
+
* @param text - rendered entry text.
|
|
566
|
+
* @param kind - entry kind (see the kind list above).
|
|
567
|
+
*/
|
|
568
|
+
const pushEntry = (seq, text, kind) => {
|
|
569
|
+
entries.push({ seq, text, kind });
|
|
570
|
+
stats.entries += 1;
|
|
571
|
+
stats.tokens += estimateEntryTokens(text);
|
|
572
|
+
};
|
|
573
|
+
const roleHeader = (role, seq) => {
|
|
574
|
+
if (!open || lastRole !== role) {
|
|
575
|
+
open = true;
|
|
576
|
+
lastRole = role;
|
|
577
|
+
return `[${role}]\n`;
|
|
578
|
+
}
|
|
579
|
+
return "";
|
|
580
|
+
};
|
|
581
|
+
|
|
582
|
+
for (const node of nodes) {
|
|
583
|
+
const message = node.message;
|
|
584
|
+
// ⚠️ 2026-11-07 — `content` must be an iterable array here. A null/absent
|
|
585
|
+
// `content` used to fall through to `message.content[0]` below and throw
|
|
586
|
+
// `Cannot read properties of null (reading '0')`, which aborted the WHOLE
|
|
587
|
+
// compile: one malformed node cost the entire compaction, silently (the
|
|
588
|
+
// host's pre-step handler swallows the error as a warning). Skip the node
|
|
589
|
+
// instead so the rest of the region still compiles.
|
|
590
|
+
if (message === null || message === undefined) continue;
|
|
591
|
+
if (!Array.isArray(message.content)) continue;
|
|
592
|
+
if (message.role === "assistant") {
|
|
593
|
+
let header = "";
|
|
594
|
+
for (const block of message.content) {
|
|
595
|
+
// ⚠️ 2026-11-07 — a null/undefined block in the array used to throw on
|
|
596
|
+
// `block.type` and abort the whole compile. Skip it.
|
|
597
|
+
if (block === null || block === undefined) continue;
|
|
598
|
+
if (block.type === "text") {
|
|
599
|
+
const text = sanitize(block.text ?? "");
|
|
600
|
+
if (text.trim().length === 0) continue;
|
|
601
|
+
header = roleHeader("assistant", node.seq);
|
|
602
|
+
const kept = truncateTokens(text, effective.textTokens, seqRef(node.seq));
|
|
603
|
+
pushEntry(node.seq, header + kept.text, "text");
|
|
604
|
+
continue;
|
|
605
|
+
}
|
|
606
|
+
if (block.type === "reasoning") {
|
|
607
|
+
if (config.includeReasoning !== true) {
|
|
608
|
+
stats.reasoningElided += 1;
|
|
609
|
+
continue;
|
|
610
|
+
}
|
|
611
|
+
const text = sanitize(block.text ?? "");
|
|
612
|
+
if (text.trim().length === 0) continue;
|
|
613
|
+
header = roleHeader("assistant", node.seq);
|
|
614
|
+
const kept = truncateTokens(text, effective.textTokens, seqRef(node.seq));
|
|
615
|
+
pushEntry(node.seq, header + kept.text, "reasoning");
|
|
616
|
+
continue;
|
|
617
|
+
}
|
|
618
|
+
if (block.type === "tool-call") {
|
|
619
|
+
const name = block.name ?? "unknown";
|
|
620
|
+
stats.toolCalls += 1;
|
|
621
|
+
if (hiddenTools.has(name)) {
|
|
622
|
+
debugLog(config, "tool", `seq=${node.seq} name=${name} HIDDEN (hideTools)`);
|
|
623
|
+
continue;
|
|
624
|
+
}
|
|
625
|
+
header = roleHeader("assistant", node.seq);
|
|
626
|
+
// One line, always: key argument only for whitelisted tools, and a
|
|
627
|
+
// VCC-style pointer to the result node (`* read "a.js" (seq 2 ->
|
|
628
|
+
// result 3)`) so the dropped result stays one recall away.
|
|
629
|
+
let oneLine;
|
|
630
|
+
let diag;
|
|
631
|
+
if (argTools.includes(name)) {
|
|
632
|
+
const parsed = parseToolArguments(block.arguments);
|
|
633
|
+
const arg = pickToolKeyArg(name, parsed, keyFields);
|
|
634
|
+
oneLine = arg === undefined ? `* ${name}` : `* ${name} "${oneLineArg(arg)}"`;
|
|
635
|
+
diag = `whitelist=yes argsType=${typeof block.arguments} argsLen=${block.arguments === null || block.arguments === undefined ? 0 : String(block.arguments).length} argsHead=${JSON.stringify(String(block.arguments).slice(0, 60))} parse=${parsed === null ? "FAIL" : "ok"} key=${keyFields[name] ?? "(fallback)"} arg=${arg === undefined ? "(none)" : JSON.stringify(String(arg).slice(0, 60))}`;
|
|
636
|
+
} else {
|
|
637
|
+
oneLine = `* ${name}`;
|
|
638
|
+
diag = `whitelist=no argsType=${typeof block.arguments} argsLen=${block.arguments === null || block.arguments === undefined ? 0 : String(block.arguments).length} argsHead=${JSON.stringify(String(block.arguments).slice(0, 60))}`;
|
|
639
|
+
}
|
|
640
|
+
const resultSeq = resultSeqByCallId.get(block.id);
|
|
641
|
+
const ref = resultSeq === undefined ? seqRef(node.seq) : `${seqRef(node.seq)} -> result ${resultSeq}`;
|
|
642
|
+
const kept = truncateTokens(oneLine, effective.toolCallTokens, seqRef(node.seq));
|
|
643
|
+
debugLog(config, "tool", `seq=${node.seq} name=${name} ${diag} line=${JSON.stringify(oneLine.slice(0, 80))} truncated=${kept.truncated} ref=${ref}`);
|
|
644
|
+
pushEntry(node.seq, header + `${kept.text} (${ref})`, "tool");
|
|
645
|
+
continue;
|
|
646
|
+
}
|
|
647
|
+
if (block.type === "image") {
|
|
648
|
+
stats.images += 1;
|
|
649
|
+
header = roleHeader("assistant", node.seq);
|
|
650
|
+
pushEntry(node.seq, header + `[image] (${seqRef(node.seq)})`, "media");
|
|
651
|
+
continue;
|
|
652
|
+
}
|
|
653
|
+
if (block.type === "document") {
|
|
654
|
+
stats.documents += 1;
|
|
655
|
+
header = roleHeader("assistant", node.seq);
|
|
656
|
+
pushEntry(node.seq, header + `[document] (${seqRef(node.seq)})`, "media");
|
|
657
|
+
continue;
|
|
658
|
+
}
|
|
659
|
+
/* unknown merge-extensible block types render as labels */
|
|
660
|
+
{
|
|
661
|
+
header = roleHeader("assistant", node.seq);
|
|
662
|
+
pushEntry(node.seq, header + `[${String(block.type)}] (${seqRef(node.seq)})`, "media");
|
|
663
|
+
|
|
664
|
+
}
|
|
665
|
+
}
|
|
666
|
+
/* reasoning-only and empty assistant messages produce no entry */
|
|
667
|
+
continue;
|
|
668
|
+
}
|
|
669
|
+
// ⚠️ 2026-11-07 — host 0.2.x derives a tool result as `role: "tool"` with
|
|
670
|
+
// `toolCallId` on the MESSAGE and a plain text block as `content[0]`. The
|
|
671
|
+
// loop used to handle only `assistant` and `user`, so every real tool result
|
|
672
|
+
// fell through and produced no entry: `stats.toolResults` stayed at 0, the
|
|
673
|
+
// `toolResultExcerptTokens` budget was inert, and the tool output never
|
|
674
|
+
// appeared in the checkpoint at all. Only the `-> result N` pointer survived
|
|
675
|
+
// (that map is built by the separate pre-pass above), so `recall` could still
|
|
676
|
+
// fetch the original — but the compiled view was misdescribed as "lossless"
|
|
677
|
+
// while silently dropping this whole class of content.
|
|
678
|
+
if (message.role === "tool") {
|
|
679
|
+
stats.toolResults += 1;
|
|
680
|
+
const body = projectToolResultText(message.content);
|
|
681
|
+
if (body.length > 0) {
|
|
682
|
+
const header = roleHeader("tool", node.seq);
|
|
683
|
+
// Excerpt to the configured budget: tool output is the bulk of a long
|
|
684
|
+
// session, and the full text stays one `recall(type:"result")` away via
|
|
685
|
+
// the call line's `-> result N` pointer.
|
|
686
|
+
const kept = excerptToolResult(body, effective.toolResultExcerptTokens, seqRef(node.seq));
|
|
687
|
+
pushEntry(node.seq, header + kept, "result");
|
|
688
|
+
}
|
|
689
|
+
continue;
|
|
690
|
+
}
|
|
691
|
+
if (message.role === "user") {
|
|
692
|
+
const first = message.content[0];
|
|
693
|
+
if (first !== undefined && first !== null && first.type === "tool-result") {
|
|
694
|
+
// Tool results never occupy an entry: the content is either absorbed
|
|
695
|
+
// into the following assistant text or one recall away via the call
|
|
696
|
+
// line's `-> result N` pointer (VCC drops results from brief mode too).
|
|
697
|
+
stats.toolResults += 1;
|
|
698
|
+
continue;
|
|
699
|
+
}
|
|
700
|
+
if (isCheckpointSource(message.source)) {
|
|
701
|
+
stats.checkpoints += 1;
|
|
702
|
+
let text = projectToolResultText(message.content);
|
|
703
|
+
if (text.length > 0 && config.elideCheckpointInjectedBodies === true) {
|
|
704
|
+
// A prior checkpoint body still carries every memory snapshot and
|
|
705
|
+
// rules block that was live when it was compiled. Those are re-sent
|
|
706
|
+
// in full after this compaction, so keeping them here is a duplicate.
|
|
707
|
+
// Opt-in (default off) because it rewrites previously compiled prose.
|
|
708
|
+
const collapsed = elideCheckpointInjectedBodies(text);
|
|
709
|
+
if (collapsed.spans > 0) {
|
|
710
|
+
text = collapsed.text;
|
|
711
|
+
stats.elidedCheckpointSpans += collapsed.spans;
|
|
712
|
+
stats.elidedCheckpointChars += collapsed.chars;
|
|
713
|
+
}
|
|
714
|
+
}
|
|
715
|
+
if (text.length > 0) {
|
|
716
|
+
// The checkpoint node is a user/message in the durable protocol
|
|
717
|
+
// (surface replacement only allows message nodes), but it is
|
|
718
|
+
// harness-generated framing — display it as [system].
|
|
719
|
+
const header = roleHeader("system", node.seq);
|
|
720
|
+
pushEntry(node.seq, header + text, "checkpoint");
|
|
721
|
+
}
|
|
722
|
+
continue;
|
|
723
|
+
}
|
|
724
|
+
if (isElidableSource(message.source, config.elideSourceKinds)) {
|
|
725
|
+
// Harness-injected instruction blocks (hot memory, workspace rules,
|
|
726
|
+
// skill catalog…) are re-injected in full after every compaction, so
|
|
727
|
+
// keeping their body here duplicates them: one copy in the checkpoint,
|
|
728
|
+
// one copy re-injected, every time. Collapse to a recall pointer.
|
|
729
|
+
// `elideSourceKinds` is empty by default, so stock behavior is
|
|
730
|
+
// unchanged; we enable it because our memory plugin re-injects.
|
|
731
|
+
const text = projectToolResultText(message.content);
|
|
732
|
+
if (text.length === 0) continue;
|
|
733
|
+
const label = ELIDABLE_SOURCE_LABELS[message.source.kind] ?? message.source.kind;
|
|
734
|
+
stats.elidedSourceRows += 1;
|
|
735
|
+
stats.elidedSourceChars += text.length;
|
|
736
|
+
pushEntry(node.seq, `${roleHeader("user", node.seq)}* ${elidedSourceMarker(label, node.seq, text.length)}`, "note");
|
|
737
|
+
continue;
|
|
738
|
+
}
|
|
739
|
+
let surviving = 0;
|
|
740
|
+
let header = "";
|
|
741
|
+
for (const block of message.content) {
|
|
742
|
+
// ⚠️ 2026-11-07 — a null/undefined block in the array used to throw on
|
|
743
|
+
// `block.type` and abort the whole compile. Skip it.
|
|
744
|
+
if (block === null || block === undefined) continue;
|
|
745
|
+
if (block.type === "text") {
|
|
746
|
+
let text = sanitize(block.text ?? "");
|
|
747
|
+
if (config.stripNoiseXml !== false) text = stripNoiseXml(text, patterns);
|
|
748
|
+
if (text.length === 0) continue;
|
|
749
|
+
surviving += 1;
|
|
750
|
+
header = roleHeader("user", node.seq);
|
|
751
|
+
const kept = truncateTokens(text, effective.userTextTokens, seqRef(node.seq));
|
|
752
|
+
pushEntry(node.seq, header + kept.text, "text");
|
|
753
|
+
} else if (block.type === "image") {
|
|
754
|
+
stats.images += 1;
|
|
755
|
+
surviving += 1;
|
|
756
|
+
header = roleHeader("user", node.seq);
|
|
757
|
+
pushEntry(node.seq, header + `[image] (${seqRef(node.seq)})`, "media");
|
|
758
|
+
} else if (block.type === "document") {
|
|
759
|
+
stats.documents += 1;
|
|
760
|
+
surviving += 1;
|
|
761
|
+
header = roleHeader("user", node.seq);
|
|
762
|
+
pushEntry(node.seq, header + `[document] (${seqRef(node.seq)})`, "media");
|
|
763
|
+
} else {
|
|
764
|
+
const text = sanitize(block.text ?? "");
|
|
765
|
+
if (text.length === 0) continue;
|
|
766
|
+
surviving += 1;
|
|
767
|
+
header = roleHeader("user", node.seq);
|
|
768
|
+
const kept = truncateTokens(text, effective.userTextTokens, seqRef(node.seq));
|
|
769
|
+
pushEntry(node.seq, header + kept.text, "text");
|
|
770
|
+
}
|
|
771
|
+
}
|
|
772
|
+
if (surviving === 0) {
|
|
773
|
+
stats.noiseElided += 1;
|
|
774
|
+
pushEntry(node.seq, `${roleHeader("user", node.seq)}* [user message elided: noise-only] (${seqRef(node.seq)})`, "note");
|
|
775
|
+
}
|
|
776
|
+
}
|
|
777
|
+
}
|
|
778
|
+
return { entries, stats };
|
|
779
|
+
}
|
|
780
|
+
|
|
781
|
+
/**
|
|
782
|
+
* Per-budget elision floors: conversation text keeps a meaningful minimum
|
|
783
|
+
* even when the total cap forces heavy rescaling, while tool one-liners and
|
|
784
|
+
* result excerpts may collapse to near-nothing first.
|
|
785
|
+
*/
|
|
786
|
+
const BUDGET_FLOORS = Object.freeze({
|
|
787
|
+
textTokens: 32,
|
|
788
|
+
userTextTokens: 32,
|
|
789
|
+
toolCallTokens: 8,
|
|
790
|
+
toolResultExcerptTokens: 8
|
|
791
|
+
});
|
|
792
|
+
|
|
793
|
+
/** Entry kinds dropped by the first (low-value) elision pass. */
|
|
794
|
+
const LOW_VALUE_KINDS = new Set(["result", "tool", "media", "note"]);
|
|
795
|
+
|
|
796
|
+
/**
|
|
797
|
+
* Scale ONLY the conversation-text budgets toward a total cap, never below
|
|
798
|
+
* the per-key floor. Tool-call rows stay one-liners at their configured
|
|
799
|
+
* budget: conversation shrinks first, tool rows are sacrificed whole by the
|
|
800
|
+
* elision pass, so tool calls can never squeeze the dialogue out.
|
|
801
|
+
* @param budgets - budget map to scale.
|
|
802
|
+
* @param factor - uniform scale factor for text budgets.
|
|
803
|
+
* @returns the scaled budget map (tool budgets untouched).
|
|
804
|
+
*/
|
|
805
|
+
function scaleBudgets(budgets, factor) {
|
|
806
|
+
const scaled = { ...budgets };
|
|
807
|
+
for (const key of ["textTokens", "userTextTokens"]) {
|
|
808
|
+
scaled[key] = Math.max(BUDGET_FLOORS[key] ?? 8, Math.floor(budgets[key] * factor));
|
|
809
|
+
}
|
|
810
|
+
return scaled;
|
|
811
|
+
}
|
|
812
|
+
|
|
813
|
+
/** Render the provenance range of one dropped-entry list. */
|
|
814
|
+
function seqRangeOf(dropped) {
|
|
815
|
+
const first = dropped[0].seq;
|
|
816
|
+
const last = dropped[dropped.length - 1].seq;
|
|
817
|
+
return first === last ? `seq ${first}` : `seqs ${first}-${last}`;
|
|
818
|
+
}
|
|
819
|
+
|
|
820
|
+
/**
|
|
821
|
+
* Compile a region to completion under the total checkpoint cap: budgets are
|
|
822
|
+
* rescaled toward `config.maxTokens` up to a fixed number of attempts, then
|
|
823
|
+
* the oldest low-value rows (tool/result/media/note) are front-elided first,
|
|
824
|
+
* and only then the oldest remaining entries — so the conversation's words
|
|
825
|
+
* survive elision before its logs do. The newest entries always survive —
|
|
826
|
+
* recency is the value ranking, as in VCC chunking.
|
|
827
|
+
* @param nodes - ordered `{ seq, message }` projections.
|
|
828
|
+
* @param config - resolved compiler configuration.
|
|
829
|
+
* @returns `{ entries, stats, capped }` — `capped` records cap enforcement.
|
|
830
|
+
*/
|
|
831
|
+
export function compileRegion(nodes, config) {
|
|
832
|
+
const baseBudgets = {
|
|
833
|
+
textTokens: config.textTokens,
|
|
834
|
+
userTextTokens: config.userTextTokens,
|
|
835
|
+
toolCallTokens: config.toolCallTokens,
|
|
836
|
+
toolResultExcerptTokens: config.toolResultExcerptTokens
|
|
837
|
+
};
|
|
838
|
+
let budgets = baseBudgets;
|
|
839
|
+
let result = compileNodes(nodes, config, budgets);
|
|
840
|
+
let attempts = 0;
|
|
841
|
+
const maxAttempts = 4;
|
|
842
|
+
while (result.stats.tokens > config.maxTokens && attempts < maxAttempts) {
|
|
843
|
+
budgets = scaleBudgets(budgets, config.maxTokens / result.stats.tokens * 0.95);
|
|
844
|
+
result = compileNodes(nodes, config, budgets);
|
|
845
|
+
attempts += 1;
|
|
846
|
+
}
|
|
847
|
+
let capped = false;
|
|
848
|
+
if (result.stats.tokens > config.maxTokens) {
|
|
849
|
+
const entries = result.entries;
|
|
850
|
+
// Phase A: drop the oldest low-value rows (tool/result/media/note)
|
|
851
|
+
// anywhere in the list first, so conversation text and prior-checkpoint
|
|
852
|
+
// knowledge survive elision before the logs do. The marker's seq range is
|
|
853
|
+
// the dropped rows' span (approximate: surviving text may sit inside it).
|
|
854
|
+
const toolRows = [];
|
|
855
|
+
while (entries.length > 1 && result.stats.tokens > config.maxTokens) {
|
|
856
|
+
const index = entries.findIndex((entry, position) => position < entries.length - 1 && LOW_VALUE_KINDS.has(entry.kind));
|
|
857
|
+
if (index === -1) break;
|
|
858
|
+
const dropped = entries.splice(index, 1)[0];
|
|
859
|
+
result.stats.tokens -= estimateEntryTokens(dropped.text);
|
|
860
|
+
toolRows.push(dropped);
|
|
861
|
+
}
|
|
862
|
+
if (toolRows.length > 0) {
|
|
863
|
+
capped = true;
|
|
864
|
+
result.stats.elidedToolRows += toolRows.length;
|
|
865
|
+
entries.unshift({ seq: toolRows[0].seq, text: `[${toolRows.length} tool/result entries elided: ${seqRangeOf(toolRows)}]`, kind: "note" });
|
|
866
|
+
result.stats.tokens += estimateEntryTokens(entries[0].text);
|
|
867
|
+
}
|
|
868
|
+
const elided = [];
|
|
869
|
+
const droppedCheckpoints = [];
|
|
870
|
+
while (entries.length > 1 && result.stats.tokens > config.maxTokens) {
|
|
871
|
+
const dropped = entries.shift();
|
|
872
|
+
result.stats.tokens -= estimateEntryTokens(dropped.text);
|
|
873
|
+
elided.push(dropped);
|
|
874
|
+
if (dropped.kind === "checkpoint") droppedCheckpoints.push(dropped);
|
|
875
|
+
}
|
|
876
|
+
if (elided.length > 0) {
|
|
877
|
+
capped = true;
|
|
878
|
+
result.stats.elidedRows += elided.length;
|
|
879
|
+
// Dropped prior checkpoints never vanish silently: each leaves a
|
|
880
|
+
// single `[checkpoint N]` line the agent can restore in full via
|
|
881
|
+
// recall(type:"checkpoint", id:"N").
|
|
882
|
+
const markerLines = [`[${elided.length} earlier entries elided: ${seqRangeOf(elided)}]`];
|
|
883
|
+
for (const dropped of droppedCheckpoints) {
|
|
884
|
+
const ordinal = config.checkpointOrdinals?.get(dropped.seq);
|
|
885
|
+
const ref = ordinal === undefined ? `@ seq ${dropped.seq}` : String(ordinal);
|
|
886
|
+
markerLines.push(`[checkpoint ${ref}]`);
|
|
887
|
+
}
|
|
888
|
+
entries.unshift({ seq: elided[0].seq, text: markerLines.join("\n"), kind: "note" });
|
|
889
|
+
result.stats.tokens += estimateEntryTokens(entries[0].text);
|
|
890
|
+
}
|
|
891
|
+
if (result.stats.tokens > config.maxTokens && entries.length === 1) {
|
|
892
|
+
capped = true;
|
|
893
|
+
const hard = truncateTokens(entries[0].text, Math.max(8, config.maxTokens - 4), "total cap");
|
|
894
|
+
entries[0] = { seq: entries[0].seq, text: hard.text, kind: entries[0].kind };
|
|
895
|
+
result.stats.tokens = estimateEntryTokens(entries[0].text);
|
|
896
|
+
}
|
|
897
|
+
}
|
|
898
|
+
return { ...result, capped };
|
|
899
|
+
}
|
|
900
|
+
|
|
901
|
+
/**
|
|
902
|
+
* Normalize one compiled entry to its text: entries are `{ seq, text }`
|
|
903
|
+
* pairs, but a custom compile hook may supply plain strings.
|
|
904
|
+
* @param entry - entry value.
|
|
905
|
+
* @returns the entry's text.
|
|
906
|
+
*/
|
|
907
|
+
export function entryText(entry) {
|
|
908
|
+
return typeof entry === "string" ? entry : entry.text;
|
|
909
|
+
}
|
|
910
|
+
|
|
911
|
+
/**
|
|
912
|
+
* Join compiled entries with separators that keep tool runs compact:
|
|
913
|
+
* consecutive `tool` rows join with a single newline (no blank line), while
|
|
914
|
+
* every other boundary keeps the blank line that separates entry groups.
|
|
915
|
+
* Strings (guide, header, elision markers) behave like `text` entries.
|
|
916
|
+
* @param entries - ordered entry texts (strings or `{ seq, text, kind }` pairs).
|
|
917
|
+
* @returns the joined body text.
|
|
918
|
+
*/
|
|
919
|
+
export function joinCompiledEntries(entries) {
|
|
920
|
+
let out = "";
|
|
921
|
+
let previousKind;
|
|
922
|
+
for (const entry of entries) {
|
|
923
|
+
const kind = typeof entry === "string" ? "text" : entry.kind;
|
|
924
|
+
if (out.length > 0) out += previousKind === "tool" && kind === "tool" ? "\n" : "\n\n";
|
|
925
|
+
out += typeof entry === "string" ? entry : entry.text;
|
|
926
|
+
previousKind = kind;
|
|
927
|
+
}
|
|
928
|
+
return out;
|
|
929
|
+
}
|
|
930
|
+
|
|
931
|
+
/**
|
|
932
|
+
* Frame compiled entries as the durable replacement checkpoint content:
|
|
933
|
+
* the shared preamble, the recall guide, then one tagged block (guide first,
|
|
934
|
+
* then the optional intro line, the optional header line, the entries, and
|
|
935
|
+
* the optional retention footer). Prior-checkpoint text copied verbatim
|
|
936
|
+
* (with its own backend's tags) nests inside harmlessly.
|
|
937
|
+
* @param entries - ordered entry texts (strings or `{ seq, text }` pairs).
|
|
938
|
+
* @param headerLine - optional first line summarizing the compiled region.
|
|
939
|
+
* @param introLine - optional first line summarizing the compaction itself.
|
|
940
|
+
* @param footerLine - optional tail line reporting the verbatim retention.
|
|
941
|
+
* @returns plain text content blocks.
|
|
942
|
+
*/
|
|
943
|
+
export function frameCheckpoint(entries, headerLine, introLine, footerLine) {
|
|
944
|
+
const body = [RECALL_GUIDE];
|
|
945
|
+
if (introLine !== undefined) body.push(introLine);
|
|
946
|
+
if (headerLine !== undefined) body.push(headerLine);
|
|
947
|
+
body.push(...entries);
|
|
948
|
+
// `footerLine` accepts one line or an ordered list, so callers can append
|
|
949
|
+
// several trailing notes (cap notice + verbatim retention) without
|
|
950
|
+
// concatenating them into one unreadable paragraph.
|
|
951
|
+
if (footerLine !== undefined) {
|
|
952
|
+
if (Array.isArray(footerLine)) body.push(...footerLine.filter((line) => line !== undefined));
|
|
953
|
+
else body.push(footerLine);
|
|
954
|
+
}
|
|
955
|
+
return [
|
|
956
|
+
{ type: "text", text: `${CHECKPOINT_PREAMBLE}\n\n${CHECKPOINT_OPEN_TAG}` },
|
|
957
|
+
{ type: "text", text: joinCompiledEntries(body) },
|
|
958
|
+
{ type: "text", text: CHECKPOINT_CLOSE_TAG }
|
|
959
|
+
];
|
|
960
|
+
}
|
|
961
|
+
|
|
962
|
+
export { CHECKPOINT_CLOSE_TAG, CHECKPOINT_OPEN_TAG, CHECKPOINT_PREAMBLE };
|