opencode-vcc-compaction 0.0.0-stage → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,962 @@
1
+ /**
2
+ * Region compiler, ported from dsh-compaction-vcc@0.7.4 `src/compiler.js`
3
+ * (MIT: Copyright (c) 2026 TsFreddie, Copyright (c) 2026 Tingfeng) —
4
+ * everything from the noise-stripping section through checkpoint framing.
5
+ * The tokenizer half lives in `./tokenize.mjs` and is imported below,
6
+ * never re-defined.
7
+ *
8
+ * Faithful port with three documented deviations:
9
+ * - Budget primitives (`sanitize`, `countTokens`, `truncateTokens`,
10
+ * `tailTokens`, `estimateEntryTokens`) are imported from `./tokenize.mjs`
11
+ * (module-private upstream; exported there so this sibling stage can
12
+ * import them across the file split).
13
+ * - DSH harness vocabulary is carried verbatim as configurable defaults, not
14
+ * forked: `DEFAULT_NOISE_PATTERNS`, `DEFAULT_TOOL_KEY_FIELDS`,
15
+ * `DEFAULT_ARG_TOOLS`, `ELIDABLE_SOURCE_LABELS`, the memory-snapshot /
16
+ * system-reminder span finders, and `isCheckpointSource` kinds all describe
17
+ * the DSH host. The OpenCode mapping lives in config (`noisePatterns`,
18
+ * `toolKeyFields`, `toolArgTools`, `hideTools`, `elideSourceKinds`) and in
19
+ * the node-projection adapter (hook `Message` + `SessionMessage.ID` ->
20
+ * `{ seq, message }`), which is a later stage per `archive/plans/vcc-compiler.md`.
21
+ * - `elideCheckpointInjectedBodies` is retained verbatim but stays inert
22
+ * unless `config.elideCheckpointInjectedBodies === true`; the OpenCode
23
+ * consumer leaves it off (REUSE.md rejects it — the preamble here is
24
+ * per-request transient, never persisted history).
25
+ *
26
+ * Note on style: this file keeps upstream's semicolons verbatim for
27
+ * line-for-line comparability with the MIT source (present and future
28
+ * re-sync). This coincides with the repo's own `.mjs` style
29
+ * (`compiler/selftest.mjs`, `scripts/check-engine-sync.mjs` both use
30
+ * semicolons), so there is no style conflict in practice.
31
+ *
32
+ * Everything below is deterministic and offline: same nodes + config always
33
+ * yield byte-identical entries, with no model call and no dependencies.
34
+ */
35
+ import {
36
+ countTokens,
37
+ estimateEntryTokens,
38
+ sanitize,
39
+ tailTokens,
40
+ truncateTokens,
41
+ } from "./tokenize.mjs";
42
+
43
+ // ── noise stripping (VCC `_BRIEF_STRIP_RE` family) ─────────────────────────
44
+
45
+ const NOISE_UNWRAP_RE = /<\/?(?:command-name|command-args)>/g;
46
+
47
+ /**
48
+ * The VCC brief-mode noise XML patterns, adapted to harness markup. Applied
49
+ * with the `s` flag; the replacement set is configurable via
50
+ * `InstantCompactionConfig.noisePatterns`.
51
+ */
52
+ export const DEFAULT_NOISE_PATTERNS = [
53
+ "<ide_opened_file>.*?</ide_opened_file>",
54
+ "<ide_selection>.*?</ide_selection>",
55
+ "<system-reminder>.*?</system-reminder>",
56
+ "<command-message>.*?</command-message>",
57
+ "<task-notification>.*?</task-notification>"
58
+ ];
59
+
60
+ /** Compile untrusted noise-pattern sources, throwing on invalid regex. */
61
+ export function compileNoisePatterns(sources) {
62
+ return sources.map((source, index) => {
63
+ try {
64
+ return new RegExp(source, "s");
65
+ } catch (error) {
66
+ throw new Error(`noisePatterns[${index}] (${JSON.stringify(source)}) is not a valid regular expression: ${error instanceof Error ? error.message : String(error)}`);
67
+ }
68
+ });
69
+ }
70
+
71
+ /**
72
+ * Strip configured noise XML blocks from user text and unwrap command-name /
73
+ * command-args tags (VCC `_strip_noise_xml` + `_BRIEF_UNWRAP_RE`).
74
+ * @param text - user text.
75
+ * @param patterns - compiled noise regexes.
76
+ * @returns the stripped text, trimmed.
77
+ */
78
+ export function stripNoiseXml(text, patterns) {
79
+ let out = text;
80
+ for (const pattern of patterns) out = out.replace(pattern, "");
81
+ out = out.replace(NOISE_UNWRAP_RE, "");
82
+ return out.trim();
83
+ }
84
+
85
+ // ── tool-call one-liner (VCC `_tool_summary`) ──────────────────────────────
86
+
87
+ /** VCC's `_TOOL_SUMMARY_FIELDS`, extended for the DSH tool catalog. */
88
+ export const DEFAULT_TOOL_KEY_FIELDS = Object.freeze({
89
+ read: "file_path",
90
+ edit: "file_path",
91
+ write: "file_path",
92
+ glob: "pattern",
93
+ grep: "pattern",
94
+ bash: "command",
95
+ web_search: "query",
96
+ skill: "name",
97
+ ralph: "objective",
98
+ subagent: "description",
99
+ subagent_fork: "description",
100
+ create_goal: "objective",
101
+ update_goal: "objective",
102
+ workflow: "name",
103
+ interrupt_agent: "agent_id",
104
+ job_kill: "job_id",
105
+ job_output: "job_id",
106
+ send_message: "message"
107
+ });
108
+
109
+ /**
110
+ * Tools whose key argument is kept in the one-liner. Everything else renders
111
+ * name-only (`* tool (seq N)`) — bookkeeping tools carry no semantic payload
112
+ * worth the tokens (VCC's brief mode keeps `Read`/`Edit`/… args and drops the
113
+ * rest; `_BRIEF_HIDE_TOOLS` goes further and hides TodoWrite/ToolSearch
114
+ * entirely, which `hideTools` config covers here).
115
+ */
116
+ export const DEFAULT_ARG_TOOLS = Object.freeze([
117
+ "read",
118
+ "write",
119
+ "edit",
120
+ "glob",
121
+ "grep",
122
+ "bash",
123
+ "shell",
124
+ "web_search",
125
+ "skill",
126
+ "subagent",
127
+ "subagent_fork",
128
+ "ralph",
129
+ "workflow"
130
+ ]);
131
+
132
+ /**
133
+ * First-line projection for a tool-call one-liner: multiline arguments
134
+ * collapse to the first physical line plus a `[+ N lines]` marker, so
135
+ * scripted commands (bash heredocs, multi-line pipelines) stay one compact
136
+ * line instead of echoing the whole payload.
137
+ * @param value - the key-argument string.
138
+ * @returns the display value.
139
+ */
140
+ export function oneLineArg(value) {
141
+ const lines = value.split(/\r?\n/u);
142
+ if (lines.length === 1) return value;
143
+ const first = lines[0] !== "" || lines.every((line) => line.trim() === "")
144
+ ? lines[0]
145
+ : lines.find((line) => line.trim() !== "");
146
+ return `${first} [+ ${lines.length - 1} lines]`;
147
+ }
148
+
149
+ /**
150
+ * Pick the key argument for a tool-call one-liner: the tool-specific field
151
+ * first, then the first string-valued argument, else nothing.
152
+ * @param name - tool name.
153
+ * @param input - parsed arguments object (or null when unparseable).
154
+ * @param keyFields - merged tool-name → preferred-field map.
155
+ * @returns the display value, or undefined.
156
+ */
157
+ export function pickToolKeyArg(name, input, keyFields) {
158
+ if (input === null || typeof input !== "object" || Array.isArray(input)) return undefined;
159
+ const preferred = keyFields[name];
160
+ if (preferred !== undefined) {
161
+ const value = input[preferred];
162
+ if (typeof value === "string" && value.length > 0) return value;
163
+ }
164
+ for (const value of Object.values(input)) {
165
+ if (typeof value === "string" && value.length > 0) return value;
166
+ }
167
+ return undefined;
168
+ }
169
+
170
+ /**
171
+ * Safely parse a tool-call's raw JSON arguments string.
172
+ * @param arguments_ - raw JSON as produced by the model.
173
+ * @returns the parsed value, or null when it is not valid JSON.
174
+ */
175
+ export function parseToolArguments(arguments_) {
176
+ try {
177
+ return JSON.parse(arguments_);
178
+ } catch {
179
+ return null;
180
+ }
181
+ }
182
+
183
+ // ── content projection helpers ─────────────────────────────────────────────
184
+
185
+ /**
186
+ * Project nested tool-result blocks into one plain-text string: text blocks
187
+ * are joined, images and documents render as VCC-style labels.
188
+ * @param blocks - nested result content.
189
+ * @returns projected text.
190
+ */
191
+ export function projectToolResultText(blocks) {
192
+ const parts = [];
193
+ // ⚠️ 2026-11-07 — tolerate a missing/non-array projection input and null
194
+ // blocks inside it. Both used to throw (`block.type` on null), and this helper
195
+ // sits on the tool-result, memory-recall and search paths alike.
196
+ if (!Array.isArray(blocks)) return "";
197
+ for (const block of blocks) {
198
+ if (block === null || block === undefined) continue;
199
+ if (block.type === "text") {
200
+ const text = sanitize(block.text ?? "");
201
+ if (text.length > 0) parts.push(text);
202
+ } else if (block.type === "image") parts.push("[image]");
203
+ else if (block.type === "document") parts.push("[document]");
204
+ else parts.push(`[${String(block.type)}]`);
205
+ }
206
+ return parts.join("\n");
207
+ }
208
+
209
+ /** Whether a message source identifies a landed compaction checkpoint node. */
210
+ export function isCheckpointSource(source) {
211
+ if (source === undefined || source === null) return false;
212
+ // Host 0.2.x builds the marker with `compactCheckpointSource()`, which yields
213
+ // `{ kind: "compact-checkpoint", compactionId, sourceCommandId? }`. The legacy
214
+ // `{ kind: "plugin", plugin: "compact" }` shape from earlier hosts is still
215
+ // accepted so checkpoints written by an older host stay recallable.
216
+ if (source.kind === "compact-checkpoint") return true;
217
+ return source.kind === "plugin" && source.plugin === "compact";
218
+ }
219
+
220
+ /**
221
+ * Whether a message source is one of the harness-injected instruction blocks
222
+ * the caller asked to elide (see `elideSourceKinds`). Matching is on the
223
+ * structured `source.kind` field the harness records, never on message text:
224
+ * a user quoting "Instructions from:" must not be mistaken for the real thing.
225
+ *
226
+ * The default (`undefined`/empty) matches nothing, so the open-source default
227
+ * behavior is byte-identical to before this option existed.
228
+ * @param source - message source descriptor, possibly absent.
229
+ * @param kinds - configured exact `source.kind` values to elide.
230
+ * @returns true when this node should collapse to a recall pointer.
231
+ */
232
+ export function isElidableSource(source, kinds) {
233
+ if (kinds === undefined || kinds.length === 0) return false;
234
+ const kind = source?.kind;
235
+ return typeof kind === "string" && kinds.includes(kind);
236
+ }
237
+
238
+ /**
239
+ * Display labels for elidable instruction sources. Anything not listed falls
240
+ * back to its raw `source.kind`, so a new host source still gets a readable
241
+ * pointer without a code change.
242
+ */
243
+ const ELIDABLE_SOURCE_LABELS = Object.freeze({
244
+ "dsh-mnemon": "记忆快照",
245
+ "agent-instructions": "规则与冷库索引",
246
+ "skill-catalog": "技能清单",
247
+ "runtime-context": "运行时上下文",
248
+ "time-context": "时间上下文"
249
+ });
250
+
251
+ /**
252
+ * Markers that open a hot-memory snapshot as our memory plugin renders it.
253
+ * `[MNEMON]` is the routing hint, `MNEMON RUNTIME MEMORY SNAPSHOT` the payload.
254
+ */
255
+ const MEMORY_SNAPSHOT_MARKERS = Object.freeze(["MNEMON RUNTIME MEMORY SNAPSHOT", "[MNEMON]"]);
256
+ /** Closing tag of one memory file block inside a snapshot. */
257
+ const MEMORY_FILE_CLOSE = "</runtime-memory-file>";
258
+ /**
259
+ * Text that may separate two consecutive memory file blocks of ONE snapshot
260
+ * (`USER.md` then `MEMORY.md`), so the two are treated as a single span.
261
+ */
262
+ const MEMORY_BLOCK_JOINER = /^\s*(?:MEMORY\.md \(working reference\)|USER\.md \(user profile\)|<runtime-memory-file )/;
263
+ /**
264
+ * One role header line of this compiler's own checkpoint format. Memory snapshots
265
+ * never contain one, so finding it between a file block's opening and its close
266
+ * means the close belongs to a LATER block and the span must stop before this.
267
+ * Guards the truncation case: a block cut mid-way loses its closing tag, and
268
+ * without this the span would swallow the conversation that follows.
269
+ */
270
+ const MEMORY_ROLE_HEAD = /\n\[(?:user|assistant|tool)\]\s*\n/;
271
+
272
+ /**
273
+ * Marker text the host puts inside a REAL rules/index wrapper. Prose that only
274
+ * mentions `<system-reminder>` as a string must not qualify, so a candidate
275
+ * span has to prove itself by carrying one of these phrases early on.
276
+ */
277
+ const SYSTEM_REMINDER_BODY_MARKERS = Object.freeze([
278
+ "The following workspace instructions may be relevant",
279
+ "Instructions from:"
280
+ ]);
281
+ /** How far into a candidate wrapper the proof phrase must appear. */
282
+ const SYSTEM_REMINDER_PROOF_WINDOW = 600;
283
+ /** How close a memory marker must sit to the first file block to count. */
284
+ const MEMORY_MARKER_WINDOW = 800;
285
+
286
+ /**
287
+ * Locate injected memory snapshots embedded in a PRIOR checkpoint body.
288
+ *
289
+ * A snapshot is a run of consecutive `<runtime-memory-file …>…</…>` blocks.
290
+ * The span starts at the nearest preceding snapshot marker so the routing hint
291
+ * and the "Entry annotations default…" preamble go with it. The marker must sit
292
+ * close to the first file block, so prose that merely quotes the marker text
293
+ * (e.g. a discussion about this very feature) is not mistaken for a snapshot.
294
+ * @param text - prior checkpoint body.
295
+ * @returns `[start, end)` character spans, left to right.
296
+ */
297
+ function findMemorySnapshotSpans(text) {
298
+ const spans = [];
299
+ let searchFrom = 0;
300
+ while (searchFrom < text.length) {
301
+ const markerAt = MEMORY_SNAPSHOT_MARKERS
302
+ .map((marker) => text.indexOf(marker, searchFrom))
303
+ .filter((index) => index !== -1)
304
+ .reduce((lowest, index) => (lowest === undefined || index < lowest ? index : lowest), undefined);
305
+ if (markerAt === undefined) break;
306
+ const firstFile = text.indexOf("<runtime-memory-file ", markerAt);
307
+ if (firstFile === -1) break;
308
+ if (firstFile - markerAt > MEMORY_MARKER_WINDOW) {
309
+ // Just prose quoting the marker — skip past it and keep looking.
310
+ searchFrom = markerAt + 1;
311
+ continue;
312
+ }
313
+ let end;
314
+ let cursor = firstFile;
315
+ for (;;) {
316
+ const close = text.indexOf(MEMORY_FILE_CLOSE, cursor);
317
+ if (close === -1) break;
318
+ // A memory block whose closing tag was truncated away has no close of its
319
+ // own, so the search below would run on to the NEXT block's close and
320
+ // swallow every real turn in between. Role headers are this compiler's own
321
+ // output and never appear inside a memory snapshot, so a role header before
322
+ // the close proves the span has left the block: stop there and keep the
323
+ // dialogue. Erring toward eliding less is safe; eliding a turn is not.
324
+ const roleHead = MEMORY_ROLE_HEAD.exec(text.slice(cursor, close));
325
+ if (roleHead !== null) {
326
+ end = cursor + roleHead.index;
327
+ break;
328
+ }
329
+ const after = close + MEMORY_FILE_CLOSE.length;
330
+ end = after;
331
+ cursor = after;
332
+ if (!MEMORY_BLOCK_JOINER.test(text.slice(after, after + 240))) break;
333
+ }
334
+ if (end === undefined) break;
335
+ spans.push([markerAt, end]);
336
+ searchFrom = end;
337
+ }
338
+ return spans;
339
+ }
340
+
341
+ /**
342
+ * Locate `<system-reminder>…</system-reminder>` wrappers that really frame
343
+ * workspace rules, the cold-store index or runtime context.
344
+ *
345
+ * The wrapper must open at a line start, be followed by the block body marker
346
+ * within a short window, and close before the body repeats — the host emits one
347
+ * wrapper per injection, never nested. A sentence that merely mentions the tag
348
+ * name fails all three checks, which is what keeps live prose intact.
349
+ * @param text - prior checkpoint body.
350
+ * @returns `[start, end)` character spans, left to right.
351
+ */
352
+ function findSystemReminderSpans(text) {
353
+ const spans = [];
354
+ const pattern = /(?:^|\n)<system-reminder>\n/g;
355
+ let match;
356
+ while ((match = pattern.exec(text)) !== null) {
357
+ const open = text.indexOf("<system-reminder>", match.index);
358
+ const head = text.slice(open, open + SYSTEM_REMINDER_PROOF_WINDOW);
359
+ if (!SYSTEM_REMINDER_BODY_MARKERS.some((marker) => head.includes(marker))) continue;
360
+ const close = text.indexOf("</system-reminder>", open);
361
+ if (close === -1) continue;
362
+ spans.push([open, close + "</system-reminder>".length]);
363
+ }
364
+ return spans;
365
+ }
366
+
367
+ /**
368
+ * Collapse harness-injected bodies embedded in a PRIOR checkpoint to one-line
369
+ * pointers.
370
+ *
371
+ * Why this exists: those bodies are re-sent in full after every compaction, so
372
+ * any copy kept inside a checkpoint body is a duplicate — the checkpoint grows
373
+ * by one snapshot per compaction even though nothing new was said. The originals
374
+ * stay in the append-only session log, so `recall` still restores them verbatim.
375
+ *
376
+ * Only checkpoint prose is touched; live conversation text is never scanned.
377
+ * @param text - prior checkpoint body.
378
+ * @returns the rewritten body plus how many spans and characters were removed.
379
+ */
380
+ export function elideCheckpointInjectedBodies(text) {
381
+ const spans = [...findMemorySnapshotSpans(text), ...findSystemReminderSpans(text)]
382
+ .sort((left, right) => left[0] - right[0]);
383
+ if (spans.length === 0) return { text, spans: 0, chars: 0 };
384
+ let out = "";
385
+ let cursor = 0;
386
+ let chars = 0;
387
+ let removed = 0;
388
+ for (const [start, end] of spans) {
389
+ // Overlapping spans would otherwise duplicate text; keep the outer one.
390
+ if (start < cursor) continue;
391
+ out += text.slice(cursor, start);
392
+ const length = end - start;
393
+ chars += length;
394
+ removed += 1;
395
+ out += `[注入内容已压缩为指针(原文 ${length} 字符)— 压缩后会重新注入,原文可用 recall 调回]`;
396
+ cursor = end;
397
+ }
398
+ out += text.slice(cursor);
399
+ return { text: out, spans: removed, chars };
400
+ }
401
+
402
+ /**
403
+ * One-line pointer standing in for an elided instruction block. Carries the
404
+ * seq so the agent can `recall` the original, plus what it was and how big it
405
+ * was — a bare `(seq N)` would read as content loss rather than a substitution.
406
+ * @param label - human name of the block ("记忆快照" / "规则与索引").
407
+ * @param seq - source sequence of the elided message.
408
+ * @param length - original character count.
409
+ * @returns the marker line.
410
+ */
411
+ function elidedSourceMarker(label, seq, length) {
412
+ return `[${label} 已压缩为指针(${seqRef(seq)},原文 ${length} 字符)— 原文可用 recall 调回,且下一次压缩后会重新注入]`;
413
+ }
414
+
415
+ /**
416
+ * Head/tail excerpt of a tool result under one shared token budget. The tail
417
+ * is anchored to the end of the text at character granularity, so a result
418
+ * whose tail matters more than its head (file listings, logs) keeps its end.
419
+ * @param text - projected result text.
420
+ * @param budget - combined non-whitespace token budget for head + tail.
421
+ * @param ref - provenance for the elision marker.
422
+ * @returns the excerpt, or the full text when it already fits.
423
+ */
424
+ export function excerptToolResult(text, budget, ref) {
425
+ const overBudget = countTokens(text) > budget || text.length > budget * 4;
426
+ if (budget <= 0 || !overBudget) return text;
427
+ const half = Math.floor(budget / 2);
428
+ const headLimit = Math.max(8, half);
429
+ const tailLimit = Math.max(8, budget - headLimit);
430
+ const head = truncateTokens(text, headLimit, ref);
431
+ const tail = tailTokens(text, tailLimit, ref);
432
+ return `${head.text}\n...(elided from ${ref})\n${tail.text}`;
433
+ }
434
+
435
+ // ── the region compiler ────────────────────────────────────────────────────
436
+
437
+ const CHECKPOINT_PREAMBLE = "This is an automatically generated checkpoint condensing an earlier span of the conversation to free up context. Treat the captured context as established background and build on it without restating it. Continue the task directly from the messages that follow, without acknowledging this checkpoint.";
438
+ const CHECKPOINT_OPEN_TAG = "<compacted-checkpoint>";
439
+ const CHECKPOINT_CLOSE_TAG = "</compacted-checkpoint>";
440
+
441
+ /**
442
+ * Model-facing guide framed into the head of every checkpoint: tells the
443
+ * agent how to recover content the compiler elided or truncated, using the
444
+ * two recall tools that read the append-only durable log.
445
+ */
446
+ export const RECALL_GUIDE = "RECALL: append-only log — nothing is lost. `recall` restores original content: type \"seq\", id \"3-7\"; \"result\", id \"3\"; \"checkpoint\", id \"1\" (`[checkpoint N]` = dropped). `search` finds by keyword.";
447
+
448
+ /** Per-node reference marker: the durable seq is the lossless pointer. */
449
+ function seqRef(seq) {
450
+ return `seq ${seq}`;
451
+ }
452
+
453
+ /**
454
+ * Build marker for this compiler revision. Bumped whenever the compiled
455
+ * output format changes, so debug logs can prove which code version ran.
456
+ */
457
+ export const COMPILER_REV = "vcc-compiler-debug-2";
458
+
459
+ /**
460
+ * Emit one debug line through the configured sink (and stderr). The sink is
461
+ * installed by the engine (`config.debugSink`) when `debug` is enabled; the
462
+ * compiler itself stays dependency-free.
463
+ * @param config - resolved compiler configuration.
464
+ * @param tag - short component tag for the line.
465
+ * @param line - the diagnostic text.
466
+ */
467
+ function debugLog(config, tag, line) {
468
+ if (config.debug !== true) return;
469
+ const text = `[dsh-compaction-vcc] ${tag}: ${line}`;
470
+ try {
471
+ if (typeof config.debugSink === "function") config.debugSink(text);
472
+ } catch {
473
+ /* a failing debug sink must never break a compaction */
474
+ }
475
+ try {
476
+ console.error(text);
477
+ } catch {
478
+ /* stderr may be unavailable in embedded contexts */
479
+ }
480
+ }
481
+
482
+ /**
483
+ * Compile one ordered region into compact entries.
484
+ * @param nodes - ordered `{ seq, message }` projections of the shadowed
485
+ * surface nodes; `message` may be null (non-projecting node).
486
+ * @param config - resolved compiler-relevant configuration (see index.js).
487
+ * @param budgets - optional per-block budgets; defaults to the config values
488
+ * (the budget-cap loop passes scaled budgets on retry).
489
+ * @returns `{ entries, stats }` where each entry is one line-group string.
490
+ */
491
+ export function compileNodes(nodes, config, budgets) {
492
+ const effective = budgets ?? {
493
+ textTokens: config.textTokens,
494
+ userTextTokens: config.userTextTokens,
495
+ toolCallTokens: config.toolCallTokens,
496
+ toolResultExcerptTokens: config.toolResultExcerptTokens
497
+ };
498
+ const keyFields = { ...DEFAULT_TOOL_KEY_FIELDS, ...(config.toolKeyFields ?? {}) };
499
+ const patterns = config.noisePatterns ?? [];
500
+ // Empty lists mean "unset": the cordis config pipeline injects `[]` for
501
+ // absent array keys, so fall back to the defaults instead of rendering
502
+ // every tool call name-only.
503
+ const argTools = config.toolArgTools?.length > 0 ? config.toolArgTools : DEFAULT_ARG_TOOLS;
504
+ const hiddenTools = new Set(config.hideTools ?? []);
505
+ debugLog(config, "compile", `rev=${COMPILER_REV} nodes=${nodes.length} argTools=[${argTools.join(",")}] hideTools=[${[...hiddenTools].join(",")}] budgets=${JSON.stringify(effective)}`);
506
+ const entries = [];
507
+ const stats = {
508
+ nodes: nodes.length,
509
+ entries: 0,
510
+ toolCalls: 0,
511
+ toolResults: 0,
512
+ images: 0,
513
+ documents: 0,
514
+ reasoningElided: 0,
515
+ noiseElided: 0,
516
+ elidedSourceRows: 0,
517
+ elidedSourceChars: 0,
518
+ elidedCheckpointSpans: 0,
519
+ elidedCheckpointChars: 0,
520
+ checkpoints: 0,
521
+ tokens: 0,
522
+ elidedToolRows: 0,
523
+ elidedRows: 0
524
+ };
525
+ // Pre-pass over the ordered nodes: map each tool-call id to the seq of its
526
+ // result node, so every call one-liner can carry a VCC-style result pointer
527
+ // (`(seq N -> result M)`) even though results no longer occupy entries.
528
+ //
529
+ // ⚠️ 2026-11-06 (this fork) — host 0.2.2 compatibility fix.
530
+ // Up to host ~0.1.x a derived tool-result message exposed `toolCallId` on its
531
+ // FIRST CONTENT BLOCK (`message.content[0].toolCallId`), and that is the only
532
+ // shape this pre-pass used to look at. Host 0.2.2 derives a tool message the
533
+ // way the model APIs do: `role: "tool"` with `toolCallId` on the MESSAGE
534
+ // (`message.toolCallId`), while `content[0]` is a plain text block. The old
535
+ // check therefore never matched, `resultSeqByCallId` stayed empty, and every
536
+ // `* read "a.js" (seq 2)` one-liner silently lost its `-> result 3` pointer —
537
+ // the hook `recall(type:"result")` relies on. Nothing threw; the checkpoint
538
+ // just quietly stopped being lossless. Read the message-level field first and
539
+ // keep the legacy `content[0]` shape as a fallback so one build serves both
540
+ // host generations.
541
+ const resultSeqByCallId = new Map();
542
+ for (const node of nodes) {
543
+ const message = node.message;
544
+ if (message === null || message === undefined) continue;
545
+ const first = message.content?.[0];
546
+ // ⚠️ 2026-11-07 — `first !== undefined` misses null, which threw on
547
+ // `first.type`. Guard the null block explicitly.
548
+ const toolResultBlock = first !== undefined && first !== null && first.type === "tool-result" ? first : undefined;
549
+ const callId = typeof message.toolCallId === "string"
550
+ ? message.toolCallId
551
+ : typeof toolResultBlock?.toolCallId === "string"
552
+ ? toolResultBlock.toolCallId
553
+ : undefined;
554
+ if (callId !== undefined) resultSeqByCallId.set(callId, node.seq);
555
+ }
556
+ let lastRole;
557
+ let open = false;
558
+
559
+ /**
560
+ * Append one compiled entry carrying a deletion-priority kind. Kinds order
561
+ * the cap-elision passes in compileRegion: `result`/`tool`/`media`/`note`
562
+ * rows are dropped before `text`/`reasoning`/`checkpoint` rows, so when a
563
+ * checkpoint must shrink, the conversation's words survive its logs.
564
+ * @param seq - durable seq the entry derives from.
565
+ * @param text - rendered entry text.
566
+ * @param kind - entry kind (see the kind list above).
567
+ */
568
+ const pushEntry = (seq, text, kind) => {
569
+ entries.push({ seq, text, kind });
570
+ stats.entries += 1;
571
+ stats.tokens += estimateEntryTokens(text);
572
+ };
573
+ const roleHeader = (role, seq) => {
574
+ if (!open || lastRole !== role) {
575
+ open = true;
576
+ lastRole = role;
577
+ return `[${role}]\n`;
578
+ }
579
+ return "";
580
+ };
581
+
582
+ for (const node of nodes) {
583
+ const message = node.message;
584
+ // ⚠️ 2026-11-07 — `content` must be an iterable array here. A null/absent
585
+ // `content` used to fall through to `message.content[0]` below and throw
586
+ // `Cannot read properties of null (reading '0')`, which aborted the WHOLE
587
+ // compile: one malformed node cost the entire compaction, silently (the
588
+ // host's pre-step handler swallows the error as a warning). Skip the node
589
+ // instead so the rest of the region still compiles.
590
+ if (message === null || message === undefined) continue;
591
+ if (!Array.isArray(message.content)) continue;
592
+ if (message.role === "assistant") {
593
+ let header = "";
594
+ for (const block of message.content) {
595
+ // ⚠️ 2026-11-07 — a null/undefined block in the array used to throw on
596
+ // `block.type` and abort the whole compile. Skip it.
597
+ if (block === null || block === undefined) continue;
598
+ if (block.type === "text") {
599
+ const text = sanitize(block.text ?? "");
600
+ if (text.trim().length === 0) continue;
601
+ header = roleHeader("assistant", node.seq);
602
+ const kept = truncateTokens(text, effective.textTokens, seqRef(node.seq));
603
+ pushEntry(node.seq, header + kept.text, "text");
604
+ continue;
605
+ }
606
+ if (block.type === "reasoning") {
607
+ if (config.includeReasoning !== true) {
608
+ stats.reasoningElided += 1;
609
+ continue;
610
+ }
611
+ const text = sanitize(block.text ?? "");
612
+ if (text.trim().length === 0) continue;
613
+ header = roleHeader("assistant", node.seq);
614
+ const kept = truncateTokens(text, effective.textTokens, seqRef(node.seq));
615
+ pushEntry(node.seq, header + kept.text, "reasoning");
616
+ continue;
617
+ }
618
+ if (block.type === "tool-call") {
619
+ const name = block.name ?? "unknown";
620
+ stats.toolCalls += 1;
621
+ if (hiddenTools.has(name)) {
622
+ debugLog(config, "tool", `seq=${node.seq} name=${name} HIDDEN (hideTools)`);
623
+ continue;
624
+ }
625
+ header = roleHeader("assistant", node.seq);
626
+ // One line, always: key argument only for whitelisted tools, and a
627
+ // VCC-style pointer to the result node (`* read "a.js" (seq 2 ->
628
+ // result 3)`) so the dropped result stays one recall away.
629
+ let oneLine;
630
+ let diag;
631
+ if (argTools.includes(name)) {
632
+ const parsed = parseToolArguments(block.arguments);
633
+ const arg = pickToolKeyArg(name, parsed, keyFields);
634
+ oneLine = arg === undefined ? `* ${name}` : `* ${name} "${oneLineArg(arg)}"`;
635
+ diag = `whitelist=yes argsType=${typeof block.arguments} argsLen=${block.arguments === null || block.arguments === undefined ? 0 : String(block.arguments).length} argsHead=${JSON.stringify(String(block.arguments).slice(0, 60))} parse=${parsed === null ? "FAIL" : "ok"} key=${keyFields[name] ?? "(fallback)"} arg=${arg === undefined ? "(none)" : JSON.stringify(String(arg).slice(0, 60))}`;
636
+ } else {
637
+ oneLine = `* ${name}`;
638
+ diag = `whitelist=no argsType=${typeof block.arguments} argsLen=${block.arguments === null || block.arguments === undefined ? 0 : String(block.arguments).length} argsHead=${JSON.stringify(String(block.arguments).slice(0, 60))}`;
639
+ }
640
+ const resultSeq = resultSeqByCallId.get(block.id);
641
+ const ref = resultSeq === undefined ? seqRef(node.seq) : `${seqRef(node.seq)} -> result ${resultSeq}`;
642
+ const kept = truncateTokens(oneLine, effective.toolCallTokens, seqRef(node.seq));
643
+ debugLog(config, "tool", `seq=${node.seq} name=${name} ${diag} line=${JSON.stringify(oneLine.slice(0, 80))} truncated=${kept.truncated} ref=${ref}`);
644
+ pushEntry(node.seq, header + `${kept.text} (${ref})`, "tool");
645
+ continue;
646
+ }
647
+ if (block.type === "image") {
648
+ stats.images += 1;
649
+ header = roleHeader("assistant", node.seq);
650
+ pushEntry(node.seq, header + `[image] (${seqRef(node.seq)})`, "media");
651
+ continue;
652
+ }
653
+ if (block.type === "document") {
654
+ stats.documents += 1;
655
+ header = roleHeader("assistant", node.seq);
656
+ pushEntry(node.seq, header + `[document] (${seqRef(node.seq)})`, "media");
657
+ continue;
658
+ }
659
+ /* unknown merge-extensible block types render as labels */
660
+ {
661
+ header = roleHeader("assistant", node.seq);
662
+ pushEntry(node.seq, header + `[${String(block.type)}] (${seqRef(node.seq)})`, "media");
663
+
664
+ }
665
+ }
666
+ /* reasoning-only and empty assistant messages produce no entry */
667
+ continue;
668
+ }
669
+ // ⚠️ 2026-11-07 — host 0.2.x derives a tool result as `role: "tool"` with
670
+ // `toolCallId` on the MESSAGE and a plain text block as `content[0]`. The
671
+ // loop used to handle only `assistant` and `user`, so every real tool result
672
+ // fell through and produced no entry: `stats.toolResults` stayed at 0, the
673
+ // `toolResultExcerptTokens` budget was inert, and the tool output never
674
+ // appeared in the checkpoint at all. Only the `-> result N` pointer survived
675
+ // (that map is built by the separate pre-pass above), so `recall` could still
676
+ // fetch the original — but the compiled view was misdescribed as "lossless"
677
+ // while silently dropping this whole class of content.
678
+ if (message.role === "tool") {
679
+ stats.toolResults += 1;
680
+ const body = projectToolResultText(message.content);
681
+ if (body.length > 0) {
682
+ const header = roleHeader("tool", node.seq);
683
+ // Excerpt to the configured budget: tool output is the bulk of a long
684
+ // session, and the full text stays one `recall(type:"result")` away via
685
+ // the call line's `-> result N` pointer.
686
+ const kept = excerptToolResult(body, effective.toolResultExcerptTokens, seqRef(node.seq));
687
+ pushEntry(node.seq, header + kept, "result");
688
+ }
689
+ continue;
690
+ }
691
+ if (message.role === "user") {
692
+ const first = message.content[0];
693
+ if (first !== undefined && first !== null && first.type === "tool-result") {
694
+ // Tool results never occupy an entry: the content is either absorbed
695
+ // into the following assistant text or one recall away via the call
696
+ // line's `-> result N` pointer (VCC drops results from brief mode too).
697
+ stats.toolResults += 1;
698
+ continue;
699
+ }
700
+ if (isCheckpointSource(message.source)) {
701
+ stats.checkpoints += 1;
702
+ let text = projectToolResultText(message.content);
703
+ if (text.length > 0 && config.elideCheckpointInjectedBodies === true) {
704
+ // A prior checkpoint body still carries every memory snapshot and
705
+ // rules block that was live when it was compiled. Those are re-sent
706
+ // in full after this compaction, so keeping them here is a duplicate.
707
+ // Opt-in (default off) because it rewrites previously compiled prose.
708
+ const collapsed = elideCheckpointInjectedBodies(text);
709
+ if (collapsed.spans > 0) {
710
+ text = collapsed.text;
711
+ stats.elidedCheckpointSpans += collapsed.spans;
712
+ stats.elidedCheckpointChars += collapsed.chars;
713
+ }
714
+ }
715
+ if (text.length > 0) {
716
+ // The checkpoint node is a user/message in the durable protocol
717
+ // (surface replacement only allows message nodes), but it is
718
+ // harness-generated framing — display it as [system].
719
+ const header = roleHeader("system", node.seq);
720
+ pushEntry(node.seq, header + text, "checkpoint");
721
+ }
722
+ continue;
723
+ }
724
+ if (isElidableSource(message.source, config.elideSourceKinds)) {
725
+ // Harness-injected instruction blocks (hot memory, workspace rules,
726
+ // skill catalog…) are re-injected in full after every compaction, so
727
+ // keeping their body here duplicates them: one copy in the checkpoint,
728
+ // one copy re-injected, every time. Collapse to a recall pointer.
729
+ // `elideSourceKinds` is empty by default, so stock behavior is
730
+ // unchanged; we enable it because our memory plugin re-injects.
731
+ const text = projectToolResultText(message.content);
732
+ if (text.length === 0) continue;
733
+ const label = ELIDABLE_SOURCE_LABELS[message.source.kind] ?? message.source.kind;
734
+ stats.elidedSourceRows += 1;
735
+ stats.elidedSourceChars += text.length;
736
+ pushEntry(node.seq, `${roleHeader("user", node.seq)}* ${elidedSourceMarker(label, node.seq, text.length)}`, "note");
737
+ continue;
738
+ }
739
+ let surviving = 0;
740
+ let header = "";
741
+ for (const block of message.content) {
742
+ // ⚠️ 2026-11-07 — a null/undefined block in the array used to throw on
743
+ // `block.type` and abort the whole compile. Skip it.
744
+ if (block === null || block === undefined) continue;
745
+ if (block.type === "text") {
746
+ let text = sanitize(block.text ?? "");
747
+ if (config.stripNoiseXml !== false) text = stripNoiseXml(text, patterns);
748
+ if (text.length === 0) continue;
749
+ surviving += 1;
750
+ header = roleHeader("user", node.seq);
751
+ const kept = truncateTokens(text, effective.userTextTokens, seqRef(node.seq));
752
+ pushEntry(node.seq, header + kept.text, "text");
753
+ } else if (block.type === "image") {
754
+ stats.images += 1;
755
+ surviving += 1;
756
+ header = roleHeader("user", node.seq);
757
+ pushEntry(node.seq, header + `[image] (${seqRef(node.seq)})`, "media");
758
+ } else if (block.type === "document") {
759
+ stats.documents += 1;
760
+ surviving += 1;
761
+ header = roleHeader("user", node.seq);
762
+ pushEntry(node.seq, header + `[document] (${seqRef(node.seq)})`, "media");
763
+ } else {
764
+ const text = sanitize(block.text ?? "");
765
+ if (text.length === 0) continue;
766
+ surviving += 1;
767
+ header = roleHeader("user", node.seq);
768
+ const kept = truncateTokens(text, effective.userTextTokens, seqRef(node.seq));
769
+ pushEntry(node.seq, header + kept.text, "text");
770
+ }
771
+ }
772
+ if (surviving === 0) {
773
+ stats.noiseElided += 1;
774
+ pushEntry(node.seq, `${roleHeader("user", node.seq)}* [user message elided: noise-only] (${seqRef(node.seq)})`, "note");
775
+ }
776
+ }
777
+ }
778
+ return { entries, stats };
779
+ }
780
+
781
+ /**
782
+ * Per-budget elision floors: conversation text keeps a meaningful minimum
783
+ * even when the total cap forces heavy rescaling, while tool one-liners and
784
+ * result excerpts may collapse to near-nothing first.
785
+ */
786
+ const BUDGET_FLOORS = Object.freeze({
787
+ textTokens: 32,
788
+ userTextTokens: 32,
789
+ toolCallTokens: 8,
790
+ toolResultExcerptTokens: 8
791
+ });
792
+
793
+ /** Entry kinds dropped by the first (low-value) elision pass. */
794
+ const LOW_VALUE_KINDS = new Set(["result", "tool", "media", "note"]);
795
+
796
+ /**
797
+ * Scale ONLY the conversation-text budgets toward a total cap, never below
798
+ * the per-key floor. Tool-call rows stay one-liners at their configured
799
+ * budget: conversation shrinks first, tool rows are sacrificed whole by the
800
+ * elision pass, so tool calls can never squeeze the dialogue out.
801
+ * @param budgets - budget map to scale.
802
+ * @param factor - uniform scale factor for text budgets.
803
+ * @returns the scaled budget map (tool budgets untouched).
804
+ */
805
+ function scaleBudgets(budgets, factor) {
806
+ const scaled = { ...budgets };
807
+ for (const key of ["textTokens", "userTextTokens"]) {
808
+ scaled[key] = Math.max(BUDGET_FLOORS[key] ?? 8, Math.floor(budgets[key] * factor));
809
+ }
810
+ return scaled;
811
+ }
812
+
813
+ /** Render the provenance range of one dropped-entry list. */
814
+ function seqRangeOf(dropped) {
815
+ const first = dropped[0].seq;
816
+ const last = dropped[dropped.length - 1].seq;
817
+ return first === last ? `seq ${first}` : `seqs ${first}-${last}`;
818
+ }
819
+
820
+ /**
821
+ * Compile a region to completion under the total checkpoint cap: budgets are
822
+ * rescaled toward `config.maxTokens` up to a fixed number of attempts, then
823
+ * the oldest low-value rows (tool/result/media/note) are front-elided first,
824
+ * and only then the oldest remaining entries — so the conversation's words
825
+ * survive elision before its logs do. The newest entries always survive —
826
+ * recency is the value ranking, as in VCC chunking.
827
+ * @param nodes - ordered `{ seq, message }` projections.
828
+ * @param config - resolved compiler configuration.
829
+ * @returns `{ entries, stats, capped }` — `capped` records cap enforcement.
830
+ */
831
+ export function compileRegion(nodes, config) {
832
+ const baseBudgets = {
833
+ textTokens: config.textTokens,
834
+ userTextTokens: config.userTextTokens,
835
+ toolCallTokens: config.toolCallTokens,
836
+ toolResultExcerptTokens: config.toolResultExcerptTokens
837
+ };
838
+ let budgets = baseBudgets;
839
+ let result = compileNodes(nodes, config, budgets);
840
+ let attempts = 0;
841
+ const maxAttempts = 4;
842
+ while (result.stats.tokens > config.maxTokens && attempts < maxAttempts) {
843
+ budgets = scaleBudgets(budgets, config.maxTokens / result.stats.tokens * 0.95);
844
+ result = compileNodes(nodes, config, budgets);
845
+ attempts += 1;
846
+ }
847
+ let capped = false;
848
+ if (result.stats.tokens > config.maxTokens) {
849
+ const entries = result.entries;
850
+ // Phase A: drop the oldest low-value rows (tool/result/media/note)
851
+ // anywhere in the list first, so conversation text and prior-checkpoint
852
+ // knowledge survive elision before the logs do. The marker's seq range is
853
+ // the dropped rows' span (approximate: surviving text may sit inside it).
854
+ const toolRows = [];
855
+ while (entries.length > 1 && result.stats.tokens > config.maxTokens) {
856
+ const index = entries.findIndex((entry, position) => position < entries.length - 1 && LOW_VALUE_KINDS.has(entry.kind));
857
+ if (index === -1) break;
858
+ const dropped = entries.splice(index, 1)[0];
859
+ result.stats.tokens -= estimateEntryTokens(dropped.text);
860
+ toolRows.push(dropped);
861
+ }
862
+ if (toolRows.length > 0) {
863
+ capped = true;
864
+ result.stats.elidedToolRows += toolRows.length;
865
+ entries.unshift({ seq: toolRows[0].seq, text: `[${toolRows.length} tool/result entries elided: ${seqRangeOf(toolRows)}]`, kind: "note" });
866
+ result.stats.tokens += estimateEntryTokens(entries[0].text);
867
+ }
868
+ const elided = [];
869
+ const droppedCheckpoints = [];
870
+ while (entries.length > 1 && result.stats.tokens > config.maxTokens) {
871
+ const dropped = entries.shift();
872
+ result.stats.tokens -= estimateEntryTokens(dropped.text);
873
+ elided.push(dropped);
874
+ if (dropped.kind === "checkpoint") droppedCheckpoints.push(dropped);
875
+ }
876
+ if (elided.length > 0) {
877
+ capped = true;
878
+ result.stats.elidedRows += elided.length;
879
+ // Dropped prior checkpoints never vanish silently: each leaves a
880
+ // single `[checkpoint N]` line the agent can restore in full via
881
+ // recall(type:"checkpoint", id:"N").
882
+ const markerLines = [`[${elided.length} earlier entries elided: ${seqRangeOf(elided)}]`];
883
+ for (const dropped of droppedCheckpoints) {
884
+ const ordinal = config.checkpointOrdinals?.get(dropped.seq);
885
+ const ref = ordinal === undefined ? `@ seq ${dropped.seq}` : String(ordinal);
886
+ markerLines.push(`[checkpoint ${ref}]`);
887
+ }
888
+ entries.unshift({ seq: elided[0].seq, text: markerLines.join("\n"), kind: "note" });
889
+ result.stats.tokens += estimateEntryTokens(entries[0].text);
890
+ }
891
+ if (result.stats.tokens > config.maxTokens && entries.length === 1) {
892
+ capped = true;
893
+ const hard = truncateTokens(entries[0].text, Math.max(8, config.maxTokens - 4), "total cap");
894
+ entries[0] = { seq: entries[0].seq, text: hard.text, kind: entries[0].kind };
895
+ result.stats.tokens = estimateEntryTokens(entries[0].text);
896
+ }
897
+ }
898
+ return { ...result, capped };
899
+ }
900
+
901
+ /**
902
+ * Normalize one compiled entry to its text: entries are `{ seq, text }`
903
+ * pairs, but a custom compile hook may supply plain strings.
904
+ * @param entry - entry value.
905
+ * @returns the entry's text.
906
+ */
907
+ export function entryText(entry) {
908
+ return typeof entry === "string" ? entry : entry.text;
909
+ }
910
+
911
+ /**
912
+ * Join compiled entries with separators that keep tool runs compact:
913
+ * consecutive `tool` rows join with a single newline (no blank line), while
914
+ * every other boundary keeps the blank line that separates entry groups.
915
+ * Strings (guide, header, elision markers) behave like `text` entries.
916
+ * @param entries - ordered entry texts (strings or `{ seq, text, kind }` pairs).
917
+ * @returns the joined body text.
918
+ */
919
+ export function joinCompiledEntries(entries) {
920
+ let out = "";
921
+ let previousKind;
922
+ for (const entry of entries) {
923
+ const kind = typeof entry === "string" ? "text" : entry.kind;
924
+ if (out.length > 0) out += previousKind === "tool" && kind === "tool" ? "\n" : "\n\n";
925
+ out += typeof entry === "string" ? entry : entry.text;
926
+ previousKind = kind;
927
+ }
928
+ return out;
929
+ }
930
+
931
+ /**
932
+ * Frame compiled entries as the durable replacement checkpoint content:
933
+ * the shared preamble, the recall guide, then one tagged block (guide first,
934
+ * then the optional intro line, the optional header line, the entries, and
935
+ * the optional retention footer). Prior-checkpoint text copied verbatim
936
+ * (with its own backend's tags) nests inside harmlessly.
937
+ * @param entries - ordered entry texts (strings or `{ seq, text }` pairs).
938
+ * @param headerLine - optional first line summarizing the compiled region.
939
+ * @param introLine - optional first line summarizing the compaction itself.
940
+ * @param footerLine - optional tail line reporting the verbatim retention.
941
+ * @returns plain text content blocks.
942
+ */
943
+ export function frameCheckpoint(entries, headerLine, introLine, footerLine) {
944
+ const body = [RECALL_GUIDE];
945
+ if (introLine !== undefined) body.push(introLine);
946
+ if (headerLine !== undefined) body.push(headerLine);
947
+ body.push(...entries);
948
+ // `footerLine` accepts one line or an ordered list, so callers can append
949
+ // several trailing notes (cap notice + verbatim retention) without
950
+ // concatenating them into one unreadable paragraph.
951
+ if (footerLine !== undefined) {
952
+ if (Array.isArray(footerLine)) body.push(...footerLine.filter((line) => line !== undefined));
953
+ else body.push(footerLine);
954
+ }
955
+ return [
956
+ { type: "text", text: `${CHECKPOINT_PREAMBLE}\n\n${CHECKPOINT_OPEN_TAG}` },
957
+ { type: "text", text: joinCompiledEntries(body) },
958
+ { type: "text", text: CHECKPOINT_CLOSE_TAG }
959
+ ];
960
+ }
961
+
962
+ export { CHECKPOINT_CLOSE_TAG, CHECKPOINT_OPEN_TAG, CHECKPOINT_PREAMBLE };