acuvo-code 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +328 -0
- package/ENTERPRISE.md +927 -0
- package/LICENSE +120 -0
- package/README.md +1245 -0
- package/ROADMAP.md +556 -0
- package/bin/acuvo-mcp.mjs +208 -0
- package/bin/acuvo.mjs +3108 -0
- package/lib/acceptance-consent.mjs +168 -0
- package/lib/acceptance.mjs +859 -0
- package/lib/account.mjs +226 -0
- package/lib/acuvo-dir.mjs +72 -0
- package/lib/acuvo-models.mjs +141 -0
- package/lib/apply-patch.mjs +570 -0
- package/lib/ask-user.mjs +173 -0
- package/lib/audit.mjs +530 -0
- package/lib/auto-lease.mjs +174 -0
- package/lib/background.mjs +842 -0
- package/lib/best-of.mjs +334 -0
- package/lib/board.mjs +232 -0
- package/lib/breaker.mjs +93 -0
- package/lib/budget.mjs +1375 -0
- package/lib/builtin-skills.mjs +135 -0
- package/lib/cache-floor.mjs +204 -0
- package/lib/chain.mjs +303 -0
- package/lib/changed-paths.mjs +84 -0
- package/lib/chat.mjs +434 -0
- package/lib/checkpoint.mjs +637 -0
- package/lib/child-lifetime.mjs +71 -0
- package/lib/cli-args.mjs +1255 -0
- package/lib/code-review.mjs +1382 -0
- package/lib/colour.mjs +82 -0
- package/lib/command.mjs +2847 -0
- package/lib/compact.mjs +1151 -0
- package/lib/completion.mjs +515 -0
- package/lib/creative-engines.mjs +779 -0
- package/lib/db-inspect.mjs +1624 -0
- package/lib/delete.mjs +111 -0
- package/lib/design-loop.mjs +570 -0
- package/lib/diff-preview.mjs +1044 -0
- package/lib/doctor.mjs +2139 -0
- package/lib/dropped.mjs +216 -0
- package/lib/edit-diagnostics.mjs +277 -0
- package/lib/edit.mjs +460 -0
- package/lib/env-file.mjs +250 -0
- package/lib/escalate.mjs +702 -0
- package/lib/evaluate.mjs +284 -0
- package/lib/fetch-text.mjs +952 -0
- package/lib/fleet-budget.mjs +256 -0
- package/lib/gh.mjs +1536 -0
- package/lib/git.mjs +1341 -0
- package/lib/github.mjs +261 -0
- package/lib/h2.mjs +194 -0
- package/lib/handoff.mjs +417 -0
- package/lib/hooks.mjs +626 -0
- package/lib/http-probe.mjs +907 -0
- package/lib/image-director.mjs +322 -0
- package/lib/image-edit.mjs +522 -0
- package/lib/imagegen.mjs +998 -0
- package/lib/interrupt.mjs +234 -0
- package/lib/learned.mjs +353 -0
- package/lib/lease-watch.mjs +115 -0
- package/lib/lease.mjs +868 -0
- package/lib/localize.mjs +834 -0
- package/lib/log-tail.mjs +1052 -0
- package/lib/login.mjs +157 -0
- package/lib/lsp.mjs +1613 -0
- package/lib/mcp-consent.mjs +377 -0
- package/lib/mcp-defaults.mjs +780 -0
- package/lib/mcp-server.mjs +1343 -0
- package/lib/mcp.mjs +1263 -0
- package/lib/media.mjs +1283 -0
- package/lib/memory-workspace.mjs +179 -0
- package/lib/model-json.mjs +132 -0
- package/lib/model-tier.mjs +171 -0
- package/lib/model.mjs +1445 -0
- package/lib/parallel.mjs +144 -0
- package/lib/perchance.mjs +210 -0
- package/lib/plan-coherence.mjs +1461 -0
- package/lib/plan-ledger.mjs +981 -0
- package/lib/plan.mjs +461 -0
- package/lib/policy.mjs +783 -0
- package/lib/prefix-order.mjs +38 -0
- package/lib/project-memory.mjs +127 -0
- package/lib/prompt.mjs +109 -0
- package/lib/python.mjs +862 -0
- package/lib/rcfile.mjs +853 -0
- package/lib/read-window.mjs +743 -0
- package/lib/refute-tools.mjs +34 -0
- package/lib/refute.mjs +806 -0
- package/lib/repl-driver.mjs +264 -0
- package/lib/repl.mjs +324 -0
- package/lib/replay.mjs +1218 -0
- package/lib/repo-map.mjs +1101 -0
- package/lib/report.mjs +419 -0
- package/lib/search-rank.mjs +99 -0
- package/lib/search.mjs +659 -0
- package/lib/secret-paths.mjs +54 -0
- package/lib/session.mjs +1017 -0
- package/lib/skills.mjs +703 -0
- package/lib/slash.mjs +356 -0
- package/lib/spawn-argv.mjs +1151 -0
- package/lib/spend.mjs +250 -0
- package/lib/steer.mjs +280 -0
- package/lib/stream.mjs +253 -0
- package/lib/stuck.mjs +712 -0
- package/lib/subagent.mjs +749 -0
- package/lib/terminal-graphics.mjs +171 -0
- package/lib/tool-prefix.mjs +226 -0
- package/lib/tool-shortlist.mjs +162 -0
- package/lib/tools.mjs +2333 -0
- package/lib/tsserver.mjs +423 -0
- package/lib/turn.mjs +5672 -0
- package/lib/untrusted-block.mjs +271 -0
- package/lib/verify-claim.mjs +299 -0
- package/lib/vision.mjs +330 -0
- package/lib/voice-task.mjs +561 -0
- package/lib/warm-provider.mjs +255 -0
- package/lib/websearch.mjs +401 -0
- package/lib/workspace.mjs +928 -0
- package/lib/write-approval.mjs +235 -0
- package/lib/write-many.mjs +162 -0
- package/package.json +62 -0
- package/scripts/bundle.mjs +768 -0
- package/scripts/cache-floor.mjs +176 -0
- package/scripts/machine.mjs +226 -0
- package/scripts/test.mjs +139 -0
- package/skills/accessibility.md +87 -0
- package/skills/acuvo-design-system.md +123 -0
- package/skills/animation.md +84 -0
- package/skills/api-design.md +82 -0
- package/skills/auth-and-sessions.md +78 -0
- package/skills/build-with-a-framework.md +101 -0
- package/skills/colour-and-contrast.md +112 -0
- package/skills/creative-engines.md +81 -0
- package/skills/css-layout.md +85 -0
- package/skills/data-and-charts.md +77 -0
- package/skills/debugging.md +76 -0
- package/skills/designing-by-looking.md +84 -0
- package/skills/error-handling.md +78 -0
- package/skills/forms-and-validation.md +93 -0
- package/skills/nextjs-app-router.md +75 -0
- package/skills/page-composition.md +103 -0
- package/skills/performance.md +77 -0
- package/skills/plan-before-building.md +52 -0
- package/skills/planning-and-delegating.md +72 -0
- package/skills/refactoring.md +70 -0
- package/skills/security-basics.md +76 -0
- package/skills/state-management.md +73 -0
- package/skills/supabase-multitenant.md +72 -0
- package/skills/typescript-strict.md +90 -0
- package/skills/typography.md +135 -0
- package/skills/verify-your-own-work.md +62 -0
- package/skills/web-app-quality.md +62 -0
- package/skills/working-in-the-background.md +64 -0
package/lib/compact.mjs
ADDED
|
@@ -0,0 +1,1151 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ── lib/compact.mjs — THE HORIZON ──────────────────────────────────────────
|
|
3
|
+
*
|
|
4
|
+
* ⚠️⭐⭐ THE DEFECT THIS EXISTS FOR: THERE IS NO CONTEXT MANAGEMENT AT ALL.
|
|
5
|
+
* Grep `lib/turn.mjs` for history trimming and you find none — `messages` only
|
|
6
|
+
* ever grows. Every round appends an assistant message plus one tool result per
|
|
7
|
+
* call, and nothing is ever removed, replaced or shrunk. The round cap hides it
|
|
8
|
+
* today: at 8 rounds a normal session simply ends before the wall. Raise the
|
|
9
|
+
* horizon and the wall arrives instead of the cap, as a provider 400 or a
|
|
10
|
+
* silent truncation, and the run dies in the middle of the work.
|
|
11
|
+
*
|
|
12
|
+
* That is the single biggest structural limit on this product. It is why the
|
|
13
|
+
* CLI is an 8-round tool rather than an all-day one.
|
|
14
|
+
*
|
|
15
|
+
* ── ⭐ WHY THIS IS STRUCTURAL AND NOT A SUMMARISER ─────────────────────────
|
|
16
|
+
*
|
|
17
|
+
* The obvious fix is "ask the model to summarise the history". It is the wrong
|
|
18
|
+
* fix here, for three reasons that are specific to an AGENTIC transcript:
|
|
19
|
+
*
|
|
20
|
+
* 1. THE BULK IS NOT PROSE. Measured on a real session (see the module test
|
|
21
|
+
* and the lane report): the assistant's own words are a rounding error.
|
|
22
|
+
* The weight is tool RESULTS — file bodies, command dumps, hit lists.
|
|
23
|
+
* 2. MOST OF THAT WEIGHT IS PROVABLY STALE. The same file read three times is
|
|
24
|
+
* two dead copies and one live one, and which is which is a fact about the
|
|
25
|
+
* transcript, not a judgement call. No model is needed to know it.
|
|
26
|
+
* 3. A SUMMARISER COSTS A MODEL CALL, ADDS LATENCY, AND CAN LIE. Structural
|
|
27
|
+
* compaction costs nothing, is deterministic, and can be tested. When the
|
|
28
|
+
* free, exact, verifiable version handles the bulk, spending a paid,
|
|
29
|
+
* approximate, unverifiable one on the same bytes is a bad trade.
|
|
30
|
+
*
|
|
31
|
+
* So this module is PURE: data in, data out. No LLM, no network, no clock, no
|
|
32
|
+
* `Math.random()`. Given the same transcript and the same options it returns
|
|
33
|
+
* the same bytes, which is what makes it testable at all.
|
|
34
|
+
*
|
|
35
|
+
* ── ⚠️⚠️ THE INVARIANT. IT IS THE WHOLE JOB ────────────────────────────────
|
|
36
|
+
*
|
|
37
|
+
* Every assistant message carrying `tool_calls` MUST keep a matching `tool`
|
|
38
|
+
* message for EVERY call id, in order. OpenAI-shaped APIs hard-reject a history
|
|
39
|
+
* where a `tool_call` has no answering `tool` message — and the rejection is a
|
|
40
|
+
* 400 on that round AND on every round after it, because the broken history is
|
|
41
|
+
* resent each time. From the outside the tool appears to break completely and
|
|
42
|
+
* mysteriously, having worked a minute earlier.
|
|
43
|
+
*
|
|
44
|
+
* ⭐ SO COMPACTION REPLACES A RESULT'S CONTENT AND NEVER REMOVES A MESSAGE.
|
|
45
|
+
* That is not a guideline in here, it is the shape of the code: the output
|
|
46
|
+
* array is built by `map`, index for index, and the only field that can differ
|
|
47
|
+
* is `content` on a `role: 'tool'` message. Nothing else is reachable.
|
|
48
|
+
*
|
|
49
|
+
* ⭐⭐ AND IT IS CHECKED, NOT ASSUMED. `structuralFingerprint()` serialises the
|
|
50
|
+
* skeleton — roles, call ids, result ids, in order, with all content stripped —
|
|
51
|
+
* and `compactMessages` compares the fingerprint of its own output against its
|
|
52
|
+
* input. If they ever differ the compactor RETURNS THE INPUT UNCHANGED and says
|
|
53
|
+
* so in the report. A compactor that would break the conversation hands the
|
|
54
|
+
* conversation back. Losing the saving is a bad round; losing the session is
|
|
55
|
+
* the product.
|
|
56
|
+
*
|
|
57
|
+
* ── WHAT IS NEVER COMPACTED ───────────────────────────────────────────────
|
|
58
|
+
*
|
|
59
|
+
* · the system prompt(s) and the opening task — the cacheable prefix, and a
|
|
60
|
+
* history without them is not a session, it is a different session;
|
|
61
|
+
* · every message that is not `role: 'tool'` — assistant notes, plan banners
|
|
62
|
+
* and user turns are cheap and load-bearing, and touching them is how a
|
|
63
|
+
* resumed model starts reasoning about instructions it was never given;
|
|
64
|
+
* · the most recent `keepLastRounds` tool rounds, verbatim;
|
|
65
|
+
* · anything the NEWEST assistant message names. If the model just said "the
|
|
66
|
+
* earlier version of lib/slug.mjs", compacting that read deletes the thing
|
|
67
|
+
* it is in the middle of using.
|
|
68
|
+
*
|
|
69
|
+
* ── ⚠️ TOKENS ARE ESTIMATED AND THE REPORT SAYS SO ────────────────────────
|
|
70
|
+
*
|
|
71
|
+
* There is no tokeniser here and there must not be one — that is an npm
|
|
72
|
+
* dependency, and this package's headline property is that it has none. Every
|
|
73
|
+
* token number this module produces is `chars / 4`, the industry approximation
|
|
74
|
+
* for English prose, and it is wrong in both directions: CJK and emoji cost
|
|
75
|
+
* more per character, long repeated identifiers cost less. `report.method`
|
|
76
|
+
* says this in words, because a number presented as exact when it is a guess is
|
|
77
|
+
* the specific dishonesty this repo hunts.
|
|
78
|
+
*/
|
|
79
|
+
|
|
80
|
+
/** The industry approximation. English prose, roughly. See the header. */
|
|
81
|
+
export const CHARS_PER_TOKEN = 4;
|
|
82
|
+
|
|
83
|
+
/**
|
|
84
|
+
* Every provider wraps each message in some role/format scaffolding that costs
|
|
85
|
+
* tokens the content does not account for. ⚠️ This is a STATED GUESS, not a
|
|
86
|
+
* measurement — it is here so a transcript of 400 tiny messages does not
|
|
87
|
+
* estimate as almost free, which is the direction that hurts.
|
|
88
|
+
*/
|
|
89
|
+
export const MESSAGE_OVERHEAD_CHARS = 16;
|
|
90
|
+
|
|
91
|
+
export const DEFAULT_BUDGET_TOKENS = 24_000;
|
|
92
|
+
export const DEFAULT_KEEP_LAST_ROUNDS = 2;
|
|
93
|
+
/** Past this, a single tool result is clamped head-and-tail. */
|
|
94
|
+
export const DEFAULT_MAX_RESULT_CHARS = 4_000;
|
|
95
|
+
/**
|
|
96
|
+
* The clamp halves when a fixed one cannot reach the budget, and stops here.
|
|
97
|
+
* ⚠️ Below roughly this, a clamped result is nothing but its own apology — a
|
|
98
|
+
* hole in the history dressed as a message. Refusing to reach the budget is the
|
|
99
|
+
* honest failure; a transcript of stubs is the dishonest one.
|
|
100
|
+
*/
|
|
101
|
+
export const MIN_CLAMP_CHARS = 400;
|
|
102
|
+
|
|
103
|
+
export const TOKEN_ESTIMATE_NOTE =
|
|
104
|
+
'every token number here is an ESTIMATE — characters divided by 4, the industry approximation. '
|
|
105
|
+
+ 'This package ships zero dependencies and therefore carries no tokeniser, so these are not exact counts '
|
|
106
|
+
+ 'and are least accurate on non-English text.';
|
|
107
|
+
|
|
108
|
+
/* ── tool vocabulary ─────────────────────────────────────────────────────── */
|
|
109
|
+
|
|
110
|
+
/** Tools whose result is a snapshot of a file or directory at a point in time. */
|
|
111
|
+
const READ_TOOLS = new Set(['read_file', 'read_lines', 'read_around', 'list_dir', 'read_skill']);
|
|
112
|
+
|
|
113
|
+
/** Tools whose result is the output of running something. */
|
|
114
|
+
const COMMAND_TOOLS = new Set(['run_command', 'run_program', 'evaluate', 'check_types', 'check_acceptance']);
|
|
115
|
+
|
|
116
|
+
/** Tools whose result is a list of places to look. */
|
|
117
|
+
const SEARCH_TOOLS = new Set(['search_text', 'find_files', 'find_definition', 'find_references', 'list_symbols']);
|
|
118
|
+
|
|
119
|
+
/**
|
|
120
|
+
* Tools that constitute "acting on" a path. A search whose hits were later
|
|
121
|
+
* opened or edited has done its job; the hit list is navigation exhaust.
|
|
122
|
+
*/
|
|
123
|
+
const ACT_TOOLS = new Set([
|
|
124
|
+
'read_file', 'read_lines', 'read_around', 'edit_file', 'write_file', 'delete_file', 'make_document',
|
|
125
|
+
]);
|
|
126
|
+
|
|
127
|
+
/* ── estimation ──────────────────────────────────────────────────────────── */
|
|
128
|
+
|
|
129
|
+
/**
|
|
130
|
+
* @param {unknown} text
|
|
131
|
+
* @param {number} [charsPerToken]
|
|
132
|
+
* @returns {number} an ESTIMATE, rounded up. Non-strings are 0 — they are not
|
|
133
|
+
* silently coerced, because `estimateTokens(someObject)` returning a number
|
|
134
|
+
* derived from "[object Object]" is a fake measurement.
|
|
135
|
+
*/
|
|
136
|
+
export function estimateTokens(text, charsPerToken = CHARS_PER_TOKEN) {
|
|
137
|
+
if (typeof text !== 'string' || text.length === 0) return 0;
|
|
138
|
+
const per = charsPerToken > 0 ? charsPerToken : CHARS_PER_TOKEN;
|
|
139
|
+
return Math.ceil(text.length / per);
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
/**
|
|
143
|
+
* The estimated character weight of ONE message, un-rounded.
|
|
144
|
+
*
|
|
145
|
+
* ⚠️ COUNTS `tool_calls`, AND THE FIRST VERSION DID NOT. An assistant message
|
|
146
|
+
* that dispatches six calls often has empty `content` and several hundred
|
|
147
|
+
* characters of JSON arguments; counting only `content` made the heaviest
|
|
148
|
+
* messages in a parallel-call transcript estimate as free.
|
|
149
|
+
*/
|
|
150
|
+
function messageChars(m) {
|
|
151
|
+
if (!m || typeof m !== 'object') return MESSAGE_OVERHEAD_CHARS;
|
|
152
|
+
let chars = MESSAGE_OVERHEAD_CHARS;
|
|
153
|
+
chars += typeof m.role === 'string' ? m.role.length : 0;
|
|
154
|
+
chars += contentChars(m);
|
|
155
|
+
if (typeof m.name === 'string') chars += m.name.length;
|
|
156
|
+
if (typeof m.tool_call_id === 'string') chars += m.tool_call_id.length;
|
|
157
|
+
if (Array.isArray(m.tool_calls)) {
|
|
158
|
+
for (const c of m.tool_calls) {
|
|
159
|
+
try { chars += JSON.stringify(c ?? null).length; } catch { chars += 64; }
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
return chars;
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
function contentChars(m) {
|
|
166
|
+
const c = m?.content;
|
|
167
|
+
if (typeof c === 'string') return c.length;
|
|
168
|
+
if (c == null) return 0;
|
|
169
|
+
try { return JSON.stringify(c).length; } catch { return 0; }
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
/**
|
|
173
|
+
* @param {unknown} messages
|
|
174
|
+
* @param {number} [charsPerToken]
|
|
175
|
+
* @returns {number} an ESTIMATE of the whole payload, rounded up ONCE at the
|
|
176
|
+
* end — per-message rounding would drift by hundreds of tokens over a long
|
|
177
|
+
* transcript and make `before - after` disagree with the sum of the parts.
|
|
178
|
+
*/
|
|
179
|
+
export function estimateMessagesTokens(messages, charsPerToken = CHARS_PER_TOKEN) {
|
|
180
|
+
if (!Array.isArray(messages) || messages.length === 0) return 0;
|
|
181
|
+
const per = charsPerToken > 0 ? charsPerToken : CHARS_PER_TOKEN;
|
|
182
|
+
let chars = 0;
|
|
183
|
+
for (const m of messages) chars += messageChars(m);
|
|
184
|
+
return Math.ceil(chars / per);
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
/**
|
|
188
|
+
* ── ⚠️⚠️ THE TOOL SCHEMAS ARE PART OF THE PAYLOAD AND WERE COUNTED AS ZERO ──
|
|
189
|
+
*
|
|
190
|
+
* `estimateMessagesTokens` measures MESSAGES. A chat completion sends messages
|
|
191
|
+
* AND the `tools` array, and the provider counts both against the context — so
|
|
192
|
+
* every round was under-measured by the whole offer.
|
|
193
|
+
*
|
|
194
|
+
* Measured 2026-08-13 on a bare machine: 34 tools, 26,287 characters of JSON
|
|
195
|
+
* schema, **~6,572 tokens — 6.8% of the 96,000-token budget, invisible**. With
|
|
196
|
+
* MCP servers attached it is far larger, because their schemas are written by
|
|
197
|
+
* somebody else and nobody here is trimming them.
|
|
198
|
+
*
|
|
199
|
+
* ⭐ THE FAILURE THIS PREVENTS IS THE WORST-SHAPED ONE AVAILABLE: the compactor
|
|
200
|
+
* reports headroom while the provider returns a context-length error. It looks
|
|
201
|
+
* like a provider bug, it is not reproducible from the transcript we kept, and
|
|
202
|
+
* compaction can never rescue it — compaction rewrites MESSAGES, and the offer
|
|
203
|
+
* is not a message.
|
|
204
|
+
*
|
|
205
|
+
* ⚠️ Deliberately the same crude chars/4 as the rest of this module rather than
|
|
206
|
+
* a real tokeniser. It is an ESTIMATE used to decide when to compact, and a
|
|
207
|
+
* consistent estimate is worth more here than an accurate one that disagrees
|
|
208
|
+
* with the number it is compared against.
|
|
209
|
+
*/
|
|
210
|
+
export function estimateToolOfferTokens(tools, charsPerToken = CHARS_PER_TOKEN) {
|
|
211
|
+
if (!Array.isArray(tools) || tools.length === 0) return 0;
|
|
212
|
+
const per = charsPerToken > 0 ? charsPerToken : CHARS_PER_TOKEN;
|
|
213
|
+
let chars = 0;
|
|
214
|
+
for (const t of tools) {
|
|
215
|
+
try { chars += JSON.stringify(t)?.length ?? 0; } catch { /* a circular schema is not worth crashing a run over */ }
|
|
216
|
+
}
|
|
217
|
+
return Math.ceil(chars / per);
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
/* ── structure ───────────────────────────────────────────────────────────── */
|
|
221
|
+
|
|
222
|
+
/**
|
|
223
|
+
* Split a transcript into the untouchable HEAD and the rounds after it.
|
|
224
|
+
*
|
|
225
|
+
* The head is the leading `system` messages plus the first `user` message —
|
|
226
|
+
* the rules and the task. A "round" is an assistant message with `tool_calls`
|
|
227
|
+
* plus the `tool` messages answering it; anything else (a plan banner, a bare
|
|
228
|
+
* assistant note, a follow-up user turn) is its own single-message round so
|
|
229
|
+
* that indices stay total and nothing falls between the cracks.
|
|
230
|
+
*
|
|
231
|
+
* @param {unknown[]} messages
|
|
232
|
+
* @returns {{ head: number[], rounds: number[][] }} INDICES, not messages —
|
|
233
|
+
* the caller needs to address the original array by position.
|
|
234
|
+
*/
|
|
235
|
+
export function groupRounds(messages) {
|
|
236
|
+
const head = [];
|
|
237
|
+
const rounds = [];
|
|
238
|
+
if (!Array.isArray(messages)) return { head, rounds };
|
|
239
|
+
let i = 0;
|
|
240
|
+
while (i < messages.length && messages[i]?.role === 'system') head.push(i++);
|
|
241
|
+
if (i < messages.length && messages[i]?.role === 'user') head.push(i++);
|
|
242
|
+
while (i < messages.length) {
|
|
243
|
+
const m = messages[i];
|
|
244
|
+
if (m?.role === 'assistant' && Array.isArray(m.tool_calls) && m.tool_calls.length > 0) {
|
|
245
|
+
const group = [i++];
|
|
246
|
+
while (i < messages.length && messages[i]?.role === 'tool') group.push(i++);
|
|
247
|
+
rounds.push(group);
|
|
248
|
+
continue;
|
|
249
|
+
}
|
|
250
|
+
rounds.push([i++]);
|
|
251
|
+
}
|
|
252
|
+
return { head, rounds };
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
/**
|
|
256
|
+
* ⭐⭐ THE SKELETON, WITH EVERY BYTE OF CONTENT REMOVED. Roles, call ids and
|
|
257
|
+
* result ids, in order. Two transcripts with the same fingerprint are the same
|
|
258
|
+
* conversation as far as an OpenAI-shaped provider is concerned; they differ
|
|
259
|
+
* only in what was said, never in what was asked and answered.
|
|
260
|
+
*
|
|
261
|
+
* This is the guard `compactMessages` runs against itself. A compactor that
|
|
262
|
+
* dropped, reordered or inserted a message changes the fingerprint, and the
|
|
263
|
+
* change is caught before the payload ever reaches a provider.
|
|
264
|
+
*
|
|
265
|
+
* @param {unknown[]} messages
|
|
266
|
+
* @returns {string}
|
|
267
|
+
*/
|
|
268
|
+
export function structuralFingerprint(messages) {
|
|
269
|
+
if (!Array.isArray(messages)) return '[]';
|
|
270
|
+
const skeleton = messages.map((m) => {
|
|
271
|
+
const row = { r: m?.role ?? null };
|
|
272
|
+
if (typeof m?.tool_call_id === 'string') row.a = m.tool_call_id;
|
|
273
|
+
if (Array.isArray(m?.tool_calls)) {
|
|
274
|
+
row.c = m.tool_calls.map((c) => `${c?.id ?? ''}:${c?.function?.name ?? c?.name ?? ''}`);
|
|
275
|
+
}
|
|
276
|
+
return row;
|
|
277
|
+
});
|
|
278
|
+
return JSON.stringify(skeleton);
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
/**
|
|
282
|
+
* Is every declared call answered, in order?
|
|
283
|
+
*
|
|
284
|
+
* ⚠️ AN ORPHAN `tool` MESSAGE IS NOT A VIOLATION OF THIS RULE. The provider
|
|
285
|
+
* rejects an unanswered CALL; a stray result with no declaring call is a
|
|
286
|
+
* different (and rarer) problem, and reporting it here would make this checker
|
|
287
|
+
* refuse histories the API accepts — a check that fails correct work.
|
|
288
|
+
*
|
|
289
|
+
* @param {unknown[]} messages
|
|
290
|
+
* @returns {{ ok: boolean, checked: number, unanswered: string[] }}
|
|
291
|
+
*/
|
|
292
|
+
export function verifyToolPairing(messages) {
|
|
293
|
+
const unanswered = [];
|
|
294
|
+
let checked = 0;
|
|
295
|
+
if (!Array.isArray(messages)) return { ok: true, checked: 0, unanswered };
|
|
296
|
+
for (let i = 0; i < messages.length; i += 1) {
|
|
297
|
+
const m = messages[i];
|
|
298
|
+
if (m?.role !== 'assistant' || !Array.isArray(m.tool_calls) || m.tool_calls.length === 0) continue;
|
|
299
|
+
const answers = [];
|
|
300
|
+
let j = i + 1;
|
|
301
|
+
while (j < messages.length && messages[j]?.role === 'tool') answers.push(messages[j++].tool_call_id);
|
|
302
|
+
const seen = new Set(answers);
|
|
303
|
+
for (const c of m.tool_calls) {
|
|
304
|
+
checked += 1;
|
|
305
|
+
if (!seen.has(c?.id)) unanswered.push(String(c?.id ?? '(no id)'));
|
|
306
|
+
}
|
|
307
|
+
}
|
|
308
|
+
return { ok: unanswered.length === 0, checked, unanswered };
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
/* ── request identity ────────────────────────────────────────────────────── */
|
|
312
|
+
|
|
313
|
+
/**
|
|
314
|
+
* ⚠️⭐ WHY SUPERSESSION IS KEYED ON THE WHOLE REQUEST AND NOT ON THE PATH.
|
|
315
|
+
*
|
|
316
|
+
* The tempting rule is "the same file read twice — keep the last". It is WRONG
|
|
317
|
+
* for windowed reads, and wrong in the expensive direction. `read_lines` of
|
|
318
|
+
* lines 1–200 followed by `read_lines` of lines 201–400 is the same path and
|
|
319
|
+
* two entirely different, entirely current pieces of information. Declaring the
|
|
320
|
+
* first superseded deletes half the file the model just paged through, and the
|
|
321
|
+
* model then reasons confidently about code it can no longer see.
|
|
322
|
+
*
|
|
323
|
+
* ⭐ So the identity is TOOL NAME + CANONICAL ARGUMENTS: the same request,
|
|
324
|
+
* issued again. That is conservative by construction — it can miss a saving,
|
|
325
|
+
* and it cannot invent one. Missing a saving costs context; inventing one costs
|
|
326
|
+
* correctness, and only one of those is recoverable.
|
|
327
|
+
*/
|
|
328
|
+
function requestKey(name, args) {
|
|
329
|
+
return `${name}|${canonicalJson(normaliseArgs(args))}`;
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
function normaliseArgs(args) {
|
|
333
|
+
if (!args || typeof args !== 'object') return {};
|
|
334
|
+
const out = {};
|
|
335
|
+
for (const [k, v] of Object.entries(args)) {
|
|
336
|
+
out[k] = k === 'path' || k === 'file' || k === 'dir' ? normalisePath(v) : v;
|
|
337
|
+
}
|
|
338
|
+
return out;
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
/** `./lib\slug.mjs` and `lib/slug.mjs` are the same file to every human. */
|
|
342
|
+
function normalisePath(value) {
|
|
343
|
+
if (typeof value !== 'string') return value;
|
|
344
|
+
let p = value.replace(/\\/g, '/').trim();
|
|
345
|
+
while (p.startsWith('./')) p = p.slice(2);
|
|
346
|
+
return p.replace(/\/{2,}/g, '/').replace(/\/+$/, '') || '/';
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
function canonicalJson(value) {
|
|
350
|
+
if (value === null || typeof value !== 'object') return JSON.stringify(value ?? null);
|
|
351
|
+
if (Array.isArray(value)) return `[${value.map(canonicalJson).join(',')}]`;
|
|
352
|
+
const keys = Object.keys(value).sort();
|
|
353
|
+
return `{${keys.map((k) => `${JSON.stringify(k)}:${canonicalJson(value[k])}`).join(',')}}`;
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
function parseCallArgs(call) {
|
|
357
|
+
const raw = call?.function?.arguments ?? call?.arguments;
|
|
358
|
+
if (raw && typeof raw === 'object') return raw;
|
|
359
|
+
if (typeof raw !== 'string' || raw.trim() === '') return {};
|
|
360
|
+
try {
|
|
361
|
+
const parsed = JSON.parse(raw);
|
|
362
|
+
return parsed && typeof parsed === 'object' ? parsed : {};
|
|
363
|
+
} catch {
|
|
364
|
+
// ⚠️ A model that emits malformed JSON arguments is common enough that
|
|
365
|
+
// throwing here would take the whole session down over a comma. An
|
|
366
|
+
// unparseable call simply has no identity, so it is never compacted.
|
|
367
|
+
return {};
|
|
368
|
+
}
|
|
369
|
+
}
|
|
370
|
+
|
|
371
|
+
/**
|
|
372
|
+
* Index every call and every result, and join them by id.
|
|
373
|
+
* @returns {{ calls: any[], results: any[] }}
|
|
374
|
+
*/
|
|
375
|
+
function indexTranscript(messages) {
|
|
376
|
+
const byId = new Map();
|
|
377
|
+
const calls = [];
|
|
378
|
+
const results = [];
|
|
379
|
+
messages.forEach((m, index) => {
|
|
380
|
+
if (m?.role === 'assistant' && Array.isArray(m.tool_calls)) {
|
|
381
|
+
for (const c of m.tool_calls) {
|
|
382
|
+
const name = c?.function?.name ?? c?.name ?? null;
|
|
383
|
+
const args = parseCallArgs(c);
|
|
384
|
+
const rec = { id: c?.id ?? null, name, args, index, path: pathOf(args) };
|
|
385
|
+
calls.push(rec);
|
|
386
|
+
if (rec.id != null && !byId.has(rec.id)) byId.set(rec.id, rec);
|
|
387
|
+
}
|
|
388
|
+
}
|
|
389
|
+
if (m?.role === 'tool') {
|
|
390
|
+
const call = m.tool_call_id != null ? byId.get(m.tool_call_id) : undefined;
|
|
391
|
+
const name = (typeof m.name === 'string' && m.name) ? m.name : (call?.name ?? null);
|
|
392
|
+
const args = call?.args ?? {};
|
|
393
|
+
results.push({
|
|
394
|
+
index,
|
|
395
|
+
id: m.tool_call_id ?? null,
|
|
396
|
+
name,
|
|
397
|
+
args,
|
|
398
|
+
path: pathOf(args),
|
|
399
|
+
key: name ? requestKey(name, args) : null,
|
|
400
|
+
chars: typeof m.content === 'string' ? m.content.length : -1,
|
|
401
|
+
});
|
|
402
|
+
}
|
|
403
|
+
});
|
|
404
|
+
return { calls, results };
|
|
405
|
+
}
|
|
406
|
+
|
|
407
|
+
function pathOf(args) {
|
|
408
|
+
const p = args?.path ?? args?.file ?? args?.dir;
|
|
409
|
+
return typeof p === 'string' && p ? normalisePath(p) : null;
|
|
410
|
+
}
|
|
411
|
+
|
|
412
|
+
function subjectLabel(name, args) {
|
|
413
|
+
if (READ_TOOLS.has(name)) return pathOf(args) ?? name;
|
|
414
|
+
if (COMMAND_TOOLS.has(name)) {
|
|
415
|
+
if (typeof args?.command === 'string') return args.command;
|
|
416
|
+
if (Array.isArray(args?.argv)) return args.argv.join(' ');
|
|
417
|
+
return name;
|
|
418
|
+
}
|
|
419
|
+
if (SEARCH_TOOLS.has(name)) {
|
|
420
|
+
const q = args?.pattern ?? args?.query ?? args?.name ?? args?.symbol;
|
|
421
|
+
return typeof q === 'string' ? q : name;
|
|
422
|
+
}
|
|
423
|
+
return pathOf(args) ?? name;
|
|
424
|
+
}
|
|
425
|
+
|
|
426
|
+
/* ── text surgery ────────────────────────────────────────────────────────── */
|
|
427
|
+
|
|
428
|
+
/**
|
|
429
|
+
* Slice without splitting a surrogate pair.
|
|
430
|
+
*
|
|
431
|
+
* ⚠️ FOUND BY TESTING A LEGITIMATE SHAPE, NOT A DEFECT. `String.slice` works on
|
|
432
|
+
* UTF-16 code units, so cutting a run of emoji at an arbitrary offset produces
|
|
433
|
+
* a lone high surrogate — a replacement character on screen and, worse, a byte
|
|
434
|
+
* sequence that is not valid UTF-8 going back over the wire.
|
|
435
|
+
*/
|
|
436
|
+
function safeHead(s, n) {
|
|
437
|
+
if (n >= s.length) return s;
|
|
438
|
+
let end = n;
|
|
439
|
+
const code = s.charCodeAt(end - 1);
|
|
440
|
+
if (code >= 0xd800 && code <= 0xdbff) end -= 1; // do not leave a lone high surrogate
|
|
441
|
+
return s.slice(0, Math.max(0, end));
|
|
442
|
+
}
|
|
443
|
+
|
|
444
|
+
function safeTail(s, n) {
|
|
445
|
+
if (n >= s.length) return s;
|
|
446
|
+
let start = s.length - n;
|
|
447
|
+
const code = s.charCodeAt(start);
|
|
448
|
+
if (code >= 0xdc00 && code <= 0xdfff) start += 1; // do not start on a lone low surrogate
|
|
449
|
+
return s.slice(Math.min(s.length, start));
|
|
450
|
+
}
|
|
451
|
+
|
|
452
|
+
const n = (x) => x.toLocaleString('en-US');
|
|
453
|
+
|
|
454
|
+
/**
|
|
455
|
+
* ── ⚠️ THE FIRST VERSION OF THIS WAS TOO LOOSE, AND THE TEST CAUGHT IT ──────
|
|
456
|
+
*
|
|
457
|
+
* It was `/(error|failed|assertion|…)/i`, on the reasoning that keeping one
|
|
458
|
+
* line too many costs 80 characters and keeping none costs the model the reason
|
|
459
|
+
* its build broke. That reasoning is right and the regex was still wrong: run
|
|
460
|
+
* it over real TAP output and `ok 3 - filler assertion number 0` matches, so
|
|
461
|
+
* the "first failing line" the compactor preserved was a PASSING line, and the
|
|
462
|
+
* actual failure — the one fact worth keeping — was dropped. A stale result
|
|
463
|
+
* that keeps the wrong line is worse than one that keeps none, because it reads
|
|
464
|
+
* as evidence.
|
|
465
|
+
*
|
|
466
|
+
* ⭐ SO IT MATCHES NAMED CONVENTIONS, NOT KEYWORDS. Each entry below is a
|
|
467
|
+
* specific runner's specific failure format. Anchoring and case matter: `FAIL`
|
|
468
|
+
* at the start of a line is jest, `fail` inside a sentence is prose.
|
|
469
|
+
*/
|
|
470
|
+
const FAILURE_PATTERNS = [
|
|
471
|
+
/^\s*not ok\b/, // TAP / node:test
|
|
472
|
+
/^\s*(FAIL|FAILED|ERROR|ERR)\b/, // jest, mocha, ctest, many CI runners
|
|
473
|
+
/^\s*[✗✘✖×⨯]\s/, // vitest, ava, tap-spec
|
|
474
|
+
/^\s*npm ERR!/, // npm
|
|
475
|
+
/^\s*panic:/, // go
|
|
476
|
+
/^Traceback \(most recent call last\)/, // python
|
|
477
|
+
/\b(Assertion|Type|Reference|Syntax|Range|Eval|URI)Error\b/, // js runtimes
|
|
478
|
+
/(^|\s)Error:\s/, // thrown errors, tsc --pretty false
|
|
479
|
+
/:\s*error\s(TS\d+|[A-Z]{1,4}\d+)/, // tsc / msvc style "file(1,2): error TS2345"
|
|
480
|
+
/\s{2}error\s{2}/, // eslint stylish
|
|
481
|
+
/^\s*\d+\)\s/, // mocha's numbered failure list
|
|
482
|
+
];
|
|
483
|
+
|
|
484
|
+
function isFailureLine(line) {
|
|
485
|
+
for (const rx of FAILURE_PATTERNS) if (rx.test(line)) return true;
|
|
486
|
+
return false;
|
|
487
|
+
}
|
|
488
|
+
|
|
489
|
+
/**
|
|
490
|
+
* Keep the header of a command result (the `$ cmd` line and the exit/timeout
|
|
491
|
+
* line, both written by `command.mjs`'s `formatRunForModel`) plus the first
|
|
492
|
+
* failing line, and drop the rest.
|
|
493
|
+
*/
|
|
494
|
+
function gutCommandOutput(text) {
|
|
495
|
+
const lines = text.split('\n');
|
|
496
|
+
const kept = [];
|
|
497
|
+
// The two header lines, when they are there. A result that does not look like
|
|
498
|
+
// formatRunForModel output still keeps its first two lines, which is the
|
|
499
|
+
// honest fallback: something is better than a bare stub.
|
|
500
|
+
for (let i = 0; i < Math.min(2, lines.length); i += 1) kept.push(lines[i]);
|
|
501
|
+
let firstFailure = null;
|
|
502
|
+
for (let i = 2; i < lines.length; i += 1) {
|
|
503
|
+
if (isFailureLine(lines[i])) { firstFailure = lines[i]; break; }
|
|
504
|
+
}
|
|
505
|
+
if (firstFailure !== null) kept.push(firstFailure.length > 400 ? `${safeHead(firstFailure, 400)}…` : firstFailure);
|
|
506
|
+
return kept.join('\n');
|
|
507
|
+
}
|
|
508
|
+
|
|
509
|
+
/* ── the passes ──────────────────────────────────────────────────────────── */
|
|
510
|
+
|
|
511
|
+
/**
|
|
512
|
+
* Every pass returns candidates OLDEST FIRST. Order matters twice over: oldest
|
|
513
|
+
* content is the least likely to be load-bearing, and processing in transcript
|
|
514
|
+
* order makes the result deterministic without a sort key.
|
|
515
|
+
*/
|
|
516
|
+
function passSupersededReads(messages, results, compactable) {
|
|
517
|
+
const lastIndexForKey = new Map();
|
|
518
|
+
// ⚠️ Built over ALL results, including protected ones. A protected recent read
|
|
519
|
+
// still SUPERSEDES an old one — protection says "do not compact this message",
|
|
520
|
+
// not "pretend this message does not exist".
|
|
521
|
+
for (const r of results) if (r.key && READ_TOOLS.has(r.name)) lastIndexForKey.set(r.key, r.index);
|
|
522
|
+
|
|
523
|
+
const out = [];
|
|
524
|
+
for (const r of results) {
|
|
525
|
+
if (!compactable.has(r.index) || !READ_TOOLS.has(r.name) || !r.key) continue;
|
|
526
|
+
const newest = lastIndexForKey.get(r.key);
|
|
527
|
+
if (newest === undefined || newest <= r.index) continue;
|
|
528
|
+
const label = subjectLabel(r.name, r.args);
|
|
529
|
+
out.push({
|
|
530
|
+
index: r.index,
|
|
531
|
+
pass: 'superseded-reads',
|
|
532
|
+
tool: r.name,
|
|
533
|
+
subject: label,
|
|
534
|
+
content:
|
|
535
|
+
`[superseded ${r.name} of ${label} — the identical read was issued again later in this session, and THAT result `
|
|
536
|
+
+ `is the current one. ${n(r.chars)} characters of stale content (~${n(estimateTokens(messages[r.index].content))} `
|
|
537
|
+
+ 'estimated tokens) were dropped here by the context compactor. Nothing about the file changed; only this stale copy was removed.]',
|
|
538
|
+
});
|
|
539
|
+
}
|
|
540
|
+
return out;
|
|
541
|
+
}
|
|
542
|
+
|
|
543
|
+
function passStaleCommands(messages, results, compactable) {
|
|
544
|
+
const lastIndexForKey = new Map();
|
|
545
|
+
for (const r of results) if (r.key && COMMAND_TOOLS.has(r.name)) lastIndexForKey.set(r.key, r.index);
|
|
546
|
+
|
|
547
|
+
const out = [];
|
|
548
|
+
for (const r of results) {
|
|
549
|
+
if (!compactable.has(r.index) || !COMMAND_TOOLS.has(r.name) || !r.key) continue;
|
|
550
|
+
const newest = lastIndexForKey.get(r.key);
|
|
551
|
+
if (newest === undefined || newest <= r.index) continue;
|
|
552
|
+
const original = messages[r.index].content;
|
|
553
|
+
const gutted = gutCommandOutput(original);
|
|
554
|
+
const removed = original.length - gutted.length;
|
|
555
|
+
// ⚠️ A "compaction" that grows the message is not a compaction. Short
|
|
556
|
+
// outputs hit this constantly, and without the guard the report would count
|
|
557
|
+
// savings that are losses.
|
|
558
|
+
if (removed <= 200) continue;
|
|
559
|
+
out.push({
|
|
560
|
+
index: r.index,
|
|
561
|
+
pass: 'stale-commands',
|
|
562
|
+
tool: r.name,
|
|
563
|
+
subject: subjectLabel(r.name, r.args),
|
|
564
|
+
content:
|
|
565
|
+
`${gutted}\n[stale command output — this exact command was re-run later in this session and the newer result `
|
|
566
|
+
+ `supersedes it. The exit status above and the first failing line are kept; ${n(removed)} characters of superseded `
|
|
567
|
+
+ `output (~${n(estimateTokens(original.slice(gutted.length)))} estimated tokens) were dropped by the context compactor.]`,
|
|
568
|
+
});
|
|
569
|
+
}
|
|
570
|
+
return out;
|
|
571
|
+
}
|
|
572
|
+
|
|
573
|
+
function passGiantResults(messages, results, compactable, maxResultChars) {
|
|
574
|
+
const out = [];
|
|
575
|
+
for (const r of results) {
|
|
576
|
+
if (!compactable.has(r.index)) continue;
|
|
577
|
+
const original = messages[r.index].content;
|
|
578
|
+
if (typeof original !== 'string' || original.length <= maxResultChars) continue;
|
|
579
|
+
const headChars = Math.max(1, Math.floor(maxResultChars * 0.6));
|
|
580
|
+
const tailChars = Math.max(1, maxResultChars - headChars);
|
|
581
|
+
const headText = safeHead(original, headChars);
|
|
582
|
+
const tailText = safeTail(original, tailChars);
|
|
583
|
+
const removed = original.length - headText.length - tailText.length;
|
|
584
|
+
if (removed <= 200) continue;
|
|
585
|
+
out.push({
|
|
586
|
+
index: r.index,
|
|
587
|
+
pass: 'giant-results',
|
|
588
|
+
tool: r.name ?? 'tool',
|
|
589
|
+
subject: subjectLabel(r.name, r.args),
|
|
590
|
+
content:
|
|
591
|
+
`${headText}\n\n[… ${n(removed)} characters removed from the MIDDLE of this ${r.name ?? 'tool'} result `
|
|
592
|
+
+ `(~${n(Math.ceil(removed / CHARS_PER_TOKEN))} estimated tokens) to fit the context budget. `
|
|
593
|
+
+ `The text before and after this marker is verbatim. Re-run the tool if you need the middle. …]\n\n${tailText}`,
|
|
594
|
+
});
|
|
595
|
+
}
|
|
596
|
+
return out;
|
|
597
|
+
}
|
|
598
|
+
|
|
599
|
+
/**
|
|
600
|
+
* ── ⭐⭐ THE FIFTH PASS: A RECEIPT, NOT A FRAGMENT ───────────────────────────
|
|
601
|
+
*
|
|
602
|
+
* ⚠️ THE FLOOR STILL SCALED WITH THE RESULT COUNT, AND TIGHTENING ONLY MOVED IT.
|
|
603
|
+
* `MIN_CLAMP_CHARS` is 400, and a clamped result is the 400 characters PLUS the
|
|
604
|
+
* marker explaining the cut — measured on this module's own text, **~714
|
|
605
|
+
* characters per result**. So a hundred-result transcript floors at ~71,000
|
|
606
|
+
* characters and a two-hundred-result one at ~143,000, whatever the budget says.
|
|
607
|
+
* The tightening loop above fixed the *fixed*-clamp plateau (4,000 → 400 per
|
|
608
|
+
* result); it did not fix the fact that the floor is still N × a constant, which
|
|
609
|
+
* is the thing that bites at the 60-round horizon this module exists to unlock.
|
|
610
|
+
*
|
|
611
|
+
* ⭐ SO THE LAST RESORT KEEPS ONE LINE PER RESULT, AND MAKES IT A POINTER. A
|
|
612
|
+
* 400-character head-and-tail of a source file is two fragments nobody can act
|
|
613
|
+
* on; the same 150 characters spent on *"read_file src/foo.ts — 8,000 chars,
|
|
614
|
+
* re-run read_file for src/foo.ts"* is a fact the model can DO something with.
|
|
615
|
+
* Recovery is the whole difference: the clamp leaves the model guessing what it
|
|
616
|
+
* lost, the receipt names the tool and the subject that fetch it back.
|
|
617
|
+
*
|
|
618
|
+
* ⚠️ AND THE COUNTER-ARGUMENT, KEPT VISIBLE BECAUSE IT IS NOT SILLY: for a
|
|
619
|
+
* COMMAND result the head-and-tail clamp really is better than a receipt — the
|
|
620
|
+
* head is the `$ cmd` line and the exit status, and those are the two facts
|
|
621
|
+
* worth keeping. That is why this runs LAST, below the tightening loop, and only
|
|
622
|
+
* on a transcript that is STILL over budget after everything else. Before today
|
|
623
|
+
* that transcript's honest report was `underBudget: false`; nothing that fits
|
|
624
|
+
* without this pass ever reaches it, so no run that works today changes at all.
|
|
625
|
+
*
|
|
626
|
+
* ⚠️ IT OVERWRITES ONLY A `giant-results` CLAMP, exactly like the tightening
|
|
627
|
+
* loop, and for the same reason: a superseded-read stub, a gutted command and a
|
|
628
|
+
* dead-search stub are all SMALLER than a receipt and strictly more informative,
|
|
629
|
+
* so replacing one would be a loss in both directions.
|
|
630
|
+
*/
|
|
631
|
+
const MAX_RECEIPT_SUBJECT_CHARS = 100;
|
|
632
|
+
|
|
633
|
+
function passReceipts(messages, results, compactable) {
|
|
634
|
+
const out = [];
|
|
635
|
+
for (const r of results) {
|
|
636
|
+
if (!compactable.has(r.index)) continue;
|
|
637
|
+
const original = messages[r.index].content;
|
|
638
|
+
if (typeof original !== 'string' || original.length === 0) continue;
|
|
639
|
+
const tool = r.name ?? 'tool';
|
|
640
|
+
const raw = subjectLabel(r.name, r.args);
|
|
641
|
+
const label = typeof raw === 'string' && raw !== tool
|
|
642
|
+
? (raw.length > MAX_RECEIPT_SUBJECT_CHARS ? `${safeHead(raw, MAX_RECEIPT_SUBJECT_CHARS)}…` : raw)
|
|
643
|
+
: '';
|
|
644
|
+
/**
|
|
645
|
+
* ⚠️ ONE LINE, NO NEWLINES. The point of the pass is that the floor becomes
|
|
646
|
+
* a line per result; a receipt that wrapped onto three would be a third of
|
|
647
|
+
* the saving, and the arithmetic in the header would stop being true.
|
|
648
|
+
*/
|
|
649
|
+
const where = label ? ` ${label}` : '';
|
|
650
|
+
const back = label ? `Re-run ${tool} for ${label}` : `Re-run ${tool}`;
|
|
651
|
+
out.push({
|
|
652
|
+
index: r.index,
|
|
653
|
+
pass: 'receipts',
|
|
654
|
+
tool,
|
|
655
|
+
subject: raw,
|
|
656
|
+
content:
|
|
657
|
+
`[dropped:${` ${tool}`}${where} — ${n(original.length)} chars (~${n(estimateTokens(original))} est. tokens) `
|
|
658
|
+
+ `removed WHOLE by the context compactor to fit the budget; nothing of it is kept. ${back} if you need it.]`,
|
|
659
|
+
});
|
|
660
|
+
}
|
|
661
|
+
return out;
|
|
662
|
+
}
|
|
663
|
+
|
|
664
|
+
function passDeadSearches(messages, results, calls, compactable) {
|
|
665
|
+
const out = [];
|
|
666
|
+
for (const r of results) {
|
|
667
|
+
if (!compactable.has(r.index) || !SEARCH_TOOLS.has(r.name)) continue;
|
|
668
|
+
const body = messages[r.index].content;
|
|
669
|
+
if (typeof body !== 'string' || body.length === 0) continue;
|
|
670
|
+
// Which of the paths this search surfaced were opened or edited AFTERWARDS?
|
|
671
|
+
// "Afterwards" is the load-bearing word: a file opened BEFORE the search was
|
|
672
|
+
// not found by it, and the search is still the only record of the others.
|
|
673
|
+
const acted = [];
|
|
674
|
+
for (const c of calls) {
|
|
675
|
+
if (c.index <= r.index || !ACT_TOOLS.has(c.name) || !c.path) continue;
|
|
676
|
+
if (body.includes(c.path) && !acted.includes(c.path)) acted.push(c.path);
|
|
677
|
+
}
|
|
678
|
+
if (acted.length === 0) continue;
|
|
679
|
+
const hits = body.split('\n').filter((l) => l.trim()).length;
|
|
680
|
+
const shown = acted.slice(0, 5).join(', ');
|
|
681
|
+
out.push({
|
|
682
|
+
index: r.index,
|
|
683
|
+
pass: 'dead-searches',
|
|
684
|
+
tool: r.name,
|
|
685
|
+
subject: subjectLabel(r.name, r.args),
|
|
686
|
+
content:
|
|
687
|
+
`[dead search — ${r.name} for "${subjectLabel(r.name, r.args)}" returned ${n(hits)} hit lines, and the files it `
|
|
688
|
+
+ `pointed at were opened or edited later in this session (${shown}${acted.length > 5 ? `, +${acted.length - 5} more` : ''}). `
|
|
689
|
+
+ `The hit list has served its purpose, so ${n(body.length)} characters (~${n(estimateTokens(body))} estimated tokens) `
|
|
690
|
+
+ 'were dropped by the context compactor. Re-run the search if you need the full list again.]',
|
|
691
|
+
});
|
|
692
|
+
}
|
|
693
|
+
return out;
|
|
694
|
+
}
|
|
695
|
+
|
|
696
|
+
/* ── the entry point ─────────────────────────────────────────────────────── */
|
|
697
|
+
|
|
698
|
+
/**
|
|
699
|
+
* @typedef {object} CompactAction
|
|
700
|
+
* @property {number} index position in the message array
|
|
701
|
+
* @property {string} pass which pass made the change
|
|
702
|
+
* @property {string} tool the tool whose result was rewritten
|
|
703
|
+
* @property {string} subject the file, command or query it was about
|
|
704
|
+
* @property {number} beforeChars
|
|
705
|
+
* @property {number} afterChars
|
|
706
|
+
*
|
|
707
|
+
* @typedef {object} CompactReport
|
|
708
|
+
* @property {string} method ⚠️ says out loud that the tokens are estimates
|
|
709
|
+
* @property {number} budgetTokens
|
|
710
|
+
* @property {number} keepLastRounds
|
|
711
|
+
* @property {number} beforeTokens
|
|
712
|
+
* @property {number} afterTokens
|
|
713
|
+
* @property {number} freedTokens
|
|
714
|
+
* @property {number} freedPercent
|
|
715
|
+
* @property {boolean} underBudget false means compaction could NOT reach the budget
|
|
716
|
+
* @property {number} totalMessages
|
|
717
|
+
* @property {number} totalResults
|
|
718
|
+
* @property {number} compactableResults
|
|
719
|
+
* @property {string[]} protectedByReference subjects the newest assistant message names
|
|
720
|
+
* @property {{ ok: boolean, checked: number, unanswered: string[] }} inputPairing
|
|
721
|
+
* @property {{ ok: boolean, checked: number, unanswered: string[] }} pairing
|
|
722
|
+
* @property {boolean} refused true when the guard rejected the compactor's own output
|
|
723
|
+
* @property {{ pass: string, applied: number }[]} passes
|
|
724
|
+
* @property {number} clampChars the per-result clamp finally used (tightened if needed)
|
|
725
|
+
* @property {number} tightenRounds how many times the clamp had to halve
|
|
726
|
+
* @property {CompactAction[]} actions
|
|
727
|
+
* @property {string[]} lines a human-readable summary
|
|
728
|
+
*/
|
|
729
|
+
|
|
730
|
+
/**
|
|
731
|
+
* Compact an agentic transcript by attacking stale TOOL RESULTS. Pure and
|
|
732
|
+
* deterministic: no model, no network, no clock.
|
|
733
|
+
*
|
|
734
|
+
* ⭐ IT DOES THE LEAST DAMAGE THAT CLEARS THE BUDGET. The passes run in order of
|
|
735
|
+
* how obviously dead their target is, and the loop STOPS the moment the estimate
|
|
736
|
+
* is under `budgetTokens`. A transcript that already fits is returned byte-for-
|
|
737
|
+
* byte unchanged, which is the common case and must cost nothing.
|
|
738
|
+
*
|
|
739
|
+
* @param {any[]} messages an OpenAI-shaped message array
|
|
740
|
+
* @param {object} [options]
|
|
741
|
+
* @param {number} [options.budgetTokens] target size. Default 24,000.
|
|
742
|
+
* @param {number} [options.keepLastRounds] recent tool rounds kept verbatim. Default 2.
|
|
743
|
+
* @param {number} [options.maxResultChars] clamp threshold for one result. Default 4,000.
|
|
744
|
+
* @param {number} [options.charsPerToken] the estimator's divisor. Default 4.
|
|
745
|
+
* @returns {{ messages: any[], dropped: number, freedTokens: number, report: CompactReport }}
|
|
746
|
+
* `dropped` counts tool RESULTS whose body was stubbed, gutted or clamped —
|
|
747
|
+
* never messages, because messages are never removed.
|
|
748
|
+
*/
|
|
749
|
+
export function compactMessages(messages, options = {}) {
|
|
750
|
+
const budgetTokens = Number.isFinite(options.budgetTokens) ? Math.max(0, options.budgetTokens) : DEFAULT_BUDGET_TOKENS;
|
|
751
|
+
const keepLastRounds = Number.isFinite(options.keepLastRounds) ? Math.max(0, Math.floor(options.keepLastRounds)) : DEFAULT_KEEP_LAST_ROUNDS;
|
|
752
|
+
const maxResultChars = Number.isFinite(options.maxResultChars) && options.maxResultChars > 0 ? Math.floor(options.maxResultChars) : DEFAULT_MAX_RESULT_CHARS;
|
|
753
|
+
const charsPerToken = Number.isFinite(options.charsPerToken) && options.charsPerToken > 0 ? options.charsPerToken : CHARS_PER_TOKEN;
|
|
754
|
+
|
|
755
|
+
const baseReport = {
|
|
756
|
+
method: TOKEN_ESTIMATE_NOTE,
|
|
757
|
+
budgetTokens,
|
|
758
|
+
keepLastRounds,
|
|
759
|
+
beforeTokens: 0,
|
|
760
|
+
afterTokens: 0,
|
|
761
|
+
freedTokens: 0,
|
|
762
|
+
freedPercent: 0,
|
|
763
|
+
underBudget: true,
|
|
764
|
+
totalMessages: 0,
|
|
765
|
+
totalResults: 0,
|
|
766
|
+
compactableResults: 0,
|
|
767
|
+
protectedByReference: [],
|
|
768
|
+
inputPairing: { ok: true, checked: 0, unanswered: [] },
|
|
769
|
+
pairing: { ok: true, checked: 0, unanswered: [] },
|
|
770
|
+
refused: false,
|
|
771
|
+
clampChars: 0,
|
|
772
|
+
tightenRounds: 0,
|
|
773
|
+
passes: [],
|
|
774
|
+
actions: [],
|
|
775
|
+
lines: [],
|
|
776
|
+
};
|
|
777
|
+
|
|
778
|
+
if (!Array.isArray(messages) || messages.length === 0) {
|
|
779
|
+
return {
|
|
780
|
+
messages: [],
|
|
781
|
+
dropped: 0,
|
|
782
|
+
freedTokens: 0,
|
|
783
|
+
report: { ...baseReport, lines: ['nothing to compact — the transcript is empty.'] },
|
|
784
|
+
};
|
|
785
|
+
}
|
|
786
|
+
|
|
787
|
+
const beforeTokens = estimateMessagesTokens(messages, charsPerToken);
|
|
788
|
+
const inputPairing = verifyToolPairing(messages);
|
|
789
|
+
|
|
790
|
+
const { rounds } = groupRounds(messages);
|
|
791
|
+
const { calls, results } = indexTranscript(messages);
|
|
792
|
+
|
|
793
|
+
/**
|
|
794
|
+
* ⭐ THE KEEP WINDOW COUNTS TOOL ROUNDS, NOT MESSAGES OR ENTRIES. A trailing
|
|
795
|
+
* run of bare assistant notes and plan banners would otherwise eat the whole
|
|
796
|
+
* window and protect nothing that costs anything.
|
|
797
|
+
*/
|
|
798
|
+
const toolRoundStarts = rounds
|
|
799
|
+
.filter((group) => group.some((i) => messages[i]?.role === 'tool'))
|
|
800
|
+
.map((group) => group[0]);
|
|
801
|
+
const protectFromIndex = keepLastRounds <= 0
|
|
802
|
+
? messages.length
|
|
803
|
+
: (toolRoundStarts.length <= keepLastRounds ? (toolRoundStarts[0] ?? messages.length) : toolRoundStarts[toolRoundStarts.length - keepLastRounds]);
|
|
804
|
+
|
|
805
|
+
/**
|
|
806
|
+
* ⚠️ WHAT THE MODEL JUST NAMED IS OFF LIMITS. The newest assistant message is
|
|
807
|
+
* the model's live train of thought; if it says "compare against the earlier
|
|
808
|
+
* lib/slug.mjs" then that stale read is not stale, it is the working set.
|
|
809
|
+
* Matching is a substring test on subjects of 3+ characters — shorter ones
|
|
810
|
+
* would match by accident and protect everything.
|
|
811
|
+
*/
|
|
812
|
+
const newestAssistant = [...messages].reverse().find((m) => m?.role === 'assistant');
|
|
813
|
+
const newestText = typeof newestAssistant?.content === 'string' ? newestAssistant.content : '';
|
|
814
|
+
const protectedByReference = [];
|
|
815
|
+
const referencedSubjects = new Set();
|
|
816
|
+
if (newestText) {
|
|
817
|
+
for (const r of results) {
|
|
818
|
+
const label = subjectLabel(r.name, r.args);
|
|
819
|
+
if (typeof label === 'string' && label.length >= 3 && newestText.includes(label) && !referencedSubjects.has(label)) {
|
|
820
|
+
referencedSubjects.add(label);
|
|
821
|
+
protectedByReference.push(label);
|
|
822
|
+
}
|
|
823
|
+
}
|
|
824
|
+
}
|
|
825
|
+
|
|
826
|
+
const compactable = new Set();
|
|
827
|
+
for (const r of results) {
|
|
828
|
+
if (r.index >= protectFromIndex) continue;
|
|
829
|
+
if (typeof messages[r.index].content !== 'string') continue; // never rewrite structured content
|
|
830
|
+
if (messages[r.index].content.length === 0) continue;
|
|
831
|
+
if (referencedSubjects.has(subjectLabel(r.name, r.args))) continue;
|
|
832
|
+
compactable.add(r.index);
|
|
833
|
+
}
|
|
834
|
+
|
|
835
|
+
const report = {
|
|
836
|
+
...baseReport,
|
|
837
|
+
beforeTokens,
|
|
838
|
+
afterTokens: beforeTokens,
|
|
839
|
+
totalMessages: messages.length,
|
|
840
|
+
totalResults: results.length,
|
|
841
|
+
compactableResults: compactable.size,
|
|
842
|
+
protectedByReference,
|
|
843
|
+
inputPairing,
|
|
844
|
+
pairing: inputPairing,
|
|
845
|
+
underBudget: beforeTokens <= budgetTokens,
|
|
846
|
+
};
|
|
847
|
+
|
|
848
|
+
if (beforeTokens <= budgetTokens || compactable.size === 0) {
|
|
849
|
+
report.lines = [
|
|
850
|
+
`${n(beforeTokens)} estimated tokens vs a ${n(budgetTokens)} budget — `
|
|
851
|
+
+ (beforeTokens <= budgetTokens ? 'already fits, nothing compacted.' : 'nothing is eligible for compaction.'),
|
|
852
|
+
];
|
|
853
|
+
report.underBudget = beforeTokens <= budgetTokens;
|
|
854
|
+
return { messages: messages.slice(), dropped: 0, freedTokens: 0, report };
|
|
855
|
+
}
|
|
856
|
+
|
|
857
|
+
// ── apply, cheapest-truth-first, and stop as soon as it fits ─────────────
|
|
858
|
+
const working = messages.slice();
|
|
859
|
+
const perMessageChars = messages.map((m) => messageChars(m));
|
|
860
|
+
let totalChars = perMessageChars.reduce((a, b) => a + b, 0);
|
|
861
|
+
const budgetChars = budgetTokens * charsPerToken;
|
|
862
|
+
|
|
863
|
+
/** index -> the action that rewrote it. A Map, not an array, because the
|
|
864
|
+
* tightening loop below may REVISE an earlier clamp and must update the
|
|
865
|
+
* record rather than report the same message twice. */
|
|
866
|
+
const applied = new Map();
|
|
867
|
+
const passCounts = [];
|
|
868
|
+
|
|
869
|
+
/**
|
|
870
|
+
* Apply one candidate, keeping `totalChars` and the action record in step.
|
|
871
|
+
*
|
|
872
|
+
* ⚠️ `allowReclamp` MEANS "MAY REPLACE A `giant-results` CLAMP", and it now has
|
|
873
|
+
* two users: the tightening loop (a smaller clamp over a bigger one) and the
|
|
874
|
+
* receipts pass (a one-line pointer over a clamp). It deliberately does NOT
|
|
875
|
+
* permit replacing a superseded-read stub, a gutted command or a dead-search
|
|
876
|
+
* stub — each of those is already smaller AND more informative than what would
|
|
877
|
+
* replace it, so the swap would lose on both counts.
|
|
878
|
+
*/
|
|
879
|
+
const apply = (candidate, passName, { allowReclamp = false } = {}) => {
|
|
880
|
+
const prior = applied.get(candidate.index);
|
|
881
|
+
if (prior && !(allowReclamp && prior.pass === 'giant-results')) return false;
|
|
882
|
+
const current = working[candidate.index];
|
|
883
|
+
if (typeof current.content !== 'string') return false;
|
|
884
|
+
// ⚠️ A candidate that does not shrink the message is skipped, not applied.
|
|
885
|
+
if (candidate.content.length >= current.content.length) return false;
|
|
886
|
+
working[candidate.index] = { ...current, content: candidate.content };
|
|
887
|
+
const newChars = messageChars(working[candidate.index]);
|
|
888
|
+
totalChars += newChars - perMessageChars[candidate.index];
|
|
889
|
+
perMessageChars[candidate.index] = newChars;
|
|
890
|
+
applied.set(candidate.index, {
|
|
891
|
+
index: candidate.index,
|
|
892
|
+
pass: passName,
|
|
893
|
+
tool: candidate.tool,
|
|
894
|
+
subject: candidate.subject,
|
|
895
|
+
// ⚠️ ALWAYS the ORIGINAL length, even on a re-clamp — reporting the length
|
|
896
|
+
// of a previous clamp as "before" would understate what was removed.
|
|
897
|
+
beforeChars: prior?.beforeChars ?? current.content.length,
|
|
898
|
+
afterChars: candidate.content.length,
|
|
899
|
+
});
|
|
900
|
+
return true;
|
|
901
|
+
};
|
|
902
|
+
|
|
903
|
+
/**
|
|
904
|
+
* ── ⭐ THE ORDER IS BY HOW PROVABLY DEAD THE TARGET IS, NOT BY PASS NUMBER ──
|
|
905
|
+
*
|
|
906
|
+
* Because the loop stops the moment the transcript fits, the ORDER decides
|
|
907
|
+
* what gets destroyed when only partial compaction is needed. The first three
|
|
908
|
+
* passes target content that is dead as a matter of fact:
|
|
909
|
+
*
|
|
910
|
+
* · a superseded read — an identical later read exists, this copy is stale;
|
|
911
|
+
* · a stale command — the identical command was re-run, this result is old;
|
|
912
|
+
* · a dead search — the files it found were subsequently opened or
|
|
913
|
+
* edited, so the hit list has done its navigating.
|
|
914
|
+
*
|
|
915
|
+
* `giant-results` runs LAST because it is the only pass that cuts into
|
|
916
|
+
* CURRENT information: it clamps the middle out of the one and only copy of
|
|
917
|
+
* a live result. ⚠️ It is listed third in the brief and it is applied fourth
|
|
918
|
+
* here, deliberately. Applying it earlier means a transcript holding one dead
|
|
919
|
+
* search and one large live file read would gut the live file and leave the
|
|
920
|
+
* dead search sitting there — strictly the worse of the two available cuts.
|
|
921
|
+
* `report.passes` prints the applied order, so this is visible, not hidden.
|
|
922
|
+
*/
|
|
923
|
+
const passes = [
|
|
924
|
+
['superseded-reads', () => passSupersededReads(messages, results, compactable)],
|
|
925
|
+
['stale-commands', () => passStaleCommands(messages, results, compactable)],
|
|
926
|
+
['dead-searches', () => passDeadSearches(messages, results, calls, compactable)],
|
|
927
|
+
['giant-results', () => passGiantResults(messages, results, compactable, maxResultChars)],
|
|
928
|
+
];
|
|
929
|
+
|
|
930
|
+
outer:
|
|
931
|
+
for (const [passName, build] of passes) {
|
|
932
|
+
let count = 0;
|
|
933
|
+
for (const candidate of build()) {
|
|
934
|
+
if (totalChars <= budgetChars) { passCounts.push({ pass: passName, applied: count }); break outer; }
|
|
935
|
+
if (apply(candidate, passName)) count += 1;
|
|
936
|
+
}
|
|
937
|
+
passCounts.push({ pass: passName, applied: count });
|
|
938
|
+
}
|
|
939
|
+
|
|
940
|
+
/**
|
|
941
|
+
* ── ⭐⭐ THE CLAMP TIGHTENS. MEASURED, NOT GUESSED ─────────────────────────
|
|
942
|
+
*
|
|
943
|
+
* A FIXED clamp has a floor: `maxResultChars × compactable results`. On a real
|
|
944
|
+
* captured session — 23 messages, twelve unique large `read_file` results,
|
|
945
|
+
* nothing re-read and nothing searched — that floor made the compactor
|
|
946
|
+
* PLATEAU: 33,128 → 25,816 estimated tokens at a budget of 24,000, and the
|
|
947
|
+
* identical 25,816 at a budget of 8,000. Lowering the budget did nothing at
|
|
948
|
+
* all, and the report said "22.1% freed" while the session was still four
|
|
949
|
+
* times over its target.
|
|
950
|
+
*
|
|
951
|
+
* ⚠️ That plateau is this module failing at its actual job, and it gets worse
|
|
952
|
+
* with exactly the thing it exists to unlock. At a 60-round horizon there are
|
|
953
|
+
* a hundred results, so a 4,000-character floor is 400,000 characters — a
|
|
954
|
+
* larger transcript than the one we started with.
|
|
955
|
+
*
|
|
956
|
+
* ⭐ So when the four passes leave it over budget, the clamp HALVES and the
|
|
957
|
+
* clamp pass runs again, down to `MIN_CLAMP_CHARS`. Two properties make this
|
|
958
|
+
* safe rather than merely aggressive:
|
|
959
|
+
*
|
|
960
|
+
* · IT RE-CLAMPS FROM THE ORIGINAL, never on top of a previous clamp. A
|
|
961
|
+
* clamp of a clamp would nest "[… N characters removed …]" markers inside
|
|
962
|
+
* each other and the arithmetic in every one of them would be wrong.
|
|
963
|
+
* · IT NEVER OVERWRITES A STUB. A superseded-read stub is both smaller than
|
|
964
|
+
* any clamp and strictly more informative, so a clamp replacing one would
|
|
965
|
+
* be a loss in both directions.
|
|
966
|
+
*
|
|
967
|
+
* ⚠️ AND IT STOPS AT A FLOOR. Tightening to zero produces a tool result that
|
|
968
|
+
* is nothing but an apology, which is a hole in the history dressed as a
|
|
969
|
+
* message. If MIN_CLAMP_CHARS is not enough, the honest answer is
|
|
970
|
+
* `underBudget: false` — which the report gives.
|
|
971
|
+
*/
|
|
972
|
+
let clampChars = maxResultChars;
|
|
973
|
+
let tightenRounds = 0;
|
|
974
|
+
while (totalChars > budgetChars && clampChars > MIN_CLAMP_CHARS) {
|
|
975
|
+
clampChars = Math.max(MIN_CLAMP_CHARS, Math.floor(clampChars / 2));
|
|
976
|
+
tightenRounds += 1;
|
|
977
|
+
let count = 0;
|
|
978
|
+
for (const candidate of passGiantResults(messages, results, compactable, clampChars)) {
|
|
979
|
+
if (totalChars <= budgetChars) break;
|
|
980
|
+
if (apply(candidate, 'giant-results', { allowReclamp: true })) count += 1;
|
|
981
|
+
}
|
|
982
|
+
const row = passCounts.find((p) => p.pass === 'giant-results');
|
|
983
|
+
if (row) row.applied = [...applied.values()].filter((a) => a.pass === 'giant-results').length;
|
|
984
|
+
else passCounts.push({ pass: 'giant-results', applied: count });
|
|
985
|
+
}
|
|
986
|
+
|
|
987
|
+
/**
|
|
988
|
+
* ── ⭐⭐ AND WHEN EVEN THE FLOOR CLAMP CANNOT REACH IT: RECEIPTS ────────────
|
|
989
|
+
*
|
|
990
|
+
* ⚠️ THE ONLY TRANSCRIPTS THAT GET HERE ARE THE ONES THAT USED TO REPORT
|
|
991
|
+
* `underBudget: false`. Everything above has run, the clamp is at
|
|
992
|
+
* `MIN_CLAMP_CHARS`, and the payload is still over — which is the N × 714
|
|
993
|
+
* floor described on `passReceipts`. So this pass cannot change the output of
|
|
994
|
+
* any run that works today; it can only improve the ones that were already
|
|
995
|
+
* failing, which is the bar for adding anything to this file.
|
|
996
|
+
*
|
|
997
|
+
* ⚠️ `allowReclamp` HERE MEANS "may replace a giant-results clamp", the same
|
|
998
|
+
* meaning the tightening loop gives it. A stub from one of the three cheap
|
|
999
|
+
* passes is smaller than a receipt and more informative, and `apply` refuses
|
|
1000
|
+
* to grow a message anyway — two independent guards, because this is the one
|
|
1001
|
+
* pass that destroys information that nothing else has copied.
|
|
1002
|
+
*/
|
|
1003
|
+
/**
|
|
1004
|
+
* ── ⚠️⚠️ AND IT ONLY RUNS IF IT WOULD ACTUALLY WORK ────────────────────────
|
|
1005
|
+
*
|
|
1006
|
+
* THIS IS THE GUARD THE TEST SUITE DEMANDED, and it was right to. The first
|
|
1007
|
+
* version ran receipts on any over-budget transcript, and it went red on four
|
|
1008
|
+
* existing tests — every one of them a budget that is UNREACHABLE by
|
|
1009
|
+
* construction (10, 100, 0 tokens against an untouchable head). In those the
|
|
1010
|
+
* receipt destroyed the last copy of a body AND still reported
|
|
1011
|
+
* `underBudget: false`: the most destructive cut available, spent for nothing.
|
|
1012
|
+
*
|
|
1013
|
+
* ⭐ So the precondition is derived rather than invented: simulate receipting
|
|
1014
|
+
* every eligible result and take the pass only if that CLEARS the budget. It
|
|
1015
|
+
* separates exactly the two cases — a hundred results whose 714-char clamps
|
|
1016
|
+
* floor above the budget (receipts reach it, so take them, oldest first, and
|
|
1017
|
+
* stop the moment it fits) from a single giant result under an impossible
|
|
1018
|
+
* budget (receipts cannot reach it, so keep the head and tail and report the
|
|
1019
|
+
* budget as unmet, which is the honest failure this module already prefers).
|
|
1020
|
+
*
|
|
1021
|
+
* ⚠️ THE SIMULATION IS EXACT, NOT AN ESTIMATE. `messageChars` differs only by
|
|
1022
|
+
* `content.length` between two versions of the same message — every other term
|
|
1023
|
+
* (role, ids, tool_calls, overhead) is untouched — so the delta is a
|
|
1024
|
+
* subtraction and not a re-measurement that could disagree with `apply`.
|
|
1025
|
+
*/
|
|
1026
|
+
let receiptsRan = false;
|
|
1027
|
+
let receiptsApplied = 0;
|
|
1028
|
+
if (totalChars > budgetChars) {
|
|
1029
|
+
const eligible = [];
|
|
1030
|
+
let ifAllReceipted = totalChars;
|
|
1031
|
+
for (const candidate of passReceipts(messages, results, compactable)) {
|
|
1032
|
+
const current = working[candidate.index];
|
|
1033
|
+
if (typeof current.content !== 'string') continue;
|
|
1034
|
+
const prior = applied.get(candidate.index);
|
|
1035
|
+
if (prior && prior.pass !== 'giant-results') continue;
|
|
1036
|
+
if (candidate.content.length >= current.content.length) continue;
|
|
1037
|
+
eligible.push(candidate);
|
|
1038
|
+
ifAllReceipted += candidate.content.length - current.content.length;
|
|
1039
|
+
}
|
|
1040
|
+
if (eligible.length > 0 && ifAllReceipted <= budgetChars) {
|
|
1041
|
+
receiptsRan = true;
|
|
1042
|
+
for (const candidate of eligible) {
|
|
1043
|
+
if (totalChars <= budgetChars) break;
|
|
1044
|
+
if (apply(candidate, 'receipts', { allowReclamp: true })) receiptsApplied += 1;
|
|
1045
|
+
}
|
|
1046
|
+
passCounts.push({ pass: 'receipts', applied: receiptsApplied });
|
|
1047
|
+
/**
|
|
1048
|
+
* ⚠️ THE `giant-results` TALLY IS RECOUNTED, NOT LEFT ALONE. A clamp this
|
|
1049
|
+
* pass replaced is no longer a clamp, and a report whose rows sum to more
|
|
1050
|
+
* than `actions.length` is the kind of quietly-wrong number this package
|
|
1051
|
+
* refuses to ship. The tightening loop already recounts for the same reason.
|
|
1052
|
+
*/
|
|
1053
|
+
const row = passCounts.find((p) => p.pass === 'giant-results');
|
|
1054
|
+
if (row) row.applied = [...applied.values()].filter((a) => a.pass === 'giant-results').length;
|
|
1055
|
+
}
|
|
1056
|
+
}
|
|
1057
|
+
|
|
1058
|
+
// Transcript order, so the report reads like the conversation.
|
|
1059
|
+
const actions = [...applied.values()].sort((a, b) => a.index - b.index);
|
|
1060
|
+
|
|
1061
|
+
/**
|
|
1062
|
+
* ⚠️⚠️ THE GUARD. If compaction changed the SHAPE of the conversation in any
|
|
1063
|
+
* way — a message removed, reordered, an id altered — the output is thrown
|
|
1064
|
+
* away and the input is returned. It cannot happen through the code above,
|
|
1065
|
+
* which only ever replaces `content` at a fixed index; this exists because
|
|
1066
|
+
* "it cannot happen" is what was said about the last five things that did.
|
|
1067
|
+
*/
|
|
1068
|
+
if (structuralFingerprint(working) !== structuralFingerprint(messages)) {
|
|
1069
|
+
return {
|
|
1070
|
+
messages: messages.slice(),
|
|
1071
|
+
dropped: 0,
|
|
1072
|
+
freedTokens: 0,
|
|
1073
|
+
report: {
|
|
1074
|
+
...report,
|
|
1075
|
+
refused: true,
|
|
1076
|
+
pairing: verifyToolPairing(working),
|
|
1077
|
+
lines: [
|
|
1078
|
+
'COMPACTION REFUSED: the compacted transcript did not match the shape of the original, '
|
|
1079
|
+
+ 'which would make every following round a provider 400. The original history was returned unchanged.',
|
|
1080
|
+
],
|
|
1081
|
+
},
|
|
1082
|
+
};
|
|
1083
|
+
}
|
|
1084
|
+
|
|
1085
|
+
const afterTokens = estimateMessagesTokens(working, charsPerToken);
|
|
1086
|
+
const freedTokens = beforeTokens - afterTokens;
|
|
1087
|
+
const outputPairing = verifyToolPairing(working);
|
|
1088
|
+
|
|
1089
|
+
report.afterTokens = afterTokens;
|
|
1090
|
+
report.freedTokens = freedTokens;
|
|
1091
|
+
report.freedPercent = beforeTokens > 0 ? Math.round((freedTokens / beforeTokens) * 1000) / 10 : 0;
|
|
1092
|
+
report.underBudget = afterTokens <= budgetTokens;
|
|
1093
|
+
report.pairing = outputPairing;
|
|
1094
|
+
report.passes = passCounts;
|
|
1095
|
+
report.actions = actions;
|
|
1096
|
+
report.clampChars = clampChars;
|
|
1097
|
+
report.tightenRounds = tightenRounds;
|
|
1098
|
+
report.lines = [
|
|
1099
|
+
`${n(beforeTokens)} → ${n(afterTokens)} estimated tokens (${report.freedPercent}% freed, ~${n(freedTokens)} tokens) `
|
|
1100
|
+
+ `across ${actions.length} of ${results.length} tool results.`,
|
|
1101
|
+
...passCounts.filter((p) => p.applied > 0).map((p) => ` ${p.pass}: ${p.applied}`),
|
|
1102
|
+
...(tightenRounds > 0
|
|
1103
|
+
? [` the per-result clamp tightened ${tightenRounds}× — ${n(maxResultChars)} → ${n(clampChars)} chars — `
|
|
1104
|
+
+ 'because a fixed clamp could not reach the budget.']
|
|
1105
|
+
: []),
|
|
1106
|
+
/**
|
|
1107
|
+
* ⚠️ SAID OUT LOUD, BECAUSE IT IS THE MOST DESTRUCTIVE THING THIS MODULE
|
|
1108
|
+
* DOES. A clamp keeps verbatim text at both ends; a receipt keeps none, and
|
|
1109
|
+
* a reader who does not know that would mistake the transcript for one that
|
|
1110
|
+
* still holds the content.
|
|
1111
|
+
*/
|
|
1112
|
+
...(receiptsRan
|
|
1113
|
+
? [` the clamp was already at its ${n(MIN_CLAMP_CHARS)}-char floor, so `
|
|
1114
|
+
+ `${n(receiptsApplied)} stale result${receiptsApplied === 1 ? ' was' : 's were'} replaced by a one-line `
|
|
1115
|
+
+ 'receipt naming the tool that fetches it back. Nothing of those bodies is kept.']
|
|
1116
|
+
: []),
|
|
1117
|
+
report.underBudget
|
|
1118
|
+
? ` under the ${n(budgetTokens)} token budget.`
|
|
1119
|
+
: ` ⚠️ STILL OVER the ${n(budgetTokens)} token budget by ~${n(afterTokens - budgetTokens)} tokens — `
|
|
1120
|
+
+ 'everything left is either recent, referenced, or the only copy.',
|
|
1121
|
+
` ⚠️ ${TOKEN_ESTIMATE_NOTE}`,
|
|
1122
|
+
];
|
|
1123
|
+
|
|
1124
|
+
return { messages: working, dropped: actions.length, freedTokens, report };
|
|
1125
|
+
}
|
|
1126
|
+
|
|
1127
|
+
/**
|
|
1128
|
+
* ── ⭐ THE WIRING SEAM (this module is useless until someone calls it) ──────
|
|
1129
|
+
*
|
|
1130
|
+
* A MODULE NOBODY IMPORTS IS NOT A FEATURE. In `lib/turn.mjs`, inside the round
|
|
1131
|
+
* loop, immediately BEFORE `const reply = await callModelImpl({... messages ...})`:
|
|
1132
|
+
*
|
|
1133
|
+
* import { compactMessages } from './compact.mjs'; // top of file
|
|
1134
|
+
*
|
|
1135
|
+
* const fit = compactMessages(messages, {
|
|
1136
|
+
* budgetTokens: config.contextBudgetTokens ?? 24_000,
|
|
1137
|
+
* keepLastRounds: 2,
|
|
1138
|
+
* });
|
|
1139
|
+
* if (fit.dropped > 0) {
|
|
1140
|
+
* messages.length = 0;
|
|
1141
|
+
* messages.push(...fit.messages);
|
|
1142
|
+
* onEvent({ type: 'compact', round, report: fit.report });
|
|
1143
|
+
* }
|
|
1144
|
+
*
|
|
1145
|
+
* ⚠️ `messages` is reassigned IN PLACE because the surrounding loop closes over
|
|
1146
|
+
* the same binding — rebinding it with `messages = fit.messages` would leave
|
|
1147
|
+
* later pushes writing to the old array on some code paths.
|
|
1148
|
+
*
|
|
1149
|
+
* ⚠️ AND IT MUST RUN BEFORE THE CALL, NOT AFTER. Compacting after the reply has
|
|
1150
|
+
* already been paid for saves nothing on the round that burst.
|
|
1151
|
+
*/
|