acp-kernel 0.0.1 → 0.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/DESIGN.md +220 -0
- package/LICENSE +21 -0
- package/PROVENANCE.md +92 -0
- package/README.md +96 -2
- package/dist/boundaries.d.ts +25 -0
- package/dist/boundaries.d.ts.map +1 -0
- package/dist/compress.d.ts +37 -0
- package/dist/compress.d.ts.map +1 -0
- package/dist/compression-rules.d.ts +11 -0
- package/dist/compression-rules.d.ts.map +1 -0
- package/dist/config.d.ts +4 -0
- package/dist/config.d.ts.map +1 -0
- package/dist/decompress.d.ts +14 -0
- package/dist/decompress.d.ts.map +1 -0
- package/dist/filter/apply.d.ts +10 -0
- package/dist/filter/apply.d.ts.map +1 -0
- package/dist/filter/index.d.ts +5 -0
- package/dist/filter/index.d.ts.map +1 -0
- package/dist/filter/registry.d.ts +6 -0
- package/dist/filter/registry.d.ts.map +1 -0
- package/dist/filter/types.d.ts +28 -0
- package/dist/filter/types.d.ts.map +1 -0
- package/dist/hide-consumed.d.ts +7 -0
- package/dist/hide-consumed.d.ts.map +1 -0
- package/dist/index.d.ts +33 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +2303 -0
- package/dist/index.js.map +1 -0
- package/dist/keep-markers.d.ts +9 -0
- package/dist/keep-markers.d.ts.map +1 -0
- package/dist/merge.d.ts +9 -0
- package/dist/merge.d.ts.map +1 -0
- package/dist/nudge-text.d.ts +8 -0
- package/dist/nudge-text.d.ts.map +1 -0
- package/dist/pipeline.d.ts +26 -0
- package/dist/pipeline.d.ts.map +1 -0
- package/dist/protected.d.ts +4 -0
- package/dist/protected.d.ts.map +1 -0
- package/dist/prune.d.ts +7 -0
- package/dist/prune.d.ts.map +1 -0
- package/dist/rebuild.d.ts +25 -0
- package/dist/rebuild.d.ts.map +1 -0
- package/dist/recommend.d.ts +40 -0
- package/dist/recommend.d.ts.map +1 -0
- package/dist/refs.d.ts +22 -0
- package/dist/refs.d.ts.map +1 -0
- package/dist/render-refs.d.ts +5 -0
- package/dist/render-refs.d.ts.map +1 -0
- package/dist/report.d.ts +11 -0
- package/dist/report.d.ts.map +1 -0
- package/dist/state.d.ts +10 -0
- package/dist/state.d.ts.map +1 -0
- package/dist/sync.d.ts +7 -0
- package/dist/sync.d.ts.map +1 -0
- package/dist/tokenize.d.ts +6 -0
- package/dist/tokenize.d.ts.map +1 -0
- package/dist/truncate-tools.d.ts +14 -0
- package/dist/truncate-tools.d.ts.map +1 -0
- package/dist/types.d.ts +190 -0
- package/dist/types.d.ts.map +1 -0
- package/package.json +58 -5
package/dist/index.js
ADDED
|
@@ -0,0 +1,2303 @@
|
|
|
1
|
+
// src/refs.ts
|
|
2
|
+
var REF_WIDTH = 5;
|
|
3
|
+
var MIN_INDEX = 1;
|
|
4
|
+
var MAX_INDEX = 99999;
|
|
5
|
+
var REF_PATTERN = /^m0*(\d{1,5})$/;
|
|
6
|
+
var BLOCKED_REF = "BLOCKED";
|
|
7
|
+
function emptyRefMap() {
|
|
8
|
+
return { byRaw: {}, byRef: {} };
|
|
9
|
+
}
|
|
10
|
+
function indexToRef(index) {
|
|
11
|
+
if (!Number.isInteger(index) || index < MIN_INDEX || index > MAX_INDEX) {
|
|
12
|
+
throw new RangeError(
|
|
13
|
+
`ref index out of bounds: ${index} (allowed ${MIN_INDEX}-${MAX_INDEX})`
|
|
14
|
+
);
|
|
15
|
+
}
|
|
16
|
+
return `m${String(index).padStart(REF_WIDTH, "0")}`;
|
|
17
|
+
}
|
|
18
|
+
function refToIndex(ref) {
|
|
19
|
+
const match = REF_PATTERN.exec(ref.trim().toLowerCase());
|
|
20
|
+
if (!match) return null;
|
|
21
|
+
const index = Number(match[1]);
|
|
22
|
+
if (index < MIN_INDEX || index > MAX_INDEX) return null;
|
|
23
|
+
return index;
|
|
24
|
+
}
|
|
25
|
+
function refForRaw(map, rawId) {
|
|
26
|
+
return map.byRaw[rawId] ?? null;
|
|
27
|
+
}
|
|
28
|
+
function rawForRef(map, ref) {
|
|
29
|
+
return map.byRef[ref] ?? null;
|
|
30
|
+
}
|
|
31
|
+
function assignRefs(messages, options) {
|
|
32
|
+
const map = {
|
|
33
|
+
byRaw: { ...options.existing.byRaw },
|
|
34
|
+
byRef: { ...options.existing.byRef }
|
|
35
|
+
};
|
|
36
|
+
let cursor = Number.isInteger(options.nextIndex) && options.nextIndex >= MIN_INDEX ? options.nextIndex : MIN_INDEX;
|
|
37
|
+
let newlyAssigned = 0;
|
|
38
|
+
for (const message of messages) {
|
|
39
|
+
if (!message.id || options.shouldSkip?.(message)) continue;
|
|
40
|
+
if (map.byRaw[message.id]) continue;
|
|
41
|
+
if (options.isProtected?.(message)) {
|
|
42
|
+
map.byRaw[message.id] = BLOCKED_REF;
|
|
43
|
+
continue;
|
|
44
|
+
}
|
|
45
|
+
const ref = allocateFreeRef(map, cursor);
|
|
46
|
+
cursor = ref.index + 1;
|
|
47
|
+
map.byRaw[message.id] = ref.text;
|
|
48
|
+
map.byRef[ref.text] = message.id;
|
|
49
|
+
newlyAssigned++;
|
|
50
|
+
}
|
|
51
|
+
return { map, nextIndex: cursor, newlyAssigned };
|
|
52
|
+
}
|
|
53
|
+
function allocateFreeRef(map, start) {
|
|
54
|
+
let candidate = Math.max(start, MIN_INDEX);
|
|
55
|
+
while (candidate <= MAX_INDEX) {
|
|
56
|
+
const text = indexToRef(candidate);
|
|
57
|
+
if (!map.byRef[text]) {
|
|
58
|
+
return { text, index: candidate };
|
|
59
|
+
}
|
|
60
|
+
candidate++;
|
|
61
|
+
}
|
|
62
|
+
throw new Error(
|
|
63
|
+
`ref capacity exhausted: cannot allocate beyond ${indexToRef(MAX_INDEX)}`
|
|
64
|
+
);
|
|
65
|
+
}
|
|
66
|
+
function highestUsedIndex(map) {
|
|
67
|
+
let highest = 0;
|
|
68
|
+
for (const ref of Object.values(map.byRaw)) {
|
|
69
|
+
const index = ref === BLOCKED_REF ? null : refToIndex(ref);
|
|
70
|
+
if (index !== null && index > highest) highest = index;
|
|
71
|
+
}
|
|
72
|
+
return highest;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
// src/state.ts
|
|
76
|
+
function createInitialState() {
|
|
77
|
+
return {
|
|
78
|
+
blocks: [],
|
|
79
|
+
messageRefs: { byRaw: {}, byRef: {} },
|
|
80
|
+
nudge: {
|
|
81
|
+
lastPerMessageNudgeTokens: 0,
|
|
82
|
+
lastNudgeShownTokens: 0,
|
|
83
|
+
baselineTokens: 0,
|
|
84
|
+
anchors: {}
|
|
85
|
+
},
|
|
86
|
+
stats: { tokensCompressed: 0, compressionCount: 0 },
|
|
87
|
+
nextBlockId: 1,
|
|
88
|
+
nextRunId: 1
|
|
89
|
+
};
|
|
90
|
+
}
|
|
91
|
+
function allocateBlockId(state) {
|
|
92
|
+
const id = state.nextBlockId;
|
|
93
|
+
state.nextBlockId = Math.max(1, id) + 1;
|
|
94
|
+
return `b${id}`;
|
|
95
|
+
}
|
|
96
|
+
function allocateRunId(state) {
|
|
97
|
+
const id = state.nextRunId;
|
|
98
|
+
state.nextRunId = Math.max(1, id) + 1;
|
|
99
|
+
return `r${id}`;
|
|
100
|
+
}
|
|
101
|
+
function blockById(state, blockId) {
|
|
102
|
+
return state.blocks.find((block) => block.blockId === blockId);
|
|
103
|
+
}
|
|
104
|
+
function activeBlocks(state) {
|
|
105
|
+
return state.blocks.filter((block) => block.active);
|
|
106
|
+
}
|
|
107
|
+
function coveredMessageIds(state) {
|
|
108
|
+
const covered = /* @__PURE__ */ new Set();
|
|
109
|
+
for (const block of state.blocks) {
|
|
110
|
+
if (!block.active) continue;
|
|
111
|
+
for (const id of block.effectiveMessageIds) covered.add(id);
|
|
112
|
+
}
|
|
113
|
+
return covered;
|
|
114
|
+
}
|
|
115
|
+
function highestActiveTier(state) {
|
|
116
|
+
let highest = 0;
|
|
117
|
+
for (const block of state.blocks) {
|
|
118
|
+
if (block.active && block.tier > highest) highest = block.tier;
|
|
119
|
+
}
|
|
120
|
+
return highest;
|
|
121
|
+
}
|
|
122
|
+
function advanceSurvival(state, promotionThreshold) {
|
|
123
|
+
for (const block of state.blocks) {
|
|
124
|
+
if (!block.active) continue;
|
|
125
|
+
block.survivedCount += 1;
|
|
126
|
+
if (block.survivedCount >= promotionThreshold) {
|
|
127
|
+
block.generation = "old";
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
// src/prune.ts
|
|
133
|
+
var SUMMARY_HEADER = "[Compressed conversation section]";
|
|
134
|
+
function prune(messages, state, options = {}) {
|
|
135
|
+
const covered = coveredMessageIds(state);
|
|
136
|
+
if (covered.size === 0) return [...messages];
|
|
137
|
+
const inject = options.injectSummaries ?? true;
|
|
138
|
+
const firstUserIndex = messages.findIndex(
|
|
139
|
+
(message) => message.role === "user"
|
|
140
|
+
);
|
|
141
|
+
const indexById = /* @__PURE__ */ new Map();
|
|
142
|
+
messages.forEach((message, index) => indexById.set(message.id, index));
|
|
143
|
+
const anchors = inject ? collectSummaryAnchors(state, indexById) : [];
|
|
144
|
+
return stripOrphanedToolResults(
|
|
145
|
+
rebuildMessages(messages, covered, firstUserIndex, anchors)
|
|
146
|
+
);
|
|
147
|
+
}
|
|
148
|
+
function collectSummaryAnchors(state, indexById) {
|
|
149
|
+
const anchors = [];
|
|
150
|
+
for (const block of activeBlocks(state)) {
|
|
151
|
+
let earliest = null;
|
|
152
|
+
for (const id of block.effectiveMessageIds) {
|
|
153
|
+
const index = indexById.get(id);
|
|
154
|
+
if (index !== void 0 && (earliest === null || index < earliest)) {
|
|
155
|
+
earliest = index;
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
anchors.push({
|
|
159
|
+
blockId: block.blockId,
|
|
160
|
+
summary: block.summary,
|
|
161
|
+
topic: block.topic,
|
|
162
|
+
insertAt: earliest ?? 0
|
|
163
|
+
});
|
|
164
|
+
}
|
|
165
|
+
anchors.sort((left, right) => left.insertAt - right.insertAt);
|
|
166
|
+
return anchors;
|
|
167
|
+
}
|
|
168
|
+
function rebuildMessages(messages, covered, firstUserIndex, anchors) {
|
|
169
|
+
const result = [];
|
|
170
|
+
const pending = [...anchors];
|
|
171
|
+
for (let index = 0; index < messages.length; index++) {
|
|
172
|
+
while (pending.length > 0 && pending[0].insertAt === index) {
|
|
173
|
+
result.push(renderSummary(pending.shift()));
|
|
174
|
+
}
|
|
175
|
+
if (index === firstUserIndex && firstUserIndex >= 0) {
|
|
176
|
+
result.push(messages[index]);
|
|
177
|
+
continue;
|
|
178
|
+
}
|
|
179
|
+
if (covered.has(messages[index].id)) continue;
|
|
180
|
+
result.push(messages[index]);
|
|
181
|
+
}
|
|
182
|
+
while (pending.length > 0) {
|
|
183
|
+
result.push(renderSummary(pending.shift()));
|
|
184
|
+
}
|
|
185
|
+
return result;
|
|
186
|
+
}
|
|
187
|
+
function renderSummary(anchor) {
|
|
188
|
+
const body = anchor.summary.trim();
|
|
189
|
+
const topicLine = anchor.topic ? `${SUMMARY_HEADER} \u2014 ${anchor.topic}` : SUMMARY_HEADER;
|
|
190
|
+
const text = body.length === 0 ? topicLine : `${topicLine}
|
|
191
|
+
${body}`;
|
|
192
|
+
return {
|
|
193
|
+
id: `acp_summary_${anchor.blockId}`,
|
|
194
|
+
role: "system",
|
|
195
|
+
contentType: "text",
|
|
196
|
+
text
|
|
197
|
+
};
|
|
198
|
+
}
|
|
199
|
+
function stripOrphanedToolResults(messages) {
|
|
200
|
+
const knownCallIds = /* @__PURE__ */ new Set();
|
|
201
|
+
for (const m of messages) {
|
|
202
|
+
if (m.contentType === "tool-call" && m.toolCallId) {
|
|
203
|
+
knownCallIds.add(m.toolCallId);
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
return messages.filter(
|
|
207
|
+
(m) => m.contentType !== "tool-result" || !m.toolCallId || knownCallIds.has(m.toolCallId)
|
|
208
|
+
);
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
// src/sync.ts
|
|
212
|
+
function syncBlocks(messages, state) {
|
|
213
|
+
const presentIds = new Set(messages.map((message) => message.id));
|
|
214
|
+
const deactivated = [];
|
|
215
|
+
const result = {
|
|
216
|
+
blocks: state.blocks.map((block) => ({
|
|
217
|
+
...block,
|
|
218
|
+
directMessageIds: [...block.directMessageIds],
|
|
219
|
+
effectiveMessageIds: [...block.effectiveMessageIds],
|
|
220
|
+
directBlockIds: [...block.directBlockIds]
|
|
221
|
+
})),
|
|
222
|
+
messageRefs: {
|
|
223
|
+
byRaw: { ...state.messageRefs.byRaw },
|
|
224
|
+
byRef: { ...state.messageRefs.byRef }
|
|
225
|
+
},
|
|
226
|
+
nudge: { ...state.nudge, anchors: { ...state.nudge.anchors } },
|
|
227
|
+
stats: { ...state.stats },
|
|
228
|
+
nextBlockId: state.nextBlockId,
|
|
229
|
+
nextRunId: state.nextRunId
|
|
230
|
+
};
|
|
231
|
+
const consumedBlockIds = /* @__PURE__ */ new Set();
|
|
232
|
+
for (const block of result.blocks) {
|
|
233
|
+
for (const consumedId of block.directBlockIds) {
|
|
234
|
+
consumedBlockIds.add(consumedId);
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
for (const block of result.blocks) {
|
|
238
|
+
if (consumedBlockIds.has(block.blockId)) {
|
|
239
|
+
block.active = false;
|
|
240
|
+
continue;
|
|
241
|
+
}
|
|
242
|
+
block.active = true;
|
|
243
|
+
const stillPresent = block.effectiveMessageIds.some(
|
|
244
|
+
(id) => presentIds.has(id)
|
|
245
|
+
);
|
|
246
|
+
if (!stillPresent) {
|
|
247
|
+
block.active = false;
|
|
248
|
+
deactivated.push(block.blockId);
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
return { state: result, deactivated };
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
// src/tokenize.ts
|
|
255
|
+
import { createRequire } from "module";
|
|
256
|
+
var require2 = createRequire(import.meta.url);
|
|
257
|
+
function defaultCountTokens(text) {
|
|
258
|
+
if (!text) return 0;
|
|
259
|
+
const ascii = text.match(/[a-zA-Z][a-zA-Z0-9_'-]*/g);
|
|
260
|
+
const cjk = text.match(/[\u4e00-\u9fff\u3040-\u30ff\uac00-\ud7af]/g);
|
|
261
|
+
return (ascii?.length ?? 0) + (cjk?.length ?? 0);
|
|
262
|
+
}
|
|
263
|
+
function estimateTokensFast(text) {
|
|
264
|
+
if (!text) return 0;
|
|
265
|
+
return Math.ceil(text.length / 4);
|
|
266
|
+
}
|
|
267
|
+
function createBpeTokenizer() {
|
|
268
|
+
try {
|
|
269
|
+
const mod = require2("@anthropic-ai/tokenizer");
|
|
270
|
+
const bpeCount = mod.countTokens ?? mod.default?.countTokens;
|
|
271
|
+
if (typeof bpeCount !== "function") return defaultCountTokens;
|
|
272
|
+
return (text) => {
|
|
273
|
+
try {
|
|
274
|
+
return bpeCount(text);
|
|
275
|
+
} catch {
|
|
276
|
+
return defaultCountTokens(text);
|
|
277
|
+
}
|
|
278
|
+
};
|
|
279
|
+
} catch {
|
|
280
|
+
return defaultCountTokens;
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
// src/config.ts
|
|
285
|
+
function defaultConfig(modelContextLimit, overrides = {}) {
|
|
286
|
+
const base = {
|
|
287
|
+
tiers: { enabled: true, tier2Trigger: 5, tier3Trigger: 10 },
|
|
288
|
+
nudge: {
|
|
289
|
+
maxContextLimitPct: 0.55,
|
|
290
|
+
minContextLimitPct: 0.45,
|
|
291
|
+
frequency: 5,
|
|
292
|
+
iterationThreshold: 15,
|
|
293
|
+
force: "soft",
|
|
294
|
+
growthRatio: 0.05,
|
|
295
|
+
growthFloor: Math.max(2e4, Math.round(modelContextLimit * 0.05)),
|
|
296
|
+
growthCap: 5e4,
|
|
297
|
+
minGrowthFloor: 2e4,
|
|
298
|
+
minGrowthRatio: 0.45,
|
|
299
|
+
emergencyThresholdPct: 0.8
|
|
300
|
+
},
|
|
301
|
+
promotionThreshold: 5,
|
|
302
|
+
truncate: { threshold: 1 },
|
|
303
|
+
merge: { maxSummaryLength: 3e3, minOldGenBlocks: 3 },
|
|
304
|
+
compress: {
|
|
305
|
+
minCompressRange: 5e3,
|
|
306
|
+
maxSummaryLength: 2e4,
|
|
307
|
+
minSummaryLength: 50
|
|
308
|
+
},
|
|
309
|
+
protectedTools: [],
|
|
310
|
+
preserveRecentMessages: 5,
|
|
311
|
+
preserveRecentTokens: 5e3,
|
|
312
|
+
modelContextLimit
|
|
313
|
+
};
|
|
314
|
+
return {
|
|
315
|
+
...base,
|
|
316
|
+
...overrides,
|
|
317
|
+
tiers: { ...base.tiers, ...overrides.tiers },
|
|
318
|
+
nudge: { ...base.nudge, ...overrides.nudge },
|
|
319
|
+
truncate: { ...base.truncate, ...overrides.truncate },
|
|
320
|
+
merge: { ...base.merge, ...overrides.merge },
|
|
321
|
+
compress: { ...base.compress, ...overrides.compress }
|
|
322
|
+
};
|
|
323
|
+
}
|
|
324
|
+
function validateConfig(config) {
|
|
325
|
+
const errors = [];
|
|
326
|
+
if (!Number.isFinite(config.modelContextLimit) || config.modelContextLimit <= 0) {
|
|
327
|
+
errors.push("modelContextLimit must be a positive number");
|
|
328
|
+
}
|
|
329
|
+
if (config.nudge.minContextLimitPct > config.nudge.maxContextLimitPct) {
|
|
330
|
+
errors.push(
|
|
331
|
+
"nudge.minContextLimitPct must not exceed nudge.maxContextLimitPct"
|
|
332
|
+
);
|
|
333
|
+
}
|
|
334
|
+
if (config.promotionThreshold < 1) {
|
|
335
|
+
errors.push("promotionThreshold must be >= 1");
|
|
336
|
+
}
|
|
337
|
+
if (config.truncate.threshold <= 0 || config.truncate.threshold > 1) {
|
|
338
|
+
errors.push("truncate.threshold must be in (0, 1]");
|
|
339
|
+
}
|
|
340
|
+
if (config.merge.maxSummaryLength < 1) {
|
|
341
|
+
errors.push("merge.maxSummaryLength must be >= 1");
|
|
342
|
+
}
|
|
343
|
+
if (config.merge.minOldGenBlocks < 1) {
|
|
344
|
+
errors.push("merge.minOldGenBlocks must be >= 1");
|
|
345
|
+
}
|
|
346
|
+
for (const tier of [config.tiers.tier2Trigger, config.tiers.tier3Trigger]) {
|
|
347
|
+
if (tier < 1) errors.push("tier triggers must be >= 1");
|
|
348
|
+
}
|
|
349
|
+
if (config.tiers.tier3Trigger <= config.tiers.tier2Trigger) {
|
|
350
|
+
errors.push("tiers.tier3Trigger must be greater than tiers.tier2Trigger");
|
|
351
|
+
}
|
|
352
|
+
return errors;
|
|
353
|
+
}
|
|
354
|
+
|
|
355
|
+
// src/boundaries.ts
|
|
356
|
+
var MESSAGE_REF_PATTERN = /^m0*(\d{1,5})$/;
|
|
357
|
+
var BLOCK_REF_PATTERN = /^b(\d{1,9})$/;
|
|
358
|
+
function parseBoundary(ref) {
|
|
359
|
+
const normalized = ref.trim().toLowerCase();
|
|
360
|
+
const messageMatch = MESSAGE_REF_PATTERN.exec(normalized);
|
|
361
|
+
if (messageMatch) {
|
|
362
|
+
const numericId = Number(messageMatch[1]);
|
|
363
|
+
if (numericId >= 1 && numericId <= 99999) {
|
|
364
|
+
return { kind: "message", numericId, raw: normalized };
|
|
365
|
+
}
|
|
366
|
+
}
|
|
367
|
+
const blockMatch = BLOCK_REF_PATTERN.exec(normalized);
|
|
368
|
+
if (blockMatch) {
|
|
369
|
+
const numericId = Number(blockMatch[1]);
|
|
370
|
+
if (numericId >= 1) return { kind: "block", numericId, raw: normalized };
|
|
371
|
+
}
|
|
372
|
+
return null;
|
|
373
|
+
}
|
|
374
|
+
function resolveBoundaries(input) {
|
|
375
|
+
const start = parseBoundary(input.startRef);
|
|
376
|
+
const end = parseBoundary(input.endRef);
|
|
377
|
+
if (!start || !end) {
|
|
378
|
+
throw new Error(
|
|
379
|
+
`Invalid boundary ref(s): startId="${input.startRef}", endId="${input.endRef}". Use mNNNNN or bN.`
|
|
380
|
+
);
|
|
381
|
+
}
|
|
382
|
+
const indexByRawId = /* @__PURE__ */ new Map();
|
|
383
|
+
input.messages.forEach(
|
|
384
|
+
(message, index) => indexByRawId.set(message.id, index)
|
|
385
|
+
);
|
|
386
|
+
let startIndex = resolveAnchorIndex(start, input.state, indexByRawId);
|
|
387
|
+
let endIndex = resolveAnchorIndex(end, input.state, indexByRawId);
|
|
388
|
+
if (startIndex === null || endIndex === null) {
|
|
389
|
+
throw new Error(
|
|
390
|
+
`Boundary not found in visible context (likely consumed by an existing block). startId="${input.startRef}", endId="${input.endRef}".`
|
|
391
|
+
);
|
|
392
|
+
}
|
|
393
|
+
if (startIndex > endIndex) {
|
|
394
|
+
[startIndex, endIndex] = [endIndex, startIndex];
|
|
395
|
+
}
|
|
396
|
+
const messageIds = [];
|
|
397
|
+
for (let index = startIndex; index <= endIndex; index++) {
|
|
398
|
+
const message = input.messages[index];
|
|
399
|
+
if (message) messageIds.push(message.id);
|
|
400
|
+
}
|
|
401
|
+
const boundaryKind = start.kind === "block" || end.kind === "block" ? "block" : "message";
|
|
402
|
+
const nestedBlockIds = [];
|
|
403
|
+
const nestedSeen = /* @__PURE__ */ new Set();
|
|
404
|
+
for (const block of activeBlocks(input.state)) {
|
|
405
|
+
const anchor = earliestIndexOfIds(block.effectiveMessageIds, indexByRawId);
|
|
406
|
+
if (anchor !== null && anchor >= startIndex && anchor <= endIndex) {
|
|
407
|
+
if (!nestedSeen.has(block.blockId)) {
|
|
408
|
+
nestedSeen.add(block.blockId);
|
|
409
|
+
nestedBlockIds.push(block.blockId);
|
|
410
|
+
}
|
|
411
|
+
}
|
|
412
|
+
}
|
|
413
|
+
const protectedGaps = [];
|
|
414
|
+
return {
|
|
415
|
+
startIndex,
|
|
416
|
+
endIndex,
|
|
417
|
+
messageIds,
|
|
418
|
+
nestedBlockIds,
|
|
419
|
+
boundaryKind,
|
|
420
|
+
protectedGaps
|
|
421
|
+
};
|
|
422
|
+
}
|
|
423
|
+
function resolveAnchorIndex(boundary, state, indexByRawId) {
|
|
424
|
+
if (boundary.kind === "message") {
|
|
425
|
+
const rawId = state.messageRefs.byRef[boundary.raw] ?? state.messageRefs.byRef[formatPaddedRef(boundary.numericId)];
|
|
426
|
+
if (!rawId) return null;
|
|
427
|
+
const index = indexByRawId.get(rawId);
|
|
428
|
+
return index === void 0 ? null : index;
|
|
429
|
+
}
|
|
430
|
+
const block = blockById(state, `b${boundary.numericId}`);
|
|
431
|
+
if (!block || !block.active) return null;
|
|
432
|
+
return earliestIndexOfIds(block.effectiveMessageIds, indexByRawId);
|
|
433
|
+
}
|
|
434
|
+
function formatPaddedRef(index) {
|
|
435
|
+
return `m${String(index).padStart(5, "0")}`;
|
|
436
|
+
}
|
|
437
|
+
function earliestIndexOfIds(ids, indexByRawId) {
|
|
438
|
+
let earliest = null;
|
|
439
|
+
for (const id of ids) {
|
|
440
|
+
const index = indexByRawId.get(id);
|
|
441
|
+
if (index !== void 0 && (earliest === null || index < earliest)) {
|
|
442
|
+
earliest = index;
|
|
443
|
+
}
|
|
444
|
+
}
|
|
445
|
+
return earliest;
|
|
446
|
+
}
|
|
447
|
+
|
|
448
|
+
// src/truncate-tools.ts
|
|
449
|
+
var TRUNCATION_MARKER = "[truncated for context space]";
|
|
450
|
+
var DEFAULTS = {
|
|
451
|
+
minOutputTokens: 1e3,
|
|
452
|
+
keepPrefixChars: 2e3,
|
|
453
|
+
keepSuffixChars: 2e3,
|
|
454
|
+
protectRecentMessages: 3
|
|
455
|
+
};
|
|
456
|
+
function truncateLargeToolOutputs(messages, tokenCount, config, countTokens, options = {}) {
|
|
457
|
+
const opts = { ...DEFAULTS, ...options };
|
|
458
|
+
if (config.modelContextLimit <= 0) return { messages, truncatedCount: 0, savedTokens: 0 };
|
|
459
|
+
const threshold = config.truncate.threshold * config.modelContextLimit;
|
|
460
|
+
if (tokenCount < threshold) return { messages, truncatedCount: 0, savedTokens: 0 };
|
|
461
|
+
const protectedIndex = messages.length - opts.protectRecentMessages;
|
|
462
|
+
const candidates = [];
|
|
463
|
+
for (let index = 0; index < messages.length; index++) {
|
|
464
|
+
if (index >= protectedIndex) break;
|
|
465
|
+
const message = messages[index];
|
|
466
|
+
if (message.contentType !== "tool-result") continue;
|
|
467
|
+
const text = message.text ?? "";
|
|
468
|
+
if (text.length === 0 || text.includes(TRUNCATION_MARKER)) continue;
|
|
469
|
+
const tokens = countTokens(text);
|
|
470
|
+
if (tokens < opts.minOutputTokens) continue;
|
|
471
|
+
candidates.push({ index, tokens });
|
|
472
|
+
}
|
|
473
|
+
if (candidates.length === 0) return { messages, truncatedCount: 0, savedTokens: 0 };
|
|
474
|
+
candidates.sort((left, right) => right.tokens - left.tokens);
|
|
475
|
+
const targetTokens = threshold * 0.9;
|
|
476
|
+
let savedTokens = 0;
|
|
477
|
+
const edits = /* @__PURE__ */ new Map();
|
|
478
|
+
let truncatedCount = 0;
|
|
479
|
+
for (const candidate of candidates) {
|
|
480
|
+
if (tokenCount - savedTokens <= targetTokens) break;
|
|
481
|
+
const original = messages[candidate.index].text ?? "";
|
|
482
|
+
if (original.length <= opts.keepPrefixChars + opts.keepSuffixChars) continue;
|
|
483
|
+
const prefix = original.slice(0, opts.keepPrefixChars);
|
|
484
|
+
const suffix = original.slice(-opts.keepSuffixChars);
|
|
485
|
+
const replacement = prefix + `
|
|
486
|
+
|
|
487
|
+
...${TRUNCATION_MARKER} \u2014 original ~${candidate.tokens} tokens]...
|
|
488
|
+
|
|
489
|
+
` + suffix;
|
|
490
|
+
edits.set(candidate.index, replacement);
|
|
491
|
+
savedTokens += candidate.tokens - countTokens(replacement);
|
|
492
|
+
truncatedCount++;
|
|
493
|
+
}
|
|
494
|
+
if (truncatedCount === 0) return { messages, truncatedCount: 0, savedTokens: 0 };
|
|
495
|
+
const updated = messages.map(
|
|
496
|
+
(message, index) => edits.has(index) ? { ...message, text: edits.get(index) } : message
|
|
497
|
+
);
|
|
498
|
+
return { messages: updated, truncatedCount, savedTokens };
|
|
499
|
+
}
|
|
500
|
+
|
|
501
|
+
// src/hide-consumed.ts
|
|
502
|
+
var KEEP_LAST_ORPHANED = 2;
|
|
503
|
+
function hideConsumedCompressCalls(state, messages) {
|
|
504
|
+
const activeCallIds = /* @__PURE__ */ new Set();
|
|
505
|
+
const allBlockCallIds = /* @__PURE__ */ new Set();
|
|
506
|
+
for (const block of state.blocks) {
|
|
507
|
+
if (block.compressCallId) {
|
|
508
|
+
allBlockCallIds.add(block.compressCallId);
|
|
509
|
+
if (block.active) activeCallIds.add(block.compressCallId);
|
|
510
|
+
}
|
|
511
|
+
}
|
|
512
|
+
const lastOrphanedCallIds = [];
|
|
513
|
+
for (let i = messages.length - 1; i >= 0 && lastOrphanedCallIds.length < KEEP_LAST_ORPHANED; i--) {
|
|
514
|
+
const message = messages[i];
|
|
515
|
+
if (message.toolName !== "compress" || message.contentType !== "tool-call") continue;
|
|
516
|
+
const callId = message.toolCallId;
|
|
517
|
+
if (callId && !allBlockCallIds.has(callId)) {
|
|
518
|
+
lastOrphanedCallIds.push(callId);
|
|
519
|
+
}
|
|
520
|
+
}
|
|
521
|
+
const keepCallIds = /* @__PURE__ */ new Set([...activeCallIds, ...lastOrphanedCallIds]);
|
|
522
|
+
const hiddenCallIds = /* @__PURE__ */ new Set();
|
|
523
|
+
for (const message of messages) {
|
|
524
|
+
if (message.toolName === "compress" && message.contentType === "tool-call" && (!message.toolCallId || !keepCallIds.has(message.toolCallId))) {
|
|
525
|
+
if (message.toolCallId) hiddenCallIds.add(message.toolCallId);
|
|
526
|
+
}
|
|
527
|
+
}
|
|
528
|
+
let hidden = 0;
|
|
529
|
+
const result = [];
|
|
530
|
+
for (const message of messages) {
|
|
531
|
+
if (message.toolName === "compress" && message.contentType === "tool-call" && (!message.toolCallId || !keepCallIds.has(message.toolCallId))) {
|
|
532
|
+
hidden++;
|
|
533
|
+
continue;
|
|
534
|
+
}
|
|
535
|
+
if (message.contentType === "tool-result" && message.toolCallId && hiddenCallIds.has(message.toolCallId)) {
|
|
536
|
+
hidden++;
|
|
537
|
+
continue;
|
|
538
|
+
}
|
|
539
|
+
result.push(message);
|
|
540
|
+
}
|
|
541
|
+
return { messages: result, hidden };
|
|
542
|
+
}
|
|
543
|
+
|
|
544
|
+
// src/merge.ts
|
|
545
|
+
function numericPart(blockId) {
|
|
546
|
+
const match = /^b(\d+)$/.exec(blockId);
|
|
547
|
+
return match && match[1] !== void 0 ? Number(match[1]) : 0;
|
|
548
|
+
}
|
|
549
|
+
function truncateMergedSummary(merged, maxLength) {
|
|
550
|
+
if (merged.length <= maxLength) return merged;
|
|
551
|
+
const blocks = merged.split("\n---\n");
|
|
552
|
+
const headers = blocks.map((b) => b.split("\n")[0] ?? "").filter((h) => h.trim().length > 0);
|
|
553
|
+
const marker = "\n...\n[merged and truncated by batch cleanup]";
|
|
554
|
+
const budget = Math.max(0, maxLength - marker.length);
|
|
555
|
+
const headerJoin = headers.join("\n");
|
|
556
|
+
return headerJoin.length <= budget ? headerJoin + marker : headerJoin.slice(0, budget) + marker;
|
|
557
|
+
}
|
|
558
|
+
function mergeMarkedBlocks(state, markedIds, maxMergedLength, countTokens) {
|
|
559
|
+
const sortedIds = [...new Set(markedIds)].sort((a, b) => numericPart(a) - numericPart(b));
|
|
560
|
+
const sourceBlocks = sortedIds.map((id) => state.blocks.find((b) => b.blockId === id && b.active)).filter((b) => b !== void 0);
|
|
561
|
+
if (sourceBlocks.length < 2) {
|
|
562
|
+
return { state, mergedCount: 0, savedTokens: 0 };
|
|
563
|
+
}
|
|
564
|
+
const next = {
|
|
565
|
+
...state,
|
|
566
|
+
blocks: state.blocks.map((b) => ({ ...b }))
|
|
567
|
+
};
|
|
568
|
+
const newBlockId = allocateBlockId(next);
|
|
569
|
+
const newRunId = allocateRunId(next);
|
|
570
|
+
const bodies = sourceBlocks.map((block) => block.summary.trim());
|
|
571
|
+
const mergedSummary = truncateMergedSummary(bodies.join("\n---\n"), maxMergedLength);
|
|
572
|
+
const newSummaryTokens = countTokens(mergedSummary);
|
|
573
|
+
const effectiveMessageIds = /* @__PURE__ */ new Set();
|
|
574
|
+
const directMessageIds = /* @__PURE__ */ new Set();
|
|
575
|
+
for (const block of sourceBlocks) {
|
|
576
|
+
for (const id of block.effectiveMessageIds) effectiveMessageIds.add(id);
|
|
577
|
+
for (const id of block.directMessageIds) directMessageIds.add(id);
|
|
578
|
+
}
|
|
579
|
+
const sourceIds = sourceBlocks.map((b) => b.blockId);
|
|
580
|
+
const mergedBlock = {
|
|
581
|
+
blockId: newBlockId,
|
|
582
|
+
runId: newRunId,
|
|
583
|
+
tier: sourceBlocks.reduce((min, b) => b.tier < min ? b.tier : min, 3),
|
|
584
|
+
topic: "Batch merge cleanup",
|
|
585
|
+
summary: mergedSummary,
|
|
586
|
+
directMessageIds: [...directMessageIds],
|
|
587
|
+
effectiveMessageIds: [...effectiveMessageIds],
|
|
588
|
+
directBlockIds: [...sourceIds],
|
|
589
|
+
compressedTokens: sourceBlocks.reduce((sum, b) => sum + b.compressedTokens, 0),
|
|
590
|
+
createdAt: Date.now(),
|
|
591
|
+
survivedCount: 0,
|
|
592
|
+
generation: "old",
|
|
593
|
+
active: true
|
|
594
|
+
};
|
|
595
|
+
for (const block of next.blocks) {
|
|
596
|
+
if (sourceIds.includes(block.blockId)) block.active = false;
|
|
597
|
+
}
|
|
598
|
+
next.blocks.push(mergedBlock);
|
|
599
|
+
const sourceTokens = sourceBlocks.reduce(
|
|
600
|
+
(sum, block) => sum + countTokens(block.summary),
|
|
601
|
+
0
|
|
602
|
+
);
|
|
603
|
+
const savedTokens = Math.max(0, sourceTokens - newSummaryTokens);
|
|
604
|
+
return { state: next, mergedCount: sourceBlocks.length, savedTokens };
|
|
605
|
+
}
|
|
606
|
+
function collectOldGenBlocks(state, maxOldGenSummaryLength) {
|
|
607
|
+
return state.blocks.filter(
|
|
608
|
+
(b) => b.active && (b.generation === "old" || b.summary.length > maxOldGenSummaryLength)
|
|
609
|
+
).sort((a, b) => numericPart(a.blockId) - numericPart(b.blockId));
|
|
610
|
+
}
|
|
611
|
+
|
|
612
|
+
// src/filter/registry.ts
|
|
613
|
+
var registry = /* @__PURE__ */ new Map();
|
|
614
|
+
function registerMessageFilter(filter) {
|
|
615
|
+
const existing = registry.get(filter.name);
|
|
616
|
+
if (existing && existing.version !== filter.version) {
|
|
617
|
+
throw new Error(
|
|
618
|
+
`Message filter "${filter.name}" already registered with version ${existing.version}, cannot register version ${filter.version}.`
|
|
619
|
+
);
|
|
620
|
+
}
|
|
621
|
+
registry.set(filter.name, filter);
|
|
622
|
+
}
|
|
623
|
+
function getMessageFilter(name) {
|
|
624
|
+
return registry.get(name);
|
|
625
|
+
}
|
|
626
|
+
function listMessageFilters() {
|
|
627
|
+
return [...registry.values()];
|
|
628
|
+
}
|
|
629
|
+
function clearMessageFilters() {
|
|
630
|
+
registry.clear();
|
|
631
|
+
}
|
|
632
|
+
|
|
633
|
+
// src/filter/apply.ts
|
|
634
|
+
function applyMessageFilters(messages, config) {
|
|
635
|
+
if (!config?.enabled) {
|
|
636
|
+
return { messages, partsFiltered: 0, partsDropped: 0, partsModified: 0 };
|
|
637
|
+
}
|
|
638
|
+
const active = listMessageFilters().filter(
|
|
639
|
+
(filter) => config.filters?.[filter.name]?.enabled !== false
|
|
640
|
+
);
|
|
641
|
+
if (active.length === 0) {
|
|
642
|
+
return { messages, partsFiltered: 0, partsDropped: 0, partsModified: 0 };
|
|
643
|
+
}
|
|
644
|
+
let working = messages.map((message) => ({ ...message }));
|
|
645
|
+
const tally = { partsFiltered: 0, partsDropped: 0, partsModified: 0 };
|
|
646
|
+
const total = working.length;
|
|
647
|
+
const immediate = active.filter((filter) => !filter.keepLastOnly);
|
|
648
|
+
for (let index = 0; index < working.length; index++) {
|
|
649
|
+
const message = working[index];
|
|
650
|
+
const text = message.text ?? "";
|
|
651
|
+
if (text.length === 0) continue;
|
|
652
|
+
let current = text;
|
|
653
|
+
const baseCtx = {
|
|
654
|
+
text: current,
|
|
655
|
+
role: message.role,
|
|
656
|
+
messageIndex: index,
|
|
657
|
+
totalMessages: total,
|
|
658
|
+
toolName: message.toolName
|
|
659
|
+
};
|
|
660
|
+
for (const filter of immediate) {
|
|
661
|
+
let decision;
|
|
662
|
+
try {
|
|
663
|
+
decision = filter.filter(baseCtx);
|
|
664
|
+
} catch {
|
|
665
|
+
continue;
|
|
666
|
+
}
|
|
667
|
+
if (decision.action === "keep") continue;
|
|
668
|
+
tally.partsFiltered++;
|
|
669
|
+
if (decision.action === "drop") {
|
|
670
|
+
current = "";
|
|
671
|
+
tally.partsDropped++;
|
|
672
|
+
} else if (decision.action === "modify" && decision.text !== void 0) {
|
|
673
|
+
current = decision.text;
|
|
674
|
+
tally.partsModified++;
|
|
675
|
+
}
|
|
676
|
+
baseCtx.text = current;
|
|
677
|
+
}
|
|
678
|
+
if (current !== text) working[index] = { ...message, text: current };
|
|
679
|
+
}
|
|
680
|
+
const keepLast = active.filter((filter) => filter.keepLastOnly);
|
|
681
|
+
for (const filter of keepLast) {
|
|
682
|
+
let foundLast = false;
|
|
683
|
+
for (let index = working.length - 1; index >= 0; index--) {
|
|
684
|
+
const message = working[index];
|
|
685
|
+
const text = message.text ?? "";
|
|
686
|
+
if (text.length === 0) continue;
|
|
687
|
+
const ctx = {
|
|
688
|
+
text,
|
|
689
|
+
role: message.role,
|
|
690
|
+
messageIndex: index,
|
|
691
|
+
totalMessages: total,
|
|
692
|
+
toolName: message.toolName
|
|
693
|
+
};
|
|
694
|
+
let decision;
|
|
695
|
+
try {
|
|
696
|
+
decision = filter.filter(ctx);
|
|
697
|
+
} catch {
|
|
698
|
+
continue;
|
|
699
|
+
}
|
|
700
|
+
if (decision.action !== "drop" && decision.action !== "modify") continue;
|
|
701
|
+
if (foundLast) {
|
|
702
|
+
tally.partsFiltered++;
|
|
703
|
+
tally.partsDropped++;
|
|
704
|
+
working[index] = { ...message, text: "" };
|
|
705
|
+
} else {
|
|
706
|
+
foundLast = true;
|
|
707
|
+
if (decision.action === "modify" && decision.text !== void 0) {
|
|
708
|
+
tally.partsFiltered++;
|
|
709
|
+
tally.partsModified++;
|
|
710
|
+
working[index] = { ...message, text: decision.text };
|
|
711
|
+
}
|
|
712
|
+
}
|
|
713
|
+
}
|
|
714
|
+
}
|
|
715
|
+
return { messages: working, ...tally };
|
|
716
|
+
}
|
|
717
|
+
|
|
718
|
+
// src/render-refs.ts
|
|
719
|
+
function refTag(ref) {
|
|
720
|
+
return `[${ref}] `;
|
|
721
|
+
}
|
|
722
|
+
function renderMessage(message, map) {
|
|
723
|
+
const ref = refForRaw(map, message.id);
|
|
724
|
+
if (!ref || ref === BLOCKED_REF) return message;
|
|
725
|
+
const tag = refTag(ref);
|
|
726
|
+
if (!message.text) return { ...message, text: tag };
|
|
727
|
+
let text = message.text;
|
|
728
|
+
while (text.startsWith(tag)) text = text.slice(tag.length);
|
|
729
|
+
return { ...message, text: tag + text };
|
|
730
|
+
}
|
|
731
|
+
function renderVisibleRefs(messages, state) {
|
|
732
|
+
const map = state.messageRefs;
|
|
733
|
+
return messages.map((message) => renderMessage(message, map));
|
|
734
|
+
}
|
|
735
|
+
var renderRefsNode = {
|
|
736
|
+
name: "render-refs",
|
|
737
|
+
run(io, _ctx) {
|
|
738
|
+
return {
|
|
739
|
+
...io,
|
|
740
|
+
messages: renderVisibleRefs(io.messages, io.state)
|
|
741
|
+
};
|
|
742
|
+
}
|
|
743
|
+
};
|
|
744
|
+
|
|
745
|
+
// src/protected.ts
|
|
746
|
+
function matchToolPattern(toolName, pattern) {
|
|
747
|
+
if (pattern.endsWith("*")) {
|
|
748
|
+
return toolName.startsWith(pattern.slice(0, -1));
|
|
749
|
+
}
|
|
750
|
+
return toolName === pattern;
|
|
751
|
+
}
|
|
752
|
+
function isMessageProtected(msg, config) {
|
|
753
|
+
if (msg.contentType !== "tool-call" || !msg.toolName) return false;
|
|
754
|
+
for (const pattern of config.protectedTools) {
|
|
755
|
+
if (matchToolPattern(msg.toolName, pattern)) return true;
|
|
756
|
+
}
|
|
757
|
+
if (config.isToolProtected?.(msg.toolName, msg.text)) return true;
|
|
758
|
+
return false;
|
|
759
|
+
}
|
|
760
|
+
|
|
761
|
+
// src/recommend.ts
|
|
762
|
+
function refNum(ref) {
|
|
763
|
+
const n = parseInt(ref.slice(1), 10);
|
|
764
|
+
return Number.isNaN(n) ? -1 : n;
|
|
765
|
+
}
|
|
766
|
+
function estimateMessageTokens(message) {
|
|
767
|
+
return Math.ceil((message.text ?? "").length / 4);
|
|
768
|
+
}
|
|
769
|
+
function isToolMessage(message) {
|
|
770
|
+
return message.contentType === "tool-call" || message.contentType === "tool-result";
|
|
771
|
+
}
|
|
772
|
+
function isSyntheticOrPruned(message, state) {
|
|
773
|
+
if (message.text?.startsWith("[Compressed conversation section]")) return true;
|
|
774
|
+
for (const block of state.blocks) {
|
|
775
|
+
if (block.active && block.effectiveMessageIds.includes(message.id)) return true;
|
|
776
|
+
}
|
|
777
|
+
return false;
|
|
778
|
+
}
|
|
779
|
+
function computeProtectedRefs(messages, state, config) {
|
|
780
|
+
const preserveN = config.preserveRecentMessages;
|
|
781
|
+
const preserveTokens = config.preserveRecentTokens;
|
|
782
|
+
const result = /* @__PURE__ */ new Set();
|
|
783
|
+
const visible = [];
|
|
784
|
+
for (const msg of messages) {
|
|
785
|
+
if (isSyntheticOrPruned(msg, state)) continue;
|
|
786
|
+
const ref = state.messageRefs.byRaw[msg.id];
|
|
787
|
+
if (!ref || ref === "BLOCKED") continue;
|
|
788
|
+
visible.push({ ref, tokens: estimateMessageTokens(msg) });
|
|
789
|
+
}
|
|
790
|
+
if (preserveN > 0) {
|
|
791
|
+
for (const m of visible.slice(-preserveN)) {
|
|
792
|
+
result.add(m.ref);
|
|
793
|
+
}
|
|
794
|
+
}
|
|
795
|
+
if (preserveTokens > 0) {
|
|
796
|
+
let tokenAccum = 0;
|
|
797
|
+
for (let i = visible.length - 1; i >= 0 && tokenAccum < preserveTokens; i--) {
|
|
798
|
+
result.add(visible[i].ref);
|
|
799
|
+
tokenAccum += visible[i].tokens;
|
|
800
|
+
}
|
|
801
|
+
}
|
|
802
|
+
for (let i = messages.length - 1; i >= 0; i--) {
|
|
803
|
+
const msg = messages[i];
|
|
804
|
+
if (msg.role !== "user" || isSyntheticOrPruned(msg, state)) continue;
|
|
805
|
+
const ref = state.messageRefs.byRaw[msg.id];
|
|
806
|
+
if (ref && ref !== "BLOCKED") result.add(ref);
|
|
807
|
+
break;
|
|
808
|
+
}
|
|
809
|
+
return result;
|
|
810
|
+
}
|
|
811
|
+
function buildCompressibleRanges(messages, state, config, protectedZoneRefs) {
|
|
812
|
+
const compressibleMsgs = [];
|
|
813
|
+
const protectedMsgs = [];
|
|
814
|
+
for (const msg of messages) {
|
|
815
|
+
if (isSyntheticOrPruned(msg, state)) continue;
|
|
816
|
+
const ref = state.messageRefs.byRaw[msg.id];
|
|
817
|
+
if (!ref || ref === "BLOCKED") continue;
|
|
818
|
+
const rn = refNum(ref);
|
|
819
|
+
if (isMessageProtected(msg, config)) {
|
|
820
|
+
protectedMsgs.push({
|
|
821
|
+
ref,
|
|
822
|
+
refNum: rn,
|
|
823
|
+
tokens: estimateMessageTokens(msg),
|
|
824
|
+
tools: msg.toolName ? [msg.toolName] : []
|
|
825
|
+
});
|
|
826
|
+
continue;
|
|
827
|
+
}
|
|
828
|
+
if (protectedZoneRefs?.has(ref)) {
|
|
829
|
+
continue;
|
|
830
|
+
}
|
|
831
|
+
compressibleMsgs.push({
|
|
832
|
+
ref,
|
|
833
|
+
refNum: rn,
|
|
834
|
+
tokens: estimateMessageTokens(msg),
|
|
835
|
+
isTool: isToolMessage(msg)
|
|
836
|
+
});
|
|
837
|
+
}
|
|
838
|
+
const compressible = [];
|
|
839
|
+
let cur = null;
|
|
840
|
+
let prevRefNum = -2;
|
|
841
|
+
for (const info of compressibleMsgs) {
|
|
842
|
+
const hasGap = info.refNum > prevRefNum + 1;
|
|
843
|
+
if (cur && hasGap) {
|
|
844
|
+
compressible.push(cur);
|
|
845
|
+
cur = null;
|
|
846
|
+
}
|
|
847
|
+
prevRefNum = info.refNum;
|
|
848
|
+
if (!cur) {
|
|
849
|
+
cur = {
|
|
850
|
+
startRef: info.ref,
|
|
851
|
+
endRef: info.ref,
|
|
852
|
+
count: 1,
|
|
853
|
+
tokens: info.tokens,
|
|
854
|
+
toolPct: info.isTool ? 100 : 0,
|
|
855
|
+
textPct: info.isTool ? 0 : 100
|
|
856
|
+
};
|
|
857
|
+
} else {
|
|
858
|
+
cur.endRef = info.ref;
|
|
859
|
+
cur.count++;
|
|
860
|
+
cur.tokens += info.tokens;
|
|
861
|
+
if (info.isTool) {
|
|
862
|
+
cur.toolPct = Math.round((cur.toolPct * (cur.count - 1) + 100) / cur.count);
|
|
863
|
+
} else {
|
|
864
|
+
cur.toolPct = Math.round(cur.toolPct * (cur.count - 1) / cur.count);
|
|
865
|
+
}
|
|
866
|
+
cur.textPct = 100 - cur.toolPct;
|
|
867
|
+
}
|
|
868
|
+
}
|
|
869
|
+
if (cur) compressible.push(cur);
|
|
870
|
+
const protectedRanges = [];
|
|
871
|
+
let pcur = null;
|
|
872
|
+
let pPrevRefNum = -2;
|
|
873
|
+
for (const info of protectedMsgs) {
|
|
874
|
+
const hasGap = info.refNum > pPrevRefNum + 1;
|
|
875
|
+
if (pcur && hasGap) {
|
|
876
|
+
protectedRanges.push(pcur);
|
|
877
|
+
pcur = null;
|
|
878
|
+
}
|
|
879
|
+
pPrevRefNum = info.refNum;
|
|
880
|
+
if (!pcur) {
|
|
881
|
+
pcur = {
|
|
882
|
+
startRef: info.ref,
|
|
883
|
+
endRef: info.ref,
|
|
884
|
+
count: 1,
|
|
885
|
+
tokens: info.tokens,
|
|
886
|
+
tools: [...info.tools]
|
|
887
|
+
};
|
|
888
|
+
} else {
|
|
889
|
+
pcur.endRef = info.ref;
|
|
890
|
+
pcur.count++;
|
|
891
|
+
pcur.tokens += info.tokens;
|
|
892
|
+
for (const t of info.tools) {
|
|
893
|
+
if (!pcur.tools.includes(t)) pcur.tools.push(t);
|
|
894
|
+
}
|
|
895
|
+
}
|
|
896
|
+
}
|
|
897
|
+
if (pcur) protectedRanges.push(pcur);
|
|
898
|
+
return {
|
|
899
|
+
compressible: compressible.filter((g) => g.tokens > 0),
|
|
900
|
+
protected: protectedRanges
|
|
901
|
+
};
|
|
902
|
+
}
|
|
903
|
+
|
|
904
|
+
// src/pipeline.ts
|
|
905
|
+
function makeIO(messages, state, effects = {}) {
|
|
906
|
+
return { messages, state, effects };
|
|
907
|
+
}
|
|
908
|
+
function runPipeline(nodes, initial, ctx) {
|
|
909
|
+
let io = initial;
|
|
910
|
+
for (const node of nodes) {
|
|
911
|
+
if (node.enabled && !node.enabled(io, ctx)) continue;
|
|
912
|
+
io = node.run(io, ctx);
|
|
913
|
+
}
|
|
914
|
+
return io;
|
|
915
|
+
}
|
|
916
|
+
|
|
917
|
+
// src/compress.ts
|
|
918
|
+
function createCore(ports = {}) {
|
|
919
|
+
const countTokens = ports.countTokens ?? defaultCountTokens;
|
|
920
|
+
function applyCompression(input) {
|
|
921
|
+
const state = cloneState(input.state);
|
|
922
|
+
const runId = allocateRunId(state);
|
|
923
|
+
let blocksCreated = 0;
|
|
924
|
+
let tokensCompressed = 0;
|
|
925
|
+
const errors = [];
|
|
926
|
+
const preExistingCoverage = collectCoverage(state);
|
|
927
|
+
const rangeIndexSets = [];
|
|
928
|
+
for (const spec of input.ranges) {
|
|
929
|
+
let resolved;
|
|
930
|
+
try {
|
|
931
|
+
resolved = resolveBoundaries({
|
|
932
|
+
startRef: spec.startRef,
|
|
933
|
+
endRef: spec.endRef,
|
|
934
|
+
messages: input.messages,
|
|
935
|
+
state
|
|
936
|
+
});
|
|
937
|
+
} catch {
|
|
938
|
+
continue;
|
|
939
|
+
}
|
|
940
|
+
const indices = resolved.messageIds.map(
|
|
941
|
+
(id) => input.messages.findIndex((m) => m.id === id)
|
|
942
|
+
).filter((i) => i >= 0);
|
|
943
|
+
rangeIndexSets.push({ spec, indices });
|
|
944
|
+
}
|
|
945
|
+
const sortedRanges = [...rangeIndexSets].sort((a, b) => {
|
|
946
|
+
const aMin = a.indices.length > 0 ? Math.min(...a.indices) : Infinity;
|
|
947
|
+
const bMin = b.indices.length > 0 ? Math.min(...b.indices) : Infinity;
|
|
948
|
+
return aMin - bMin;
|
|
949
|
+
});
|
|
950
|
+
for (let i = 1; i < sortedRanges.length; i++) {
|
|
951
|
+
const prev = sortedRanges[i - 1];
|
|
952
|
+
const curr = sortedRanges[i];
|
|
953
|
+
const prevMax = prev.indices.length > 0 ? Math.max(...prev.indices) : -1;
|
|
954
|
+
const currMin = curr.indices.length > 0 ? Math.min(...curr.indices) : -1;
|
|
955
|
+
if (prevMax >= currMin && prevMax >= 0) {
|
|
956
|
+
return {
|
|
957
|
+
state: input.state,
|
|
958
|
+
result: {
|
|
959
|
+
blocksCreated: 0,
|
|
960
|
+
tokensCompressed: 0,
|
|
961
|
+
errors: [
|
|
962
|
+
`content: range (${prev.spec.startRef}..${prev.spec.endRef}) overlaps (${curr.spec.startRef}..${curr.spec.endRef}). Overlapping ranges cannot be compressed in the same batch.`
|
|
963
|
+
]
|
|
964
|
+
}
|
|
965
|
+
};
|
|
966
|
+
}
|
|
967
|
+
}
|
|
968
|
+
if (input.config.compress.minCompressRange > 0 && input.ranges.length > 0) {
|
|
969
|
+
let totalRangeChars = 0;
|
|
970
|
+
let hasBlockBoundaryRange = false;
|
|
971
|
+
for (const spec of input.ranges) {
|
|
972
|
+
let resolved;
|
|
973
|
+
try {
|
|
974
|
+
resolved = resolveBoundaries({
|
|
975
|
+
startRef: spec.startRef,
|
|
976
|
+
endRef: spec.endRef,
|
|
977
|
+
messages: input.messages,
|
|
978
|
+
state
|
|
979
|
+
});
|
|
980
|
+
} catch {
|
|
981
|
+
continue;
|
|
982
|
+
}
|
|
983
|
+
if (resolved.boundaryKind === "block") {
|
|
984
|
+
hasBlockBoundaryRange = true;
|
|
985
|
+
continue;
|
|
986
|
+
}
|
|
987
|
+
for (const id of resolved.messageIds) {
|
|
988
|
+
const msg = input.messages.find((m) => m.id === id);
|
|
989
|
+
totalRangeChars += msg?.text?.length ?? 0;
|
|
990
|
+
}
|
|
991
|
+
}
|
|
992
|
+
if (!hasBlockBoundaryRange && totalRangeChars < input.config.compress.minCompressRange) {
|
|
993
|
+
return {
|
|
994
|
+
state: input.state,
|
|
995
|
+
result: {
|
|
996
|
+
blocksCreated: 0,
|
|
997
|
+
tokensCompressed: 0,
|
|
998
|
+
errors: [
|
|
999
|
+
`Total compressible content too small (${totalRangeChars} chars across ${input.ranges.length} range(s), min ${input.config.compress.minCompressRange}). Combine more messages into your range(s) to meet the threshold.`
|
|
1000
|
+
]
|
|
1001
|
+
}
|
|
1002
|
+
};
|
|
1003
|
+
}
|
|
1004
|
+
}
|
|
1005
|
+
for (const spec of input.ranges) {
|
|
1006
|
+
try {
|
|
1007
|
+
const compressed = applySingleRange({
|
|
1008
|
+
spec,
|
|
1009
|
+
messages: input.messages,
|
|
1010
|
+
state,
|
|
1011
|
+
runId,
|
|
1012
|
+
config: input.config,
|
|
1013
|
+
protectedMessageIds: input.protectedMessageIds,
|
|
1014
|
+
countTokens,
|
|
1015
|
+
preExistingCoverage
|
|
1016
|
+
});
|
|
1017
|
+
blocksCreated++;
|
|
1018
|
+
tokensCompressed += compressed;
|
|
1019
|
+
} catch (error) {
|
|
1020
|
+
errors.push(error instanceof Error ? error.message : String(error));
|
|
1021
|
+
}
|
|
1022
|
+
}
|
|
1023
|
+
state.stats.compressionCount += blocksCreated;
|
|
1024
|
+
state.stats.tokensCompressed += tokensCompressed;
|
|
1025
|
+
if (blocksCreated > 0) {
|
|
1026
|
+
state.nudge.lastPerMessageNudgeTokens = 0;
|
|
1027
|
+
state.nudge.lastNudgeShownTokens = 0;
|
|
1028
|
+
}
|
|
1029
|
+
return { state, result: { blocksCreated, tokensCompressed, errors } };
|
|
1030
|
+
}
|
|
1031
|
+
function processTurn(input) {
|
|
1032
|
+
const configErrors = validateConfig(input.config);
|
|
1033
|
+
if (configErrors.length > 0) {
|
|
1034
|
+
console.warn(`[acp-kernel] Config validation warnings: ${configErrors.join("; ")}. Thresholds may not fire correctly.`);
|
|
1035
|
+
}
|
|
1036
|
+
const ctx = {
|
|
1037
|
+
config: input.config,
|
|
1038
|
+
tokenCount: input.tokenCount,
|
|
1039
|
+
countTokens
|
|
1040
|
+
};
|
|
1041
|
+
const initial = {
|
|
1042
|
+
messages: input.messages,
|
|
1043
|
+
state: input.state,
|
|
1044
|
+
effects: {}
|
|
1045
|
+
};
|
|
1046
|
+
const result = runPipeline(defaultNodes(), initial, ctx);
|
|
1047
|
+
return {
|
|
1048
|
+
messages: result.messages,
|
|
1049
|
+
state: result.state,
|
|
1050
|
+
nudge: result.effects.nudge
|
|
1051
|
+
};
|
|
1052
|
+
}
|
|
1053
|
+
function decompress(blockId, state) {
|
|
1054
|
+
return blockById(state, blockId);
|
|
1055
|
+
}
|
|
1056
|
+
function search(query, state) {
|
|
1057
|
+
const terms = query.toLowerCase().split(/\s+/).filter((term) => term.length > 0);
|
|
1058
|
+
if (terms.length === 0) return [];
|
|
1059
|
+
const scored = activeBlocks(state).map((block) => ({ block, score: scoreRelevance(block, terms) })).filter((entry) => entry.score > 0.1).sort((left, right) => right.score - left.score);
|
|
1060
|
+
return scored.map((entry) => entry.block);
|
|
1061
|
+
}
|
|
1062
|
+
function status(state, tokenCount, config) {
|
|
1063
|
+
const active = activeBlocks(state);
|
|
1064
|
+
const usage = config.modelContextLimit > 0 ? tokenCount / config.modelContextLimit : 0;
|
|
1065
|
+
return {
|
|
1066
|
+
contextUsage: usage,
|
|
1067
|
+
tokenCount,
|
|
1068
|
+
modelContextLimit: config.modelContextLimit,
|
|
1069
|
+
activeBlocks: active.length,
|
|
1070
|
+
totalBlocks: state.blocks.length,
|
|
1071
|
+
tokensCompressed: state.stats.tokensCompressed,
|
|
1072
|
+
breakdown: { active: active.length, total: state.blocks.length }
|
|
1073
|
+
};
|
|
1074
|
+
}
|
|
1075
|
+
function defaultNodes() {
|
|
1076
|
+
return [
|
|
1077
|
+
assignRefsNode,
|
|
1078
|
+
syncBlocksNode,
|
|
1079
|
+
mergeBlocksNode,
|
|
1080
|
+
pruneNode,
|
|
1081
|
+
filterNode,
|
|
1082
|
+
hideCompressCallsNode,
|
|
1083
|
+
recommendNode,
|
|
1084
|
+
nudgeNode,
|
|
1085
|
+
emergencyTruncateNode,
|
|
1086
|
+
renderRefsNode
|
|
1087
|
+
];
|
|
1088
|
+
}
|
|
1089
|
+
return { processTurn, applyCompression, defaultNodes, decompress, search, status };
|
|
1090
|
+
}
|
|
1091
|
+
var assignRefsNode = {
|
|
1092
|
+
name: "assign-refs",
|
|
1093
|
+
run(io, ctx) {
|
|
1094
|
+
const hasProtection = ctx.config.protectedTools.length > 0 || !!ctx.config.isToolProtected;
|
|
1095
|
+
const protectedFn = hasProtection ? (m) => isMessageProtected(m, ctx.config) : void 0;
|
|
1096
|
+
const refResult = assignRefs(io.messages, {
|
|
1097
|
+
existing: io.state.messageRefs,
|
|
1098
|
+
nextIndex: highestUsedIndex(io.state.messageRefs) + 1,
|
|
1099
|
+
isProtected: protectedFn
|
|
1100
|
+
});
|
|
1101
|
+
return { ...io, state: { ...io.state, messageRefs: refResult.map } };
|
|
1102
|
+
}
|
|
1103
|
+
};
|
|
1104
|
+
var syncBlocksNode = {
|
|
1105
|
+
name: "sync-blocks",
|
|
1106
|
+
run(io, ctx) {
|
|
1107
|
+
const synced = syncBlocks(io.messages, io.state);
|
|
1108
|
+
advanceSurvival(synced.state, ctx.config.promotionThreshold);
|
|
1109
|
+
return { ...io, state: synced.state };
|
|
1110
|
+
}
|
|
1111
|
+
};
|
|
1112
|
+
var mergeBlocksNode = {
|
|
1113
|
+
name: "merge-blocks",
|
|
1114
|
+
run(io, ctx) {
|
|
1115
|
+
const oldGen = collectOldGenBlocks(
|
|
1116
|
+
io.state,
|
|
1117
|
+
ctx.config.merge.maxSummaryLength
|
|
1118
|
+
);
|
|
1119
|
+
if (oldGen.length < ctx.config.merge.minOldGenBlocks) return io;
|
|
1120
|
+
const ids = oldGen.map((b) => b.blockId);
|
|
1121
|
+
const merged = mergeMarkedBlocks(
|
|
1122
|
+
io.state,
|
|
1123
|
+
ids,
|
|
1124
|
+
ctx.config.merge.maxSummaryLength,
|
|
1125
|
+
ctx.countTokens
|
|
1126
|
+
);
|
|
1127
|
+
if (merged.mergedCount === 0) return io;
|
|
1128
|
+
return {
|
|
1129
|
+
...io,
|
|
1130
|
+
state: merged.state,
|
|
1131
|
+
effects: { ...io.effects, mergedCount: merged.mergedCount }
|
|
1132
|
+
};
|
|
1133
|
+
}
|
|
1134
|
+
};
|
|
1135
|
+
var pruneNode = {
|
|
1136
|
+
name: "prune",
|
|
1137
|
+
run(io) {
|
|
1138
|
+
return { ...io, messages: prune(io.messages, io.state) };
|
|
1139
|
+
}
|
|
1140
|
+
};
|
|
1141
|
+
var filterNode = {
|
|
1142
|
+
name: "filter",
|
|
1143
|
+
enabled: (_io, ctx) => !!ctx.config.messageFilters?.enabled && listMessageFilters().length > 0,
|
|
1144
|
+
run(io, ctx) {
|
|
1145
|
+
const applied = applyMessageFilters(io.messages, ctx.config.messageFilters);
|
|
1146
|
+
return { ...io, messages: applied.messages };
|
|
1147
|
+
}
|
|
1148
|
+
};
|
|
1149
|
+
var hideCompressCallsNode = {
|
|
1150
|
+
name: "hide-compress-calls",
|
|
1151
|
+
run(io) {
|
|
1152
|
+
const hidden = hideConsumedCompressCalls(io.state, io.messages);
|
|
1153
|
+
return { ...io, messages: hidden.messages };
|
|
1154
|
+
}
|
|
1155
|
+
};
|
|
1156
|
+
var recommendNode = {
|
|
1157
|
+
name: "recommend",
|
|
1158
|
+
run(io, ctx) {
|
|
1159
|
+
const protectedRefs = computeProtectedRefs(
|
|
1160
|
+
io.messages,
|
|
1161
|
+
io.state,
|
|
1162
|
+
ctx.config
|
|
1163
|
+
);
|
|
1164
|
+
const contextRanges = buildCompressibleRanges(
|
|
1165
|
+
io.messages,
|
|
1166
|
+
io.state,
|
|
1167
|
+
ctx.config,
|
|
1168
|
+
protectedRefs
|
|
1169
|
+
);
|
|
1170
|
+
const nothingToCompress = contextRanges.compressible.length === 0;
|
|
1171
|
+
const recommendation = {
|
|
1172
|
+
contextRanges,
|
|
1173
|
+
recommendedRanges: contextRanges.compressible,
|
|
1174
|
+
nothingToCompress
|
|
1175
|
+
};
|
|
1176
|
+
return { ...io, effects: { ...io.effects, recommendation } };
|
|
1177
|
+
}
|
|
1178
|
+
};
|
|
1179
|
+
var nudgeNode = {
|
|
1180
|
+
name: "nudge-inject",
|
|
1181
|
+
run(io, ctx) {
|
|
1182
|
+
const nudge = decideNudge({
|
|
1183
|
+
tokenCount: ctx.tokenCount,
|
|
1184
|
+
config: ctx.config,
|
|
1185
|
+
state: io.state,
|
|
1186
|
+
messages: io.messages,
|
|
1187
|
+
recommendation: io.effects.recommendation
|
|
1188
|
+
});
|
|
1189
|
+
const baseline = io.state.nudge.lastPerMessageNudgeTokens;
|
|
1190
|
+
const nudgeGrowthTokens = resolveAdaptiveGrowth(
|
|
1191
|
+
ctx.config.modelContextLimit,
|
|
1192
|
+
ctx.config.nudge
|
|
1193
|
+
);
|
|
1194
|
+
let stamped = { ...io.state.nudge };
|
|
1195
|
+
if (baseline > 0 && ctx.tokenCount < baseline - nudgeGrowthTokens) {
|
|
1196
|
+
stamped.lastPerMessageNudgeTokens = ctx.tokenCount;
|
|
1197
|
+
stamped.lastNudgeShownTokens = 0;
|
|
1198
|
+
}
|
|
1199
|
+
if (stamped.lastPerMessageNudgeTokens === 0) {
|
|
1200
|
+
stamped.lastPerMessageNudgeTokens = ctx.tokenCount;
|
|
1201
|
+
}
|
|
1202
|
+
if (nudge.shouldInject) {
|
|
1203
|
+
stamped.lastNudgeShownTokens = ctx.tokenCount;
|
|
1204
|
+
}
|
|
1205
|
+
return {
|
|
1206
|
+
...io,
|
|
1207
|
+
state: { ...io.state, nudge: stamped },
|
|
1208
|
+
effects: { ...io.effects, nudge }
|
|
1209
|
+
};
|
|
1210
|
+
}
|
|
1211
|
+
};
|
|
1212
|
+
var emergencyTruncateNode = {
|
|
1213
|
+
name: "emergency-truncate",
|
|
1214
|
+
run(io, ctx) {
|
|
1215
|
+
const usage = ctx.config.modelContextLimit > 0 ? ctx.tokenCount / ctx.config.modelContextLimit : 0;
|
|
1216
|
+
if (usage < ctx.config.truncate.threshold) return io;
|
|
1217
|
+
const trunc = truncateLargeToolOutputs(
|
|
1218
|
+
io.messages,
|
|
1219
|
+
ctx.tokenCount,
|
|
1220
|
+
ctx.config,
|
|
1221
|
+
ctx.countTokens,
|
|
1222
|
+
{ protectRecentMessages: ctx.config.preserveRecentMessages }
|
|
1223
|
+
);
|
|
1224
|
+
return {
|
|
1225
|
+
...io,
|
|
1226
|
+
messages: trunc.messages,
|
|
1227
|
+
effects: { ...io.effects, truncatedCount: trunc.truncatedCount }
|
|
1228
|
+
};
|
|
1229
|
+
}
|
|
1230
|
+
};
|
|
1231
|
+
function applySingleRange(input) {
|
|
1232
|
+
const resolved = resolveBoundaries({
|
|
1233
|
+
startRef: input.spec.startRef,
|
|
1234
|
+
endRef: input.spec.endRef,
|
|
1235
|
+
messages: input.messages,
|
|
1236
|
+
state: input.state
|
|
1237
|
+
});
|
|
1238
|
+
const isBlockBoundary = resolved.boundaryKind === "block";
|
|
1239
|
+
const targetTier = resolveTargetTier(
|
|
1240
|
+
input.state,
|
|
1241
|
+
resolved.nestedBlockIds,
|
|
1242
|
+
isBlockBoundary
|
|
1243
|
+
);
|
|
1244
|
+
const outputTier = isBlockBoundary ? Math.min(3, targetTier + 1) : 1;
|
|
1245
|
+
const consumedBlockIds = resolved.nestedBlockIds.filter((id) => {
|
|
1246
|
+
const block2 = blockById(input.state, id);
|
|
1247
|
+
return block2?.active && block2.tier === targetTier;
|
|
1248
|
+
});
|
|
1249
|
+
const effectiveMessageIds = new Set(resolved.messageIds);
|
|
1250
|
+
for (const consumedId of consumedBlockIds) {
|
|
1251
|
+
const consumed = blockById(input.state, consumedId);
|
|
1252
|
+
if (consumed) {
|
|
1253
|
+
for (const id of consumed.effectiveMessageIds)
|
|
1254
|
+
effectiveMessageIds.add(id);
|
|
1255
|
+
}
|
|
1256
|
+
}
|
|
1257
|
+
const directMessageIds = [...effectiveMessageIds].filter(
|
|
1258
|
+
(id) => !input.preExistingCoverage.has(id)
|
|
1259
|
+
);
|
|
1260
|
+
const { filteredIds, appendedProtectedText } = filterProtectedToolMessages(
|
|
1261
|
+
directMessageIds,
|
|
1262
|
+
input.messages,
|
|
1263
|
+
input.config
|
|
1264
|
+
);
|
|
1265
|
+
validateCompressionRange(input, filteredIds, consumedBlockIds.length);
|
|
1266
|
+
let compressedTokens = 0;
|
|
1267
|
+
for (const id of filteredIds) {
|
|
1268
|
+
const message = input.messages.find((entry) => entry.id === id);
|
|
1269
|
+
compressedTokens += input.countTokens(message?.text ?? "");
|
|
1270
|
+
}
|
|
1271
|
+
for (const consumedId of consumedBlockIds) {
|
|
1272
|
+
const consumed = blockById(input.state, consumedId);
|
|
1273
|
+
if (consumed) {
|
|
1274
|
+
compressedTokens += input.countTokens(consumed.summary);
|
|
1275
|
+
}
|
|
1276
|
+
}
|
|
1277
|
+
const blockId = allocateBlockId(input.state);
|
|
1278
|
+
const finalSummary = appendedProtectedText.length > 0 ? `${input.spec.summary}
|
|
1279
|
+
|
|
1280
|
+
The following protected tool calls were used in this conversation as well:
|
|
1281
|
+
${appendedProtectedText.join("")}` : input.spec.summary;
|
|
1282
|
+
const maxLen = input.spec.summaryMaxChars ?? input.config.compress.maxSummaryLength;
|
|
1283
|
+
if (maxLen > 0 && finalSummary.length > maxLen) {
|
|
1284
|
+
throw new Error(
|
|
1285
|
+
`Summary too long after appending protected content (${finalSummary.length} chars, max ${maxLen}). Reduce the summary or remove protected tools from the range.`
|
|
1286
|
+
);
|
|
1287
|
+
}
|
|
1288
|
+
const block = {
|
|
1289
|
+
blockId,
|
|
1290
|
+
runId: input.runId,
|
|
1291
|
+
tier: outputTier,
|
|
1292
|
+
topic: input.spec.topic,
|
|
1293
|
+
summary: finalSummary,
|
|
1294
|
+
directMessageIds: filteredIds,
|
|
1295
|
+
effectiveMessageIds: [...effectiveMessageIds],
|
|
1296
|
+
directBlockIds: [...consumedBlockIds],
|
|
1297
|
+
compressedTokens,
|
|
1298
|
+
createdAt: Date.now(),
|
|
1299
|
+
survivedCount: 0,
|
|
1300
|
+
generation: "young",
|
|
1301
|
+
active: true,
|
|
1302
|
+
compressCallId: input.spec.compressCallId
|
|
1303
|
+
};
|
|
1304
|
+
input.state.blocks.push(block);
|
|
1305
|
+
for (const consumedId of consumedBlockIds) {
|
|
1306
|
+
const consumed = blockById(input.state, consumedId);
|
|
1307
|
+
if (consumed) consumed.active = false;
|
|
1308
|
+
}
|
|
1309
|
+
return compressedTokens;
|
|
1310
|
+
}
|
|
1311
|
+
function validateCompressionRange(input, directMessageIds, consumedBlockCount) {
|
|
1312
|
+
const cfg = input.config.compress;
|
|
1313
|
+
const summary = input.spec.summary?.trim() ?? "";
|
|
1314
|
+
if (summary.length === 0) {
|
|
1315
|
+
throw new Error(
|
|
1316
|
+
"Summary is empty \u2014 provide a meaningful summary of the compressed range."
|
|
1317
|
+
);
|
|
1318
|
+
}
|
|
1319
|
+
if (cfg.minSummaryLength > 0 && summary.length < cfg.minSummaryLength) {
|
|
1320
|
+
throw new Error(
|
|
1321
|
+
`Summary too short (${summary.length} chars, min ${cfg.minSummaryLength}). The summary must capture the compressed range's key information.`
|
|
1322
|
+
);
|
|
1323
|
+
}
|
|
1324
|
+
const effectiveMax = input.spec.summaryMaxChars ?? cfg.maxSummaryLength;
|
|
1325
|
+
if (effectiveMax > 0 && summary.length > effectiveMax) {
|
|
1326
|
+
throw new Error(
|
|
1327
|
+
`Summary too long (${summary.length} chars, max ${effectiveMax}). Strip noise \u2014 keep critical paths, decisions, errors, and code references. Or pass summaryMaxChars to increase the limit \u2014 don't lose critical info just to fit.`
|
|
1328
|
+
);
|
|
1329
|
+
}
|
|
1330
|
+
if (directMessageIds.length === 0 && consumedBlockCount === 0) {
|
|
1331
|
+
throw new Error(
|
|
1332
|
+
"Range contains no compressible messages \u2014 all are already covered by active blocks or protected."
|
|
1333
|
+
);
|
|
1334
|
+
}
|
|
1335
|
+
}
|
|
1336
|
+
function filterProtectedToolMessages(directMessageIds, messages, config) {
|
|
1337
|
+
const removedToolCallIds = /* @__PURE__ */ new Set();
|
|
1338
|
+
const allProtectedCallIds = /* @__PURE__ */ new Set();
|
|
1339
|
+
const removedIds = /* @__PURE__ */ new Set();
|
|
1340
|
+
const appendedProtectedText = [];
|
|
1341
|
+
for (const msg of messages) {
|
|
1342
|
+
if (isMessageProtected(msg, config) && msg.toolCallId) {
|
|
1343
|
+
allProtectedCallIds.add(msg.toolCallId);
|
|
1344
|
+
}
|
|
1345
|
+
}
|
|
1346
|
+
for (const id of directMessageIds) {
|
|
1347
|
+
const msg = messages.find((m) => m.id === id);
|
|
1348
|
+
if (!msg) continue;
|
|
1349
|
+
if (!isMessageProtected(msg, config)) continue;
|
|
1350
|
+
removedIds.add(id);
|
|
1351
|
+
if (msg.toolCallId) removedToolCallIds.add(msg.toolCallId);
|
|
1352
|
+
if (msg.text) {
|
|
1353
|
+
appendedProtectedText.push(
|
|
1354
|
+
`
|
|
1355
|
+
### Protected: ${msg.toolName ?? "tool"}
|
|
1356
|
+
${msg.text}`
|
|
1357
|
+
);
|
|
1358
|
+
}
|
|
1359
|
+
}
|
|
1360
|
+
for (const id of directMessageIds) {
|
|
1361
|
+
if (removedIds.has(id)) continue;
|
|
1362
|
+
const msg = messages.find((m) => m.id === id);
|
|
1363
|
+
if (!msg) continue;
|
|
1364
|
+
if (msg.contentType === "tool-result" && msg.toolCallId && (removedToolCallIds.has(msg.toolCallId) || allProtectedCallIds.has(msg.toolCallId))) {
|
|
1365
|
+
removedIds.add(id);
|
|
1366
|
+
}
|
|
1367
|
+
}
|
|
1368
|
+
return {
|
|
1369
|
+
filteredIds: directMessageIds.filter((id) => !removedIds.has(id)),
|
|
1370
|
+
appendedProtectedText
|
|
1371
|
+
};
|
|
1372
|
+
}
|
|
1373
|
+
function resolveTargetTier(state, nestedBlockIds, isBlockBoundary) {
|
|
1374
|
+
if (!isBlockBoundary) return 1;
|
|
1375
|
+
if (nestedBlockIds.length === 0) return 1;
|
|
1376
|
+
let minTier = 3;
|
|
1377
|
+
for (const id of nestedBlockIds) {
|
|
1378
|
+
const block = blockById(state, id);
|
|
1379
|
+
if (block && block.tier < minTier) minTier = block.tier;
|
|
1380
|
+
}
|
|
1381
|
+
return minTier;
|
|
1382
|
+
}
|
|
1383
|
+
function collectCoverage(state) {
|
|
1384
|
+
const coverage = /* @__PURE__ */ new Set();
|
|
1385
|
+
for (const block of activeBlocks(state)) {
|
|
1386
|
+
for (const id of block.effectiveMessageIds) coverage.add(id);
|
|
1387
|
+
}
|
|
1388
|
+
return coverage;
|
|
1389
|
+
}
|
|
1390
|
+
function resolveAdaptiveGrowth(modelContextLimit, nudge) {
|
|
1391
|
+
if (!modelContextLimit || modelContextLimit <= 0) return nudge.growthFloor;
|
|
1392
|
+
return Math.min(
|
|
1393
|
+
nudge.growthCap,
|
|
1394
|
+
Math.max(
|
|
1395
|
+
nudge.growthFloor,
|
|
1396
|
+
Math.round(modelContextLimit * nudge.growthRatio)
|
|
1397
|
+
)
|
|
1398
|
+
);
|
|
1399
|
+
}
|
|
1400
|
+
function decideNudge(input) {
|
|
1401
|
+
const { config, state, tokenCount, recommendation } = input;
|
|
1402
|
+
const limit = config.modelContextLimit;
|
|
1403
|
+
const usage = limit > 0 ? tokenCount / limit : 0;
|
|
1404
|
+
const nudgeGrowthTokens = resolveAdaptiveGrowth(limit, config.nudge);
|
|
1405
|
+
const emergencyOverride = usage >= config.nudge.emergencyThresholdPct;
|
|
1406
|
+
const baseline = state.nudge.lastPerMessageNudgeTokens;
|
|
1407
|
+
const hadPendingNudge = state.nudge.lastNudgeShownTokens > 0;
|
|
1408
|
+
const hasPendingNudge = hadPendingNudge;
|
|
1409
|
+
const effectiveThreshold = hasPendingNudge ? Math.floor(nudgeGrowthTokens / 2) : nudgeGrowthTokens;
|
|
1410
|
+
const growthReference = state.nudge.lastNudgeShownTokens > 0 ? state.nudge.lastNudgeShownTokens : baseline > 0 ? baseline : tokenCount;
|
|
1411
|
+
const growthFloor = Math.max(
|
|
1412
|
+
config.nudge.minGrowthFloor,
|
|
1413
|
+
config.nudge.minGrowthRatio * nudgeGrowthTokens
|
|
1414
|
+
);
|
|
1415
|
+
const growthSinceReference = tokenCount - growthReference;
|
|
1416
|
+
const growthTriggered = growthSinceReference >= effectiveThreshold;
|
|
1417
|
+
let shouldContextNudge = emergencyOverride || growthTriggered;
|
|
1418
|
+
if (shouldContextNudge && !emergencyOverride) {
|
|
1419
|
+
shouldContextNudge = growthSinceReference >= growthFloor;
|
|
1420
|
+
}
|
|
1421
|
+
let tier = null;
|
|
1422
|
+
if (config.tiers.enabled) {
|
|
1423
|
+
const active = activeBlocks(state);
|
|
1424
|
+
const activeT1 = active.filter((b) => b.tier === 1);
|
|
1425
|
+
const activeT2 = active.filter((b) => b.tier === 2);
|
|
1426
|
+
if (activeT2.length >= config.tiers.tier3Trigger) tier = 3;
|
|
1427
|
+
else if (activeT1.length >= config.tiers.tier2Trigger) tier = 2;
|
|
1428
|
+
}
|
|
1429
|
+
const rec = recommendation;
|
|
1430
|
+
const nothingToCompress = rec ? rec.contextRanges.compressible.length === 0 : false;
|
|
1431
|
+
const shouldInject = shouldContextNudge || tier !== null;
|
|
1432
|
+
let reason;
|
|
1433
|
+
if (!shouldInject) {
|
|
1434
|
+
reason = `growth ${growthSinceReference} < threshold ${effectiveThreshold}${hasPendingNudge ? " (halved)" : ""} (floor ${growthFloor})`;
|
|
1435
|
+
} else if (emergencyOverride) {
|
|
1436
|
+
reason = `EMERGENCY: usage ${Math.round(usage * 100)}% >= ${Math.round(config.nudge.emergencyThresholdPct * 100)}%`;
|
|
1437
|
+
} else if (tier !== null) {
|
|
1438
|
+
reason = `tier-${tier} distillation trigger`;
|
|
1439
|
+
} else if (nothingToCompress) {
|
|
1440
|
+
reason = `growth ${growthSinceReference} >= ${effectiveThreshold} but no specific ranges worth compressing`;
|
|
1441
|
+
} else {
|
|
1442
|
+
reason = `growth ${growthSinceReference} >= ${effectiveThreshold}${hasPendingNudge ? " (halved)" : ""}, usage ${Math.round(usage * 100)}%`;
|
|
1443
|
+
}
|
|
1444
|
+
const ctxBreakdown = computeContextBreakdown(input.messages, tokenCount, growthSinceReference);
|
|
1445
|
+
return {
|
|
1446
|
+
shouldInject,
|
|
1447
|
+
reason,
|
|
1448
|
+
compressibleRanges: rec?.recommendedRanges ?? [],
|
|
1449
|
+
protectedRanges: rec?.contextRanges.protected ?? [],
|
|
1450
|
+
contextUsage: usage,
|
|
1451
|
+
tier,
|
|
1452
|
+
breakdown: {
|
|
1453
|
+
usage,
|
|
1454
|
+
growth: growthSinceReference,
|
|
1455
|
+
growthReference,
|
|
1456
|
+
effectiveThreshold,
|
|
1457
|
+
nudgeGrowthTokens,
|
|
1458
|
+
growthFloor,
|
|
1459
|
+
hasPendingNudge: hasPendingNudge ? 1 : 0,
|
|
1460
|
+
emergencyOverride: emergencyOverride ? 1 : 0
|
|
1461
|
+
},
|
|
1462
|
+
contextBreakdown: ctxBreakdown
|
|
1463
|
+
};
|
|
1464
|
+
}
|
|
1465
|
+
function computeContextBreakdown(messages, total, growth) {
|
|
1466
|
+
let system = 0, tool = 0, summaries = 0, code = 0, text = 0;
|
|
1467
|
+
for (const msg of messages) {
|
|
1468
|
+
const tokens = Math.ceil((msg.text ?? "").length / 4);
|
|
1469
|
+
if (msg.text?.startsWith("[Compressed conversation section]")) {
|
|
1470
|
+
summaries += tokens;
|
|
1471
|
+
} else if (msg.contentType === "tool-call" || msg.contentType === "tool-result") {
|
|
1472
|
+
tool += tokens;
|
|
1473
|
+
} else if (msg.role === "system") {
|
|
1474
|
+
system += tokens;
|
|
1475
|
+
} else if (msg.text?.includes("```")) {
|
|
1476
|
+
code += tokens;
|
|
1477
|
+
} else {
|
|
1478
|
+
text += tokens;
|
|
1479
|
+
}
|
|
1480
|
+
}
|
|
1481
|
+
return { system, tool, summaries, code, text, total, growth };
|
|
1482
|
+
}
|
|
1483
|
+
function cloneState(state) {
|
|
1484
|
+
return {
|
|
1485
|
+
blocks: state.blocks.map((block) => ({
|
|
1486
|
+
...block,
|
|
1487
|
+
directMessageIds: [...block.directMessageIds],
|
|
1488
|
+
effectiveMessageIds: [...block.effectiveMessageIds],
|
|
1489
|
+
directBlockIds: [...block.directBlockIds]
|
|
1490
|
+
})),
|
|
1491
|
+
messageRefs: {
|
|
1492
|
+
byRaw: { ...state.messageRefs.byRaw },
|
|
1493
|
+
byRef: { ...state.messageRefs.byRef }
|
|
1494
|
+
},
|
|
1495
|
+
nudge: { ...state.nudge, anchors: { ...state.nudge.anchors } },
|
|
1496
|
+
stats: { ...state.stats },
|
|
1497
|
+
nextBlockId: state.nextBlockId,
|
|
1498
|
+
nextRunId: state.nextRunId
|
|
1499
|
+
};
|
|
1500
|
+
}
|
|
1501
|
+
function scoreRelevance(block, terms) {
|
|
1502
|
+
const topic = (block.topic ?? "").toLowerCase();
|
|
1503
|
+
const summary = block.summary.toLowerCase();
|
|
1504
|
+
let score = 0;
|
|
1505
|
+
for (const term of terms) {
|
|
1506
|
+
const topicHits = countOccurrences(topic, term);
|
|
1507
|
+
if (topicHits > 0) score += Math.min(topicHits * 0.15, 0.45);
|
|
1508
|
+
const summaryHits = countOccurrences(summary, term);
|
|
1509
|
+
if (summaryHits > 0) score += Math.min(summaryHits * 0.04, 0.2);
|
|
1510
|
+
}
|
|
1511
|
+
return Math.min(score, 1);
|
|
1512
|
+
}
|
|
1513
|
+
function countOccurrences(haystack, needle) {
|
|
1514
|
+
if (!haystack || !needle) return 0;
|
|
1515
|
+
let count = 0;
|
|
1516
|
+
let position = 0;
|
|
1517
|
+
while ((position = haystack.indexOf(needle, position)) !== -1) {
|
|
1518
|
+
count++;
|
|
1519
|
+
position += needle.length;
|
|
1520
|
+
}
|
|
1521
|
+
return count;
|
|
1522
|
+
}
|
|
1523
|
+
|
|
1524
|
+
// src/compression-rules.ts
|
|
1525
|
+
var COMPRESS_PHILOSOPHY = `Compression Philosophy:
|
|
1526
|
+
- All compression serves the primary task, but be frugal.
|
|
1527
|
+
- Context capacity is precious. Save context by compressing consumed outputs, not by avoiding tools.
|
|
1528
|
+
- Compress by need, not by percentage.
|
|
1529
|
+
- Work from summaries, not raw tool outputs. All listed ranges (user prompts, tool outputs, code, logs, exploration, intermediate steps) should be compressed to summary format \u2014 the ONLY exceptions are protected content, content the current step is actively using, or critical content you cannot reconstruct.
|
|
1530
|
+
- Curate summaries like a well-structured document. User prompts, compressed tool outputs, code, logs, or skill-call intermediate results that are critically important should be preserved \u2014 not by exempting them from compression, but by embedding them in the summary via [[KEEP:mNNNNN]] (auto-expanded verbatim) and [[REF:mNNNNN|description]] (compact link).`;
|
|
1531
|
+
var HOW_TO_COMPRESS_RULES = `HOW TO COMPRESS
|
|
1532
|
+
|
|
1533
|
+
When you call \`compress\`, the summary you write becomes the only record of the replaced conversation. Make it self-contained and complete: every user request, experiment purpose, and work task in the range must be accurately captured. A later reader (or you, after decompressing) should be able to continue the task WITHOUT needing the original.
|
|
1534
|
+
|
|
1535
|
+
KEEP VERBATIM \u2014 never paraphrase or abbreviate these:
|
|
1536
|
+
- Full file paths with line numbers, directory prefix on every mention (\`lib/hooks.ts:347\`, \`src/index.ts:12-18\`, \`gatenet_v3/model.py:45\`). Never abbreviate to a bare filename (\`hooks.ts\`, \`model.py\`) \u2014 they are ambiguous and cannot be grepped or decompressed-to later.
|
|
1537
|
+
- Function, class, and type signatures (exact names, params, return types) AND critical code lines that encode logic \u2014 the line that IS the finding, not just the function name (e.g. \`kv_keys += define_gate * a_key[i](emb)\` is more useful than "see model_kvnet.py").
|
|
1538
|
+
- Error messages and stack traces (exact text \u2014 you need the literal string to grep for it later).
|
|
1539
|
+
- Key details from reports and analyses \u2014 not just the conclusion. Keep the comparison numbers and the mechanism, not "X is worse" alone (write "1.76\xD7 PPL gap because KV store is static", not "KVNet underperforms").
|
|
1540
|
+
- Decisions and their rationale ("chose X over Y because Z" \u2014 the "because" is load-bearing; without it the decision looks arbitrary).
|
|
1541
|
+
- Constraints discovered ("must support Node 22", "no new dependencies", "AGENTS.md forbids \`as any\`").
|
|
1542
|
+
- Exact values: versions, config keys, thresholds, magic numbers.
|
|
1543
|
+
- User intent \u2014 quote short user messages verbatim. When the message is too long to quote, preserve intent with extra care: do not change scope, constraints, priorities, acceptance criteria, or requested outcomes. Mark them clearly as past quotes (e.g., "User said: ..."), not as current directives. Losing these changes the task itself.
|
|
1544
|
+
- The user's overall goal and any changes to it \u2014 the big-picture objective plus how it evolved during the compressed range. Each summary must reflect the goal as it stood at the end of the range, including pivots (e.g., "initially: fix bug X \u2192 pivoted to: refactor module Y after discovering root cause"). Losing the goal or its evolution makes all subsequent work appear unmotivated.
|
|
1545
|
+
- Purpose behind each significant action \u2014 preserve not just what was done but why: the hypothesis behind each experiment, the question behind each exploration, the task goal behind each work action. Without purpose, the summary reads as disconnected technical steps with no through-line.
|
|
1546
|
+
- Open questions and unresolved TODOs \u2014 losing these changes what work appears to remain.
|
|
1547
|
+
- Message refs of key anchors (\`m00420\`, \`m00510\u2013m00520\`) \u2014 they let you or a later reader jump back via decompress to the exact original.
|
|
1548
|
+
|
|
1549
|
+
DROP \u2014 extract the signal, discard the vessel:
|
|
1550
|
+
- Verbose logs (build/test/\`npm\` output) once you have captured the error line or the result.
|
|
1551
|
+
- Duplicate file reads once the needed content is recorded.
|
|
1552
|
+
- Consumed exploration \u2014 search hits, agent return values, successful tool outputs \u2014 once you have extracted the facts you need (same rule as dead-ends, but nothing went wrong; the content is simply spent).
|
|
1553
|
+
- Dead-end exploration \u2014 but PRESERVE the lesson in one line: "tried X, failed because Y".
|
|
1554
|
+
- Back-and-forth discussion and self-corrections once the final position is captured (keep the outcome, drop the journey to it).
|
|
1555
|
+
- Repeated status checks (\`git status\`, \`ls\`) once state is known.
|
|
1556
|
+
|
|
1557
|
+
For each significant item you DROP (scripts, reports, large analyses, long tool outputs), add a one-line CONTENT description of what it covers \u2014 not where it lives. Bad: "probe script at /path/probe_kvnet.py". Good: "probe_kvnet.py: tests n-gram baseline, generation quality, long-range dependency, position sensitivity, op pipeline, QUERY attention." This lets a later decompress target the right block by relevance, not by guessing locations.
|
|
1558
|
+
|
|
1559
|
+
KEEP MARKERS: \`[[KEEP:mNNNNN]]\` expands original message content into the summary (truncated to a max length). Do NOT use KEEP for verbose command output, diagnostic scripts, log dumps, or any content whose value is in the conclusion rather than the raw output \u2014 summarize these or use \`[[REF:mNNNNN|desc]\` instead.
|
|
1560
|
+
|
|
1561
|
+
PRIORITY \u2014 when the summary must be compact, preserve in this order:
|
|
1562
|
+
1. User's overall goal, goal evolution, intent, and hard constraints (losing these changes the task).
|
|
1563
|
+
2. Decisions and rationale.
|
|
1564
|
+
3. Exact technical artifacts: paths, signatures, errors, values.
|
|
1565
|
+
4. Conclusions and key findings.
|
|
1566
|
+
5. Lessons learned: what failed and why.
|
|
1567
|
+
|
|
1568
|
+
Write dense, scannable bullets \u2014 not narrative prose. If the range spans distinct concerns (request \u2192 findings \u2192 decision), group bullets under short thematic headers so a reader can scan to the part they need. Every line must earn its place. Do not mimic the style of existing summaries in context; follow these rules.`;
|
|
1569
|
+
var TIER2_DISTILL_RULES = `TIER 2 COMPRESSION \u2014 DISTILLATION
|
|
1570
|
+
|
|
1571
|
+
You are compressing historical summaries (not raw conversation). These summaries have already captured the details. Your job is to DISTILL them: extract only what matters for future work, discard the process.
|
|
1572
|
+
|
|
1573
|
+
KEEP \u2014 these are the only things that survive distillation:
|
|
1574
|
+
- Decisions and their rationale ("chose X over Y because Z" \u2014 the "because" is load-bearing).
|
|
1575
|
+
- Final outcomes: version numbers shipped, PR numbers merged/closed, bugs fixed or deferred.
|
|
1576
|
+
- Key lessons: what failed and why ("tried X, failed because Y"). These prevent repeating mistakes.
|
|
1577
|
+
- Critical constraints discovered ("must support Node 22", "AGENTS.md forbids as any").
|
|
1578
|
+
- Design decisions with architectural impact ("chose compress-as-anchor over synthetic messages because prefix cache").
|
|
1579
|
+
- Whether content is OBSOLETE or SUPERSEDED \u2014 mark with one line: "[SUPERSEDED by PR #NNN]" or "[OBSOLETE: deleted in vX.Y.Z]". Do NOT keep the obsolete content's details \u2014 just the marker and reason.
|
|
1580
|
+
- Function/class/type names and module paths that are the SUBJECT of the work \u2014 e.g., "fixed filterCompressedRanges in prune.ts", "added SessionStateRegistry in state.ts". Not exact line numbers or full signatures \u2014 just enough to LOCATE the code without searching.
|
|
1581
|
+
- Exploration findings: if a block was exploratory with no decision, keep the CONCLUSION in one line ("explored X, not viable because Y"). Do not keep the exploration process.
|
|
1582
|
+
|
|
1583
|
+
DROP \u2014 these were useful during the work but are no longer needed:
|
|
1584
|
+
- Exact line numbers, diffs, verbose function signatures, full code listings.
|
|
1585
|
+
- Build/deploy process details, test execution steps.
|
|
1586
|
+
- Review process details (who reviewed, what rounds, test counts).
|
|
1587
|
+
- Verbose logs, command output, intermediate debugging steps.
|
|
1588
|
+
|
|
1589
|
+
FORMAT:
|
|
1590
|
+
- Start each distilled block with a source header line:
|
|
1591
|
+
\`Source: bN+bM+... (XK\u2192YK tok, Zx). [original topic]\`
|
|
1592
|
+
Example: \`Source: b5+b7 (56K+44K\u2192268 tok, 375x). [Tool-result recap + publish]\`
|
|
1593
|
+
- 3-5 bullet points per source block, each a self-contained fact.
|
|
1594
|
+
- Dense, scannable \u2014 no narrative prose.
|
|
1595
|
+
- Start with the outcome, not the process: "v1.13.0 shipped (7 PRs bundled)" not "implemented 7 PRs then reviewed then merged".
|
|
1596
|
+
- Cross-block synthesis: if multiple source blocks cover the same topic (same PR, same feature, same bug), MERGE them into a single group of bullets. Do not repeat the same fact from different blocks \u2014 keep it once under the most relevant source header.
|
|
1597
|
+
|
|
1598
|
+
SIZE TARGET: 50-150 tokens per source block (excluding the header). If you can't fit it in 150 tokens, you're keeping too much process. If a block has nothing worth keeping (pure noise), output just the header followed by "[no actionable content]."`;
|
|
1599
|
+
var TIER3_CONDENSE_RULES = `TIER 3 COMPRESSION \u2014 ULTRA-CONDENSATION
|
|
1600
|
+
|
|
1601
|
+
You are compressing distilled summaries (Tier 2) into ultra-condensed facts (Tier 3). The distilled summaries already contain only decisions and outcomes. Your job is to reduce them to bare factual references.
|
|
1602
|
+
|
|
1603
|
+
PRIORITY \u2014 when a source block has more facts than the size target allows, keep in this order:
|
|
1604
|
+
1. Shipped outcomes (versions released, PRs merged) \u2014 these are permanent record.
|
|
1605
|
+
2. Open work (PRs/issues still pending) \u2014 these may need follow-up.
|
|
1606
|
+
3. Key decisions with architectural impact ("chose X over Y because Z").
|
|
1607
|
+
4. Critical constraints ("must support Node 22").
|
|
1608
|
+
Drop everything else. Tier 3 is a lookup index, not a knowledge base.
|
|
1609
|
+
|
|
1610
|
+
FORMAT:
|
|
1611
|
+
- Start with a source header line:
|
|
1612
|
+
\`Source: bN+bM+... (XK\u2192YK tok, Zx). [original topic]\`
|
|
1613
|
+
- Output 1-3 facts per source block. Each fact is a single line: subject + outcome.
|
|
1614
|
+
- No explanations, no rationale, no process \u2014 just the fact.
|
|
1615
|
+
- Format: "[PR/Issue/Version] \u2014 [outcome in \u22648 words]"
|
|
1616
|
+
- Merge related facts from different source blocks if they concern the same topic.
|
|
1617
|
+
|
|
1618
|
+
EXAMPLES:
|
|
1619
|
+
- "v1.13.0 shipped \u2014 quality gate + GC fix (7 PRs)"
|
|
1620
|
+
- "PR #196 merged \u2014 preserve-first-user (supersedes #169)"
|
|
1621
|
+
- "Bug 1214 fixed \u2014 compress consumed all user messages"
|
|
1622
|
+
- "Chose compress-as-anchor \u2014 prefix cache benefit over synthetic injection"
|
|
1623
|
+
- "Constraint: AGENTS.md forbids as any \u2014 never suppress types"
|
|
1624
|
+
|
|
1625
|
+
DROP:
|
|
1626
|
+
- Multi-sentence context. If a fact needs >1 sentence, it's too detailed for Tier 3.
|
|
1627
|
+
- Lessons learned ("tried X, failed because Y") \u2014 drop UNLESS the failure is likely to recur and the block is <30 days old.
|
|
1628
|
+
- Design rationale details \u2014 keep the decision, drop the "because" unless it's a critical constraint.
|
|
1629
|
+
- Anything marked [OBSOLETE] or [SUPERSEDED] \u2014 drop entirely, note "[N blocks obsolete]" in the summary.
|
|
1630
|
+
|
|
1631
|
+
SIZE TARGET: 30-60 tokens per source block (including header). For a batch of N source blocks, total output \u2248 N \xD7 40 tokens. If a source block has only one trivial fact, output just the header + one line.`;
|
|
1632
|
+
|
|
1633
|
+
// src/nudge-text.ts
|
|
1634
|
+
var EFFICIENCY_NOTE = `This is an efficiency nudge to compress early and keep context lean \u2014 not an overflow warning. A separate, stronger alert will appear if the context is actually full.
|
|
1635
|
+
|
|
1636
|
+
${COMPRESS_PHILOSOPHY}`;
|
|
1637
|
+
var EMERGENCY_HEADER = `\u26A0\uFE0F Context limit reached \u2014 compress now. Prioritize consumed tool outputs.
|
|
1638
|
+
|
|
1639
|
+
${COMPRESS_PHILOSOPHY}`;
|
|
1640
|
+
function formatK(n) {
|
|
1641
|
+
if (n >= 1e3) return `${(n / 1e3).toFixed(1)}K`;
|
|
1642
|
+
return `${n}`;
|
|
1643
|
+
}
|
|
1644
|
+
function formatBreakdown(bd) {
|
|
1645
|
+
if (!bd) return "";
|
|
1646
|
+
const parts = [];
|
|
1647
|
+
if (bd.system > 0) parts.push(`${formatK(bd.system)} system`);
|
|
1648
|
+
if (bd.tool > 0) parts.push(`${formatK(bd.tool)} tool`);
|
|
1649
|
+
if (bd.summaries > 0) parts.push(`${formatK(bd.summaries)} summaries`);
|
|
1650
|
+
if (bd.code > 0) parts.push(`${formatK(bd.code)} code`);
|
|
1651
|
+
if (bd.text > 0) parts.push(`${formatK(bd.text)} text`);
|
|
1652
|
+
const growth = bd.growth > 0 ? `
|
|
1653
|
+
+${formatK(bd.growth)} since last nudge` : "";
|
|
1654
|
+
return `Context breakdown: ${parts.join(" | ")}${growth}`;
|
|
1655
|
+
}
|
|
1656
|
+
function refToNum(ref) {
|
|
1657
|
+
if (!ref) return 0;
|
|
1658
|
+
const m = ref.match(/^m0*(\d+)$/);
|
|
1659
|
+
return m && m[1] ? parseInt(m[1], 10) : 0;
|
|
1660
|
+
}
|
|
1661
|
+
function mergeRanges(compressible, protected_) {
|
|
1662
|
+
const entries = [];
|
|
1663
|
+
for (const r of compressible) {
|
|
1664
|
+
const sNum = refToNum(r.startRef);
|
|
1665
|
+
const eNum = refToNum(r.endRef);
|
|
1666
|
+
entries.push({
|
|
1667
|
+
startRef: r.startRef,
|
|
1668
|
+
endRef: r.endRef,
|
|
1669
|
+
startNum: sNum,
|
|
1670
|
+
endNum: eNum,
|
|
1671
|
+
count: r.count,
|
|
1672
|
+
tokens: r.tokens,
|
|
1673
|
+
compressibleTokens: r.tokens,
|
|
1674
|
+
compressibleCount: r.count,
|
|
1675
|
+
protectedTokens: 0,
|
|
1676
|
+
protectedCount: 0,
|
|
1677
|
+
protectedTools: [],
|
|
1678
|
+
toolPct: r.toolPct,
|
|
1679
|
+
textPct: r.textPct,
|
|
1680
|
+
dangerous: r.dangerous ?? false
|
|
1681
|
+
});
|
|
1682
|
+
}
|
|
1683
|
+
for (const r of protected_) {
|
|
1684
|
+
const sNum = refToNum(r.startRef);
|
|
1685
|
+
const eNum = refToNum(r.endRef);
|
|
1686
|
+
entries.push({
|
|
1687
|
+
startRef: r.startRef,
|
|
1688
|
+
endRef: r.endRef,
|
|
1689
|
+
startNum: sNum,
|
|
1690
|
+
endNum: eNum,
|
|
1691
|
+
count: r.count,
|
|
1692
|
+
tokens: r.tokens,
|
|
1693
|
+
compressibleTokens: 0,
|
|
1694
|
+
compressibleCount: 0,
|
|
1695
|
+
protectedTokens: r.tokens,
|
|
1696
|
+
protectedCount: r.count,
|
|
1697
|
+
protectedTools: [...r.tools],
|
|
1698
|
+
toolPct: 0,
|
|
1699
|
+
textPct: 0,
|
|
1700
|
+
dangerous: false
|
|
1701
|
+
});
|
|
1702
|
+
}
|
|
1703
|
+
entries.sort((a, b) => a.startNum - b.startNum);
|
|
1704
|
+
const merged = [];
|
|
1705
|
+
for (const e of entries) {
|
|
1706
|
+
const last = merged[merged.length - 1];
|
|
1707
|
+
if (last && last.endNum + 1 >= e.startNum) {
|
|
1708
|
+
last.endRef = e.endRef;
|
|
1709
|
+
last.endNum = Math.max(last.endNum, e.endNum);
|
|
1710
|
+
last.count += e.count;
|
|
1711
|
+
last.tokens += e.tokens;
|
|
1712
|
+
last.compressibleTokens += e.compressibleTokens;
|
|
1713
|
+
last.compressibleCount += e.compressibleCount;
|
|
1714
|
+
last.protectedTokens += e.protectedTokens;
|
|
1715
|
+
last.protectedCount += e.protectedCount;
|
|
1716
|
+
if (e.dangerous) last.dangerous = true;
|
|
1717
|
+
for (const t of e.protectedTools) {
|
|
1718
|
+
if (!last.protectedTools.includes(t)) last.protectedTools.push(t);
|
|
1719
|
+
}
|
|
1720
|
+
} else {
|
|
1721
|
+
merged.push({ ...e });
|
|
1722
|
+
}
|
|
1723
|
+
}
|
|
1724
|
+
return merged;
|
|
1725
|
+
}
|
|
1726
|
+
function formatRanges(compressible, protected_) {
|
|
1727
|
+
const merged = mergeRanges(compressible, protected_);
|
|
1728
|
+
if (merged.length === 0) {
|
|
1729
|
+
return "[No specific ranges detected \u2014 compress any consumed content.]";
|
|
1730
|
+
}
|
|
1731
|
+
const lines = merged.slice(0, 10).map((e) => {
|
|
1732
|
+
const suffix = e.dangerous && e.compressibleTokens > 0 ? " \u26A0\uFE0F NOT recommended unless you are certain." : "";
|
|
1733
|
+
if (e.protectedTokens > 0 && e.compressibleTokens === 0) {
|
|
1734
|
+
return ` ${e.startRef}\u2013${e.endRef} ${e.count} msgs ${formatK(e.tokens)} [PROTECTED: ${e.protectedTools.join(", ")} \u2014 not compressible]${suffix}`;
|
|
1735
|
+
}
|
|
1736
|
+
if (e.protectedTokens > 0 && e.compressibleTokens > 0) {
|
|
1737
|
+
return ` ${e.startRef}\u2013${e.endRef} ${e.count} msgs ${formatK(e.tokens)} [${formatK(e.compressibleTokens)} compressible | ${formatK(e.protectedTokens)} protected: ${e.protectedTools.join(", ")}]${suffix}`;
|
|
1738
|
+
}
|
|
1739
|
+
return ` ${e.startRef}\u2013${e.endRef} ${e.count} msgs ${formatK(e.tokens)} [tool ${e.toolPct}% | text ${e.textPct}%]${suffix}`;
|
|
1740
|
+
});
|
|
1741
|
+
return `Compressible ranges (oldest first):
|
|
1742
|
+
${lines.join("\n")}`;
|
|
1743
|
+
}
|
|
1744
|
+
function renderNudgeText(decision) {
|
|
1745
|
+
const breakdownStr = formatBreakdown(decision.contextBreakdown);
|
|
1746
|
+
const rangesStr = formatRanges(decision.compressibleRanges, decision.protectedRanges ?? []);
|
|
1747
|
+
if (decision.tier !== null && decision.tier >= 2) {
|
|
1748
|
+
const isT2 = decision.tier === 2;
|
|
1749
|
+
return {
|
|
1750
|
+
voice: "gentle",
|
|
1751
|
+
text: [
|
|
1752
|
+
EFFICIENCY_NOTE,
|
|
1753
|
+
"",
|
|
1754
|
+
breakdownStr,
|
|
1755
|
+
"",
|
|
1756
|
+
`[TIER ${decision.tier} ${isT2 ? "DISTILLATION" : "CONDENSATION"} TRIGGER]`,
|
|
1757
|
+
isT2 ? "Your tier-1 compression summaries have accumulated. Distill them into a single denser tier-2 summary. Use block IDs as boundaries." : "Your tier-2 compression summaries have accumulated. Condense them further into a tier-3 ultra-condensed summary. Use block IDs as boundaries.",
|
|
1758
|
+
`Example: compress({ content: [{ startId: "b1", endId: "b5", summary: "..." }] })`,
|
|
1759
|
+
"",
|
|
1760
|
+
HOW_TO_COMPRESS_RULES,
|
|
1761
|
+
"",
|
|
1762
|
+
isT2 ? TIER2_DISTILL_RULES : TIER3_CONDENSE_RULES
|
|
1763
|
+
].join("\n")
|
|
1764
|
+
};
|
|
1765
|
+
}
|
|
1766
|
+
const isEmergency = !!decision.breakdown?.emergencyOverride;
|
|
1767
|
+
if (isEmergency) {
|
|
1768
|
+
return {
|
|
1769
|
+
voice: "emergency",
|
|
1770
|
+
text: [
|
|
1771
|
+
EMERGENCY_HEADER,
|
|
1772
|
+
"",
|
|
1773
|
+
breakdownStr,
|
|
1774
|
+
"",
|
|
1775
|
+
HOW_TO_COMPRESS_RULES,
|
|
1776
|
+
"",
|
|
1777
|
+
`{ "topic": "...", "content": [{ "startId": "<ID>", "endId": "<ID>", "summary": "..." }] }`,
|
|
1778
|
+
"Only use IDs from visible messages above. Compress older work first.",
|
|
1779
|
+
"",
|
|
1780
|
+
rangesStr
|
|
1781
|
+
].join("\n")
|
|
1782
|
+
};
|
|
1783
|
+
}
|
|
1784
|
+
return {
|
|
1785
|
+
voice: "gentle",
|
|
1786
|
+
text: [
|
|
1787
|
+
EFFICIENCY_NOTE,
|
|
1788
|
+
"",
|
|
1789
|
+
breakdownStr,
|
|
1790
|
+
"",
|
|
1791
|
+
HOW_TO_COMPRESS_RULES,
|
|
1792
|
+
"",
|
|
1793
|
+
rangesStr,
|
|
1794
|
+
"",
|
|
1795
|
+
`\u{1F4A1} Compress all ranges in one call (pass multiple content entries: \`content: [{...}, {...}]\`).`
|
|
1796
|
+
].join("\n")
|
|
1797
|
+
};
|
|
1798
|
+
}
|
|
1799
|
+
|
|
1800
|
+
// src/keep-markers.ts
|
|
1801
|
+
var KEEP_REGEX = /\[\[KEEP:(m\d+)\]\]/g;
|
|
1802
|
+
var REF_REGEX = /\[\[REF:(m\d+)\|([^\]]+)\]\]/g;
|
|
1803
|
+
function resolveKeepMarkers(summary, messages, state, maxChars = 2e3) {
|
|
1804
|
+
const messageByRef = /* @__PURE__ */ new Map();
|
|
1805
|
+
for (const message of messages) {
|
|
1806
|
+
const ref = refForRaw(state.messageRefs, message.id);
|
|
1807
|
+
if (ref) messageByRef.set(ref, message);
|
|
1808
|
+
}
|
|
1809
|
+
let expandedCount = 0;
|
|
1810
|
+
let refCount = 0;
|
|
1811
|
+
const unresolvedRefs = [];
|
|
1812
|
+
const expanded = summary.replace(KEEP_REGEX, (match, ref) => {
|
|
1813
|
+
const normalized = normalizeRef(ref);
|
|
1814
|
+
const message = normalized ? messageByRef.get(normalized) : void 0;
|
|
1815
|
+
if (!message) {
|
|
1816
|
+
unresolvedRefs.push(ref);
|
|
1817
|
+
return match;
|
|
1818
|
+
}
|
|
1819
|
+
expandedCount++;
|
|
1820
|
+
return formatKeptMessage(message, normalized, maxChars);
|
|
1821
|
+
}).replace(REF_REGEX, (_match, ref, desc) => {
|
|
1822
|
+
const normalized = normalizeRef(ref);
|
|
1823
|
+
const message = normalized ? messageByRef.get(normalized) : void 0;
|
|
1824
|
+
if (!message) {
|
|
1825
|
+
unresolvedRefs.push(ref);
|
|
1826
|
+
return _match;
|
|
1827
|
+
}
|
|
1828
|
+
refCount++;
|
|
1829
|
+
return `[\u2192 ${normalized}: ${desc.trim()}]`;
|
|
1830
|
+
});
|
|
1831
|
+
return { summary: expanded, expandedCount, refCount, unresolvedRefs };
|
|
1832
|
+
}
|
|
1833
|
+
function normalizeRef(ref) {
|
|
1834
|
+
const match = /^m0*(\d{1,5})$/.exec(ref.trim().toLowerCase());
|
|
1835
|
+
if (!match || match[1] === void 0) return null;
|
|
1836
|
+
return `m${match[1].padStart(5, "0")}`;
|
|
1837
|
+
}
|
|
1838
|
+
function formatKeptMessage(message, ref, maxChars) {
|
|
1839
|
+
const label = labelFor(message);
|
|
1840
|
+
const body = truncate(message.text ?? "[empty message]", maxChars);
|
|
1841
|
+
return `
|
|
1842
|
+
--- [${ref}: ${label}] ---
|
|
1843
|
+
${body}
|
|
1844
|
+
--- end ---
|
|
1845
|
+
`;
|
|
1846
|
+
}
|
|
1847
|
+
function labelFor(message) {
|
|
1848
|
+
if (message.contentType === "tool-call" || message.contentType === "tool-result") {
|
|
1849
|
+
return message.toolName ?? "tool";
|
|
1850
|
+
}
|
|
1851
|
+
return message.role;
|
|
1852
|
+
}
|
|
1853
|
+
function truncate(text, maxChars) {
|
|
1854
|
+
if (text.length <= maxChars) return text;
|
|
1855
|
+
return text.slice(0, maxChars) + `
|
|
1856
|
+
... [truncated, ${text.length} chars total]`;
|
|
1857
|
+
}
|
|
1858
|
+
|
|
1859
|
+
// src/decompress.ts
|
|
1860
|
+
function parseBlockIdArg(arg) {
|
|
1861
|
+
const normalized = arg.trim().toLowerCase();
|
|
1862
|
+
const refMatch = /^b0*(\d+)$/.exec(normalized);
|
|
1863
|
+
if (refMatch && refMatch[1] !== void 0) return `b${refMatch[1]}`;
|
|
1864
|
+
const numMatch = /^(\d+)$/.exec(normalized);
|
|
1865
|
+
if (numMatch && numMatch[1] !== void 0) return `b${numMatch[1]}`;
|
|
1866
|
+
return null;
|
|
1867
|
+
}
|
|
1868
|
+
function findBlocksOverlappingMessages(state, messageIds) {
|
|
1869
|
+
if (messageIds.size === 0) return [];
|
|
1870
|
+
const matched = [];
|
|
1871
|
+
for (const block of state.blocks) {
|
|
1872
|
+
if (!block.active) continue;
|
|
1873
|
+
if (block.effectiveMessageIds.some((id) => messageIds.has(id))) {
|
|
1874
|
+
matched.push(block);
|
|
1875
|
+
}
|
|
1876
|
+
}
|
|
1877
|
+
return matched.sort((a, b) => numericPart2(a.blockId) - numericPart2(b.blockId));
|
|
1878
|
+
}
|
|
1879
|
+
function findActiveAncestor(state, blockId) {
|
|
1880
|
+
const start = state.blocks.find((b) => b.blockId === blockId);
|
|
1881
|
+
if (!start) return null;
|
|
1882
|
+
const queue = [...start.directBlockIds];
|
|
1883
|
+
const visited = /* @__PURE__ */ new Set();
|
|
1884
|
+
while (queue.length > 0) {
|
|
1885
|
+
const currentId = queue.shift();
|
|
1886
|
+
if (visited.has(currentId)) continue;
|
|
1887
|
+
visited.add(currentId);
|
|
1888
|
+
const current = state.blocks.find((b) => b.blockId === currentId);
|
|
1889
|
+
if (!current) continue;
|
|
1890
|
+
if (current.active) return current.blockId;
|
|
1891
|
+
for (const ancestorId of current.directBlockIds) {
|
|
1892
|
+
if (!visited.has(ancestorId)) queue.push(ancestorId);
|
|
1893
|
+
}
|
|
1894
|
+
}
|
|
1895
|
+
return null;
|
|
1896
|
+
}
|
|
1897
|
+
function deactivateBlock(state, blockIds, options = {}) {
|
|
1898
|
+
const targets = new Set(blockIds);
|
|
1899
|
+
const updated = state.blocks.map((block) => {
|
|
1900
|
+
if (!targets.has(block.blockId) || !block.active) return block;
|
|
1901
|
+
return {
|
|
1902
|
+
...block,
|
|
1903
|
+
active: false,
|
|
1904
|
+
durationMs: block.durationMs,
|
|
1905
|
+
createdAt: block.createdAt
|
|
1906
|
+
};
|
|
1907
|
+
});
|
|
1908
|
+
let final = updated;
|
|
1909
|
+
if (options.deep) {
|
|
1910
|
+
const visited = /* @__PURE__ */ new Set();
|
|
1911
|
+
const queue = [];
|
|
1912
|
+
for (const id of blockIds) {
|
|
1913
|
+
const block = updated.find((b) => b.blockId === id);
|
|
1914
|
+
if (block) queue.push(...block.directBlockIds);
|
|
1915
|
+
}
|
|
1916
|
+
while (queue.length > 0) {
|
|
1917
|
+
const id = queue.shift();
|
|
1918
|
+
if (visited.has(id)) continue;
|
|
1919
|
+
visited.add(id);
|
|
1920
|
+
final = final.map((block) => {
|
|
1921
|
+
if (block.blockId !== id) return block;
|
|
1922
|
+
queue.push(...block.directBlockIds);
|
|
1923
|
+
return block.active ? { ...block, active: false } : block;
|
|
1924
|
+
});
|
|
1925
|
+
}
|
|
1926
|
+
}
|
|
1927
|
+
return { ...state, blocks: final };
|
|
1928
|
+
}
|
|
1929
|
+
function buildRestoredContentPreview(messages, beforeActiveMessageIds, state) {
|
|
1930
|
+
const restored = [];
|
|
1931
|
+
for (const message of messages) {
|
|
1932
|
+
if (!beforeActiveMessageIds.has(message.id)) continue;
|
|
1933
|
+
const stillCovered = state.blocks.some(
|
|
1934
|
+
(b) => b.active && b.effectiveMessageIds.includes(message.id)
|
|
1935
|
+
);
|
|
1936
|
+
if (!stillCovered) restored.push(message);
|
|
1937
|
+
}
|
|
1938
|
+
if (restored.length === 0) return { preview: "", restoredCount: 0 };
|
|
1939
|
+
const lines = [];
|
|
1940
|
+
let totalLength = 0;
|
|
1941
|
+
const MAX_PREVIEW = 2e3;
|
|
1942
|
+
const MAX_PER_MESSAGE = 200;
|
|
1943
|
+
for (const message of restored) {
|
|
1944
|
+
if (totalLength >= MAX_PREVIEW) break;
|
|
1945
|
+
const text = message.text ?? "";
|
|
1946
|
+
const truncated = text.length > MAX_PER_MESSAGE ? text.slice(0, MAX_PER_MESSAGE) + "..." : text;
|
|
1947
|
+
const label = message.toolName && message.contentType !== "text" ? `${message.toolName}: ${truncated}` : `[${message.role}] ${truncated}`;
|
|
1948
|
+
lines.push(label);
|
|
1949
|
+
totalLength += label.length + 1;
|
|
1950
|
+
}
|
|
1951
|
+
return { preview: lines.join("\n"), restoredCount: restored.length };
|
|
1952
|
+
}
|
|
1953
|
+
function numericPart2(blockId) {
|
|
1954
|
+
const match = /^b(\d+)$/.exec(blockId);
|
|
1955
|
+
return match && match[1] !== void 0 ? Number(match[1]) : 0;
|
|
1956
|
+
}
|
|
1957
|
+
|
|
1958
|
+
// src/report.ts
|
|
1959
|
+
function formatTokens(n) {
|
|
1960
|
+
if (!Number.isFinite(n) || n <= 0) return "0";
|
|
1961
|
+
return n >= 1e3 ? `${(n / 1e3).toFixed(1)}K` : String(n);
|
|
1962
|
+
}
|
|
1963
|
+
function pct(n, total) {
|
|
1964
|
+
if (n <= 0 || total <= 0) return 0;
|
|
1965
|
+
return Math.max(1, Math.round(n / total * 100));
|
|
1966
|
+
}
|
|
1967
|
+
function numericPart3(blockId) {
|
|
1968
|
+
const match = /^b(\d+)$/.exec(blockId);
|
|
1969
|
+
return match && match[1] !== void 0 ? Number(match[1]) : 0;
|
|
1970
|
+
}
|
|
1971
|
+
function summaryTokensOf(block, countTokens) {
|
|
1972
|
+
return countTokens(block.summary);
|
|
1973
|
+
}
|
|
1974
|
+
function effectiveCompressedTokens(block, state, countTokens, visited = /* @__PURE__ */ new Set()) {
|
|
1975
|
+
if (visited.has(block.blockId)) return 0;
|
|
1976
|
+
visited.add(block.blockId);
|
|
1977
|
+
let total = block.compressedTokens;
|
|
1978
|
+
for (const nestedId of block.directBlockIds) {
|
|
1979
|
+
const nested = state.blocks.find((b) => b.blockId === nestedId);
|
|
1980
|
+
if (nested) total += effectiveCompressedTokens(nested, state, countTokens, visited);
|
|
1981
|
+
}
|
|
1982
|
+
return total;
|
|
1983
|
+
}
|
|
1984
|
+
function tierLabel(block) {
|
|
1985
|
+
return `T${block.tier}`;
|
|
1986
|
+
}
|
|
1987
|
+
function tierBreakdown(blocks, countTokens) {
|
|
1988
|
+
const tierTokens = {};
|
|
1989
|
+
for (const block of blocks) {
|
|
1990
|
+
tierTokens[block.tier] = (tierTokens[block.tier] ?? 0) + summaryTokensOf(block, countTokens);
|
|
1991
|
+
}
|
|
1992
|
+
const tiers = Object.keys(tierTokens).map(Number);
|
|
1993
|
+
if (tiers.length <= 1) return null;
|
|
1994
|
+
const parts = [];
|
|
1995
|
+
for (const tier of [1, 2, 3]) {
|
|
1996
|
+
if (tierTokens[tier]) parts.push(`T${tier}: ${formatTokens(tierTokens[tier])}`);
|
|
1997
|
+
}
|
|
1998
|
+
return parts.join(" | ");
|
|
1999
|
+
}
|
|
2000
|
+
function collectVisible(messages, state, countTokens) {
|
|
2001
|
+
const coveredIds = /* @__PURE__ */ new Set();
|
|
2002
|
+
for (const block of state.blocks) {
|
|
2003
|
+
if (!block.active) continue;
|
|
2004
|
+
for (const id of block.effectiveMessageIds) coveredIds.add(id);
|
|
2005
|
+
}
|
|
2006
|
+
let summaryTokens = 0;
|
|
2007
|
+
for (const block of state.blocks) {
|
|
2008
|
+
if (block.active) summaryTokens += summaryTokensOf(block, countTokens);
|
|
2009
|
+
}
|
|
2010
|
+
const visible = [];
|
|
2011
|
+
messages.forEach((message, index) => {
|
|
2012
|
+
if (coveredIds.has(message.id)) return;
|
|
2013
|
+
const ref = refForRaw(state.messageRefs, message.id);
|
|
2014
|
+
if (!ref) return;
|
|
2015
|
+
const tokens = countTokens(message.text ?? "");
|
|
2016
|
+
const tool = message.toolName ?? "text";
|
|
2017
|
+
if (tokens > 0) visible.push({ ref, tokens, tool, index });
|
|
2018
|
+
});
|
|
2019
|
+
return { visible, summaryTokens };
|
|
2020
|
+
}
|
|
2021
|
+
function buildStatusReport(state, messages, countTokens, options = {}) {
|
|
2022
|
+
const scope = options.scope;
|
|
2023
|
+
const view = options.view ?? "ranges";
|
|
2024
|
+
const toolFilter = options.tool;
|
|
2025
|
+
const sort = options.sort ?? "size";
|
|
2026
|
+
const limit = options.limit ?? 30;
|
|
2027
|
+
const activeBlocks2 = state.blocks.filter((b) => b.active).sort((a, b) => numericPart3(a.blockId) - numericPart3(b.blockId));
|
|
2028
|
+
if (scope === "compressed") {
|
|
2029
|
+
return renderCompressedDrilldown(activeBlocks2, state, sort, limit, countTokens);
|
|
2030
|
+
}
|
|
2031
|
+
const { visible, summaryTokens } = collectVisible(messages, state, countTokens);
|
|
2032
|
+
if (scope === "uncompressed") {
|
|
2033
|
+
if (view === "messages") {
|
|
2034
|
+
return renderMessageDrilldown(visible, toolFilter, sort, limit);
|
|
2035
|
+
}
|
|
2036
|
+
return renderUncompressedRanges(visible);
|
|
2037
|
+
}
|
|
2038
|
+
return renderOverview(visible, summaryTokens, activeBlocks2, state, countTokens, limit);
|
|
2039
|
+
}
|
|
2040
|
+
function renderOverview(visible, summaryTokens, blocks, state, countTokens, limit) {
|
|
2041
|
+
const lines = [];
|
|
2042
|
+
const toolTypeMap = /* @__PURE__ */ new Map();
|
|
2043
|
+
for (const message of visible) {
|
|
2044
|
+
toolTypeMap.set(message.tool, (toolTypeMap.get(message.tool) ?? 0) + message.tokens);
|
|
2045
|
+
}
|
|
2046
|
+
const topTool = [...toolTypeMap.entries()].sort((a, b) => b[1] - a[1])[0]?.[0];
|
|
2047
|
+
const totalTool = visible.filter((m) => m.tool !== "text").reduce((sum, m) => sum + m.tokens, 0);
|
|
2048
|
+
const totalText = visible.filter((m) => m.tool === "text").reduce((sum, m) => sum + m.tokens, 0);
|
|
2049
|
+
const total = summaryTokens + totalTool + totalText;
|
|
2050
|
+
lines.push("CONTEXT BREAKDOWN");
|
|
2051
|
+
lines.push(
|
|
2052
|
+
` ${formatTokens(totalTool)} tool (${pct(totalTool, total)}%) | ${formatTokens(totalText)} text (${pct(totalText, total)}%) | ${formatTokens(summaryTokens)} summaries (${pct(summaryTokens, total)}%)`
|
|
2053
|
+
);
|
|
2054
|
+
const topTypes = [...toolTypeMap.entries()].sort((a, b) => b[1] - a[1]).slice(0, 3);
|
|
2055
|
+
if (topTypes.length > 0) {
|
|
2056
|
+
lines.push(` Top tools: ${topTypes.map(([t, n]) => `${t} (${pct(n, total)}%)`).join(", ")}`);
|
|
2057
|
+
}
|
|
2058
|
+
lines.push("");
|
|
2059
|
+
if (blocks.length === 0) {
|
|
2060
|
+
lines.push("COMPRESSED BLOCKS");
|
|
2061
|
+
lines.push(" No compressed blocks.");
|
|
2062
|
+
} else {
|
|
2063
|
+
const totalSummary = blocks.reduce((s, b) => s + summaryTokensOf(b, countTokens), 0);
|
|
2064
|
+
const totalEffective = blocks.reduce(
|
|
2065
|
+
(s, b) => s + effectiveCompressedTokens(b, state, countTokens),
|
|
2066
|
+
0
|
|
2067
|
+
);
|
|
2068
|
+
lines.push(
|
|
2069
|
+
`COMPRESSED BLOCKS \u2014 ${blocks.length} active (${formatTokens(totalSummary)} summary, ${formatTokens(totalEffective)} original)`
|
|
2070
|
+
);
|
|
2071
|
+
const breakdown = tierBreakdown(blocks, countTokens);
|
|
2072
|
+
if (breakdown) lines.push(` Tier usage: ${breakdown}`);
|
|
2073
|
+
lines.push("");
|
|
2074
|
+
const sorted = [...blocks].sort(
|
|
2075
|
+
(a, b) => effectiveCompressedTokens(b, state, countTokens) - effectiveCompressedTokens(a, state, countTokens) || b.createdAt - a.createdAt
|
|
2076
|
+
);
|
|
2077
|
+
for (const block of sorted.slice(0, limit)) {
|
|
2078
|
+
const topic = block.topic ?? "(no topic)";
|
|
2079
|
+
const eff = effectiveCompressedTokens(block, state, countTokens);
|
|
2080
|
+
lines.push(
|
|
2081
|
+
` ${block.blockId} (${tierLabel(block)}) ${formatTokens(eff)}\u2192${formatTokens(summaryTokensOf(block, countTokens))} ${block.effectiveMessageIds.length} msgs "${topic}"`
|
|
2082
|
+
);
|
|
2083
|
+
}
|
|
2084
|
+
}
|
|
2085
|
+
lines.push("");
|
|
2086
|
+
lines.push(
|
|
2087
|
+
`Tip: buildStatusReport({scope:"uncompressed", view:"messages", tool:"${topTool ?? "bash"}"}) for per-message listing`
|
|
2088
|
+
);
|
|
2089
|
+
return lines.join("\n");
|
|
2090
|
+
}
|
|
2091
|
+
function renderUncompressedRanges(visible) {
|
|
2092
|
+
const lines = [];
|
|
2093
|
+
const totalTokens = visible.reduce((s, m) => s + m.tokens, 0);
|
|
2094
|
+
lines.push(`UNCOMPRESSED \u2014 ${formatTokens(totalTokens)} | ${visible.length} visible messages`);
|
|
2095
|
+
lines.push("");
|
|
2096
|
+
if (visible.length === 0) {
|
|
2097
|
+
lines.push(" (no uncompressed messages)");
|
|
2098
|
+
} else {
|
|
2099
|
+
for (const message of visible.slice(0, 30)) {
|
|
2100
|
+
lines.push(` ${message.ref} (${formatTokens(message.tokens)}) ${message.tool}`);
|
|
2101
|
+
}
|
|
2102
|
+
}
|
|
2103
|
+
return lines.join("\n");
|
|
2104
|
+
}
|
|
2105
|
+
function renderMessageDrilldown(visible, toolFilter, sort, limit) {
|
|
2106
|
+
let filtered = visible;
|
|
2107
|
+
if (toolFilter) filtered = filtered.filter((m) => m.tool === toolFilter);
|
|
2108
|
+
if (sort === "time") filtered.sort((a, b) => a.index - b.index);
|
|
2109
|
+
else if (sort === "tool") filtered.sort((a, b) => a.tool.localeCompare(b.tool) || b.tokens - a.tokens);
|
|
2110
|
+
else filtered.sort((a, b) => b.tokens - a.tokens);
|
|
2111
|
+
const totalTokens = filtered.reduce((s, m) => s + m.tokens, 0);
|
|
2112
|
+
const allTokens = visible.reduce((s, m) => s + m.tokens, 0);
|
|
2113
|
+
const header = toolFilter ? `UNCOMPRESSED \u2014 ${toolFilter}: ${formatTokens(totalTokens)} | ${filtered.length} msgs | ${pct(totalTokens, allTokens)}% of visible` : `UNCOMPRESSED \u2014 ${formatTokens(totalTokens)} | ${filtered.length} msgs`;
|
|
2114
|
+
const lines = [header, `Sorted by ${sort}`, ""];
|
|
2115
|
+
const shown = filtered.slice(0, limit);
|
|
2116
|
+
for (const message of shown) {
|
|
2117
|
+
lines.push(` ${message.ref} (${formatTokens(message.tokens)}) ${message.tool}`);
|
|
2118
|
+
}
|
|
2119
|
+
if (filtered.length > shown.length) {
|
|
2120
|
+
lines.push("");
|
|
2121
|
+
lines.push(`${shown.length} of ${filtered.length} shown.`);
|
|
2122
|
+
}
|
|
2123
|
+
return lines.join("\n");
|
|
2124
|
+
}
|
|
2125
|
+
function renderCompressedDrilldown(blocks, state, sort, limit, countTokens) {
|
|
2126
|
+
let sorted = [...blocks];
|
|
2127
|
+
if (sort === "time") sorted.sort((a, b) => a.createdAt - b.createdAt);
|
|
2128
|
+
else if (sort === "age") sorted.sort((a, b) => b.survivedCount - a.survivedCount);
|
|
2129
|
+
else
|
|
2130
|
+
sorted.sort(
|
|
2131
|
+
(a, b) => effectiveCompressedTokens(b, state, countTokens) - effectiveCompressedTokens(a, state, countTokens) || b.createdAt - a.createdAt
|
|
2132
|
+
);
|
|
2133
|
+
const totalSummary = sorted.reduce((s, b) => s + summaryTokensOf(b, countTokens), 0);
|
|
2134
|
+
const totalEffective = sorted.reduce(
|
|
2135
|
+
(s, b) => s + effectiveCompressedTokens(b, state, countTokens),
|
|
2136
|
+
0
|
|
2137
|
+
);
|
|
2138
|
+
const lines = [
|
|
2139
|
+
`COMPRESSED \u2014 ${sorted.length} blocks | ${formatTokens(totalEffective)} original \u2192 ${formatTokens(totalSummary)} summary`
|
|
2140
|
+
];
|
|
2141
|
+
const breakdown = tierBreakdown(sorted, countTokens);
|
|
2142
|
+
if (breakdown) lines.push(`Tier usage: ${breakdown}`);
|
|
2143
|
+
lines.push("");
|
|
2144
|
+
const shown = sorted.slice(0, limit);
|
|
2145
|
+
for (const block of shown) {
|
|
2146
|
+
const nested = block.directBlockIds.length > 0 ? ` nested=[${block.directBlockIds.join(",")}]` : "";
|
|
2147
|
+
const topic = block.topic ?? "(no topic)";
|
|
2148
|
+
const eff = effectiveCompressedTokens(block, state, countTokens);
|
|
2149
|
+
lines.push(
|
|
2150
|
+
` ${block.blockId} (${tierLabel(block)}) ${formatTokens(eff)}\u2192${formatTokens(summaryTokensOf(block, countTokens))} ${block.effectiveMessageIds.length} msgs age=${block.survivedCount} ${block.generation}${nested}`
|
|
2151
|
+
);
|
|
2152
|
+
lines.push(` "${topic}"`);
|
|
2153
|
+
}
|
|
2154
|
+
if (sorted.length > shown.length) {
|
|
2155
|
+
lines.push("");
|
|
2156
|
+
lines.push(`${shown.length} of ${sorted.length} shown.`);
|
|
2157
|
+
}
|
|
2158
|
+
return lines.join("\n");
|
|
2159
|
+
}
|
|
2160
|
+
function buildRecap(state, blockId) {
|
|
2161
|
+
const activeBlocks2 = state.blocks.filter((b) => b.active).sort((a, b) => numericPart3(a.blockId) - numericPart3(b.blockId));
|
|
2162
|
+
if (blockId !== void 0) {
|
|
2163
|
+
const block = state.blocks.find((b) => b.blockId === blockId);
|
|
2164
|
+
if (!block) {
|
|
2165
|
+
const activeList = activeBlocks2.map((b) => b.blockId).join(", ");
|
|
2166
|
+
return `Block ${blockId} not found. Active blocks: ${activeList}`;
|
|
2167
|
+
}
|
|
2168
|
+
if (!block.active) {
|
|
2169
|
+
return `Block ${blockId} is inactive (deactivated by nested compression).`;
|
|
2170
|
+
}
|
|
2171
|
+
const range = `${block.effectiveMessageIds.length} messages`;
|
|
2172
|
+
return `[Compressed conversation section]
|
|
2173
|
+
${block.summary}
|
|
2174
|
+
|
|
2175
|
+
[${blockId} | ${range} | topic: "${block.topic ?? "(none)"}"]`;
|
|
2176
|
+
}
|
|
2177
|
+
if (activeBlocks2.length === 0) return "No active compression blocks.";
|
|
2178
|
+
const lines = [`Active compression blocks (${activeBlocks2.length}):`];
|
|
2179
|
+
for (const block of activeBlocks2) {
|
|
2180
|
+
const range = `${block.effectiveMessageIds.length} messages`;
|
|
2181
|
+
const preview = block.summary.slice(0, 200);
|
|
2182
|
+
lines.push(`
|
|
2183
|
+
${block.blockId} | ${range} | "${block.topic ?? "(none)"}"`);
|
|
2184
|
+
lines.push(` ${preview}${block.summary.length > 200 ? "..." : ""}`);
|
|
2185
|
+
}
|
|
2186
|
+
lines.push(`
|
|
2187
|
+
Call with blockId to get the full summary.`);
|
|
2188
|
+
return lines.join("\n");
|
|
2189
|
+
}
|
|
2190
|
+
|
|
2191
|
+
// src/rebuild.ts
|
|
2192
|
+
function rebuildCompressionState(state, messages, config, ports = {}) {
|
|
2193
|
+
const core = createCore({ countTokens: ports.countTokens ?? defaultCountTokens });
|
|
2194
|
+
const refResult = assignRefs(messages, {
|
|
2195
|
+
existing: state.messageRefs,
|
|
2196
|
+
nextIndex: highestUsedIndex(state.messageRefs) + 1
|
|
2197
|
+
});
|
|
2198
|
+
let working = { ...state, messageRefs: refResult.map };
|
|
2199
|
+
const invocations = collectCompressInvocations(messages);
|
|
2200
|
+
let blocksRebuilt = 0;
|
|
2201
|
+
for (const invocation of invocations) {
|
|
2202
|
+
const ranges = extractRanges(invocation.input, invocation.callId);
|
|
2203
|
+
if (ranges.length === 0) continue;
|
|
2204
|
+
const result = core.applyCompression({ ranges, messages, state: working, config });
|
|
2205
|
+
working = result.state;
|
|
2206
|
+
blocksRebuilt += result.result.blocksCreated;
|
|
2207
|
+
}
|
|
2208
|
+
return { state: working, blocksRebuilt };
|
|
2209
|
+
}
|
|
2210
|
+
function collectCompressInvocations(messages) {
|
|
2211
|
+
const invocations = [];
|
|
2212
|
+
for (const message of messages) {
|
|
2213
|
+
if (message.toolName !== "compress" || message.contentType !== "tool-call") continue;
|
|
2214
|
+
let input;
|
|
2215
|
+
try {
|
|
2216
|
+
input = JSON.parse(message.text ?? "");
|
|
2217
|
+
} catch {
|
|
2218
|
+
continue;
|
|
2219
|
+
}
|
|
2220
|
+
invocations.push({ callId: message.toolCallId, input });
|
|
2221
|
+
}
|
|
2222
|
+
return invocations;
|
|
2223
|
+
}
|
|
2224
|
+
function extractRanges(input, callId) {
|
|
2225
|
+
const content = input?.content;
|
|
2226
|
+
if (!Array.isArray(content)) return [];
|
|
2227
|
+
const ranges = [];
|
|
2228
|
+
for (const entry of content) {
|
|
2229
|
+
if (!entry || typeof entry !== "object") continue;
|
|
2230
|
+
const e = entry;
|
|
2231
|
+
if (typeof e.summary !== "string") continue;
|
|
2232
|
+
const start = e.startId ?? e.messageId;
|
|
2233
|
+
const end = e.endId ?? e.messageId;
|
|
2234
|
+
if (typeof start !== "string" || typeof end !== "string") continue;
|
|
2235
|
+
ranges.push({
|
|
2236
|
+
startRef: start,
|
|
2237
|
+
endRef: end,
|
|
2238
|
+
summary: e.summary,
|
|
2239
|
+
topic: typeof e.topic === "string" ? e.topic : void 0,
|
|
2240
|
+
compressCallId: callId
|
|
2241
|
+
});
|
|
2242
|
+
}
|
|
2243
|
+
return ranges;
|
|
2244
|
+
}
|
|
2245
|
+
export {
|
|
2246
|
+
BLOCKED_REF,
|
|
2247
|
+
COMPRESS_PHILOSOPHY,
|
|
2248
|
+
HOW_TO_COMPRESS_RULES,
|
|
2249
|
+
SUMMARY_HEADER,
|
|
2250
|
+
TIER2_DISTILL_RULES,
|
|
2251
|
+
TIER3_CONDENSE_RULES,
|
|
2252
|
+
activeBlocks,
|
|
2253
|
+
advanceSurvival,
|
|
2254
|
+
allocateBlockId,
|
|
2255
|
+
allocateRunId,
|
|
2256
|
+
applyMessageFilters,
|
|
2257
|
+
assignRefs,
|
|
2258
|
+
blockById,
|
|
2259
|
+
buildRecap,
|
|
2260
|
+
buildRestoredContentPreview,
|
|
2261
|
+
buildStatusReport,
|
|
2262
|
+
clearMessageFilters,
|
|
2263
|
+
collectOldGenBlocks,
|
|
2264
|
+
coveredMessageIds,
|
|
2265
|
+
createBpeTokenizer,
|
|
2266
|
+
createCore,
|
|
2267
|
+
createInitialState,
|
|
2268
|
+
deactivateBlock,
|
|
2269
|
+
defaultConfig,
|
|
2270
|
+
defaultCountTokens,
|
|
2271
|
+
emptyRefMap,
|
|
2272
|
+
estimateTokensFast,
|
|
2273
|
+
findActiveAncestor,
|
|
2274
|
+
findBlocksOverlappingMessages,
|
|
2275
|
+
getMessageFilter,
|
|
2276
|
+
hideConsumedCompressCalls,
|
|
2277
|
+
highestActiveTier,
|
|
2278
|
+
highestUsedIndex,
|
|
2279
|
+
indexToRef,
|
|
2280
|
+
isMessageProtected,
|
|
2281
|
+
listMessageFilters,
|
|
2282
|
+
makeIO,
|
|
2283
|
+
matchToolPattern,
|
|
2284
|
+
mergeMarkedBlocks,
|
|
2285
|
+
parseBlockIdArg,
|
|
2286
|
+
parseBoundary,
|
|
2287
|
+
prune,
|
|
2288
|
+
rawForRef,
|
|
2289
|
+
rebuildCompressionState,
|
|
2290
|
+
refForRaw,
|
|
2291
|
+
refToIndex,
|
|
2292
|
+
registerMessageFilter,
|
|
2293
|
+
renderNudgeText,
|
|
2294
|
+
renderRefsNode,
|
|
2295
|
+
renderVisibleRefs,
|
|
2296
|
+
resolveBoundaries,
|
|
2297
|
+
resolveKeepMarkers,
|
|
2298
|
+
runPipeline,
|
|
2299
|
+
syncBlocks,
|
|
2300
|
+
truncateLargeToolOutputs,
|
|
2301
|
+
validateConfig
|
|
2302
|
+
};
|
|
2303
|
+
//# sourceMappingURL=index.js.map
|