acp-kernel 0.0.16 → 0.0.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +31 -0
- package/dist/compress.d.ts +14 -0
- package/dist/compress.d.ts.map +1 -1
- package/dist/hide-consumed.d.ts.map +1 -1
- package/dist/index.d.ts +2 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +109 -49
- package/dist/index.js.map +1 -1
- package/dist/recommend.d.ts +2 -2
- package/dist/recommend.d.ts.map +1 -1
- package/dist/render-refs.d.ts +16 -1
- package/dist/render-refs.d.ts.map +1 -1
- package/dist/types.d.ts +2 -0
- package/dist/types.d.ts.map +1 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -54,6 +54,37 @@ core.search("auth token", compressed); // relevance-ranked block search
|
|
|
54
54
|
core.status(compressed, tokenCount, config); // context-usage report
|
|
55
55
|
```
|
|
56
56
|
|
|
57
|
+
#### `renderTags` — host-side rendering strategy
|
|
58
|
+
|
|
59
|
+
`processTurn` always assigns a ref (e.g. `m00005`) to **every** mapped message,
|
|
60
|
+
including tool calls — that ref map is the anchor the model cites inside
|
|
61
|
+
`compress` calls. What varies is whether the `<acp>` tag is also **injected into
|
|
62
|
+
message text**. Pick the strategy that matches how your host consumes the
|
|
63
|
+
output:
|
|
64
|
+
|
|
65
|
+
```ts
|
|
66
|
+
import { processTurn, type RenderStrategy } from "acp-kernel";
|
|
67
|
+
|
|
68
|
+
type RenderStrategy = "all" | "text-only" | "none";
|
|
69
|
+
|
|
70
|
+
// default: every mapped message gets a visible <acp> tag.
|
|
71
|
+
// Use for in-process hosts (pai-acp / billion-context-pi) where the LLM
|
|
72
|
+
// reads the tag to identify compress ranges.
|
|
73
|
+
processTurn({ messages, state, config, tokenCount });
|
|
74
|
+
|
|
75
|
+
// proxy mode: tag user/assistant text, but leave tool-call args and tool
|
|
76
|
+
// results pristine (a <acp> tag inside {"command":"echo"} corrupts the JSON).
|
|
77
|
+
// Refs are still assigned to every message; only text rendering is selective.
|
|
78
|
+
processTurn({ messages, state, config, tokenCount, renderTags: "text-only" });
|
|
79
|
+
|
|
80
|
+
// host reads the ref map directly and never wants tags in the text.
|
|
81
|
+
// processTurn honors this by omitting the render-refs node entirely.
|
|
82
|
+
processTurn({ messages, state, config, tokenCount, renderTags: "none" });
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
`renderTags` is optional and defaults to `"all"`, so existing call sites keep
|
|
86
|
+
working unchanged.
|
|
87
|
+
|
|
57
88
|
### Standalone modules
|
|
58
89
|
|
|
59
90
|
| Module | Purpose |
|
package/dist/compress.d.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { createInitialState } from "./state.js";
|
|
2
|
+
import type { RenderStrategy } from "./render-refs.js";
|
|
2
3
|
import { type PipelineNode } from "./pipeline.js";
|
|
3
4
|
import type { ApplyCompressionResult, CompressionBlock, CompressionState, Config, CoreMessage, ProcessTurnResult, StatusReport } from "./types.js";
|
|
4
5
|
export interface Ports {
|
|
@@ -17,6 +18,19 @@ export interface ProcessTurnInput {
|
|
|
17
18
|
state: CompressionState;
|
|
18
19
|
config: Config;
|
|
19
20
|
tokenCount: number;
|
|
21
|
+
/**
|
|
22
|
+
* Which messages get an <acp> ref tag injected into their text
|
|
23
|
+
* (the render-refs pipeline node). Refs are ALWAYS assigned regardless
|
|
24
|
+
* (assign-refs node runs unconditionally).
|
|
25
|
+
* - "all" (default): tag every mapped message — in-process hosts
|
|
26
|
+
* like pai-acp want tags for the LLM to reference compress ranges.
|
|
27
|
+
* - "text-only": tag only user/assistant text; leave tool-call args
|
|
28
|
+
* and tool-result content pristine — proxy hosts where structured
|
|
29
|
+
* content must not be polluted.
|
|
30
|
+
* - "none": leave all text untouched — hosts that read the ref map
|
|
31
|
+
* directly from result.state.messageRefs.
|
|
32
|
+
*/
|
|
33
|
+
renderTags?: RenderStrategy;
|
|
20
34
|
}
|
|
21
35
|
export interface ApplyCompressionInput {
|
|
22
36
|
ranges: {
|
package/dist/compress.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"compress.d.ts","sourceRoot":"","sources":["../src/compress.ts"],"names":[],"mappings":"AAIA,OAAO,EAGL,kBAAkB,EACnB,MAAM,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"compress.d.ts","sourceRoot":"","sources":["../src/compress.ts"],"names":[],"mappings":"AAIA,OAAO,EAGL,kBAAkB,EACnB,MAAM,YAAY,CAAC;AAQpB,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAOvD,OAAO,EAGL,KAAK,YAAY,EAElB,MAAM,eAAe,CAAC;AACvB,OAAO,KAAK,EACV,sBAAsB,EACtB,gBAAgB,EAChB,gBAAgB,EAEhB,MAAM,EAEN,WAAW,EAGX,iBAAiB,EAEjB,YAAY,EACb,MAAM,YAAY,CAAC;AAEpB,MAAM,WAAW,KAAK;IACpB,WAAW,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM,CAAC;CACxC;AAED,MAAM,WAAW,eAAe;IAC9B,WAAW,CAAC,KAAK,EAAE,gBAAgB,GAAG,iBAAiB,CAAC;IACxD,gBAAgB,CAAC,KAAK,EAAE,qBAAqB,GAAG,sBAAsB,CAAC;IACvE,YAAY,IAAI,YAAY,EAAE,CAAC;IAC/B,UAAU,CACR,OAAO,EAAE,MAAM,EACf,KAAK,EAAE,gBAAgB,GACtB,gBAAgB,GAAG,SAAS,CAAC;IAChC,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,gBAAgB,GAAG,gBAAgB,EAAE,CAAC;IACnE,MAAM,CACJ,KAAK,EAAE,gBAAgB,EACvB,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,GACb,YAAY,CAAC;CACjB;AAED,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,EAAE,WAAW,EAAE,CAAC;IACxB,KAAK,EAAE,gBAAgB,CAAC;IACxB,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB;;;;;;;;;;;OAWG;IACH,UAAU,CAAC,EAAE,cAAc,CAAC;CAC7B;AAED,MAAM,WAAW,qBAAqB;IACpC,MAAM,EAAE;QACN,QAAQ,EAAE,MAAM,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,OAAO,EAAE,MAAM,CAAC;QAChB,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,cAAc,CAAC,EAAE,MAAM,CAAC;QACxB,eAAe,CAAC,EAAE,MAAM,CAAC;KAC1B,EAAE,CAAC;IACJ,QAAQ,EAAE,WAAW,EAAE,CAAC;IACxB,KAAK,EAAE,gBAAgB,CAAC;IACxB,MAAM,EAAE,MAAM,CAAC;IACf,mBAAmB,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;CACnC;AAED,wBAAgB,UAAU,CAAC,KAAK,GAAE,KAAU,GAAG,eAAe,CAoO7D;AAurBD,OAAO,EAAE,kBAAkB,EAAE,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"hide-consumed.d.ts","sourceRoot":"","sources":["../src/hide-consumed.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"hide-consumed.d.ts","sourceRoot":"","sources":["../src/hide-consumed.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAIhE,MAAM,WAAW,kBAAkB;IAC/B,QAAQ,EAAE,WAAW,EAAE,CAAC;IACxB,MAAM,EAAE,MAAM,CAAC;CAClB;AA8BD,wBAAgB,yBAAyB,CACrC,KAAK,EAAE,gBAAgB,EACvB,QAAQ,EAAE,WAAW,EAAE,GACxB,kBAAkB,CAmFpB"}
|
package/dist/index.d.ts
CHANGED
|
@@ -22,7 +22,8 @@ export { hideConsumedCompressCalls } from "./hide-consumed.js";
|
|
|
22
22
|
export type { HideConsumedResult } from "./hide-consumed.js";
|
|
23
23
|
export { rebuildCompressionState } from "./rebuild.js";
|
|
24
24
|
export type { RebuildResult, RebuildPorts } from "./rebuild.js";
|
|
25
|
-
export { renderVisibleRefs, renderRefsNode } from "./render-refs.js";
|
|
25
|
+
export { renderVisibleRefs, renderRefsNode, createRenderRefsNode } from "./render-refs.js";
|
|
26
|
+
export type { RenderStrategy } from "./render-refs.js";
|
|
26
27
|
export { searchBlocks, searchBlocksAsync, blockDocs, messageDocs } from "./search.js";
|
|
27
28
|
export type { SearchResult, SearchOptions, SearchAlgorithm, AsyncSearchAlgorithm, AnySearchAlgorithm, SearchDoc, SearchDocKind, ScoredBlock, MessageRole, RoleWeights, MessageInput } from "./search.js";
|
|
28
29
|
export { DEFAULT_ALGORITHM, DEFAULT_ROLE_WEIGHTS, registerSearchAlgorithm, getSearchAlgorithm, listSearchAlgorithms } from "./search.js";
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,YAAY,CAAC;AAC3B,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAC3C,YAAY,EACV,KAAK,EACL,eAAe,EACf,gBAAgB,EAChB,qBAAqB,GACtB,MAAM,eAAe,CAAC;AACvB,OAAO,EACL,kBAAkB,EAClB,eAAe,EACf,aAAa,EACb,SAAS,EACT,YAAY,EACZ,iBAAiB,EACjB,iBAAiB,EACjB,eAAe,GAChB,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAC5D,OAAO,EACL,UAAU,EACV,gBAAgB,EAChB,WAAW,EACX,UAAU,EACV,UAAU,EACV,SAAS,EACT,SAAS,EACT,WAAW,GACZ,MAAM,WAAW,CAAC;AACnB,OAAO,EAAE,KAAK,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AACnD,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AACvC,OAAO,EAAE,iBAAiB,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AACnE,OAAO,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AAC3F,YAAY,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAClD,OAAO,EAAE,eAAe,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAChE,YAAY,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AACjE,OAAO,EAAE,mBAAmB,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,MAAM,wBAAwB,CAAC;AAC/H,OAAO,EAAE,wBAAwB,EAAE,MAAM,qBAAqB,CAAC;AAC/D,YAAY,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAC3E,OAAO,EACH,eAAe,EACf,6BAA6B,EAC7B,kBAAkB,EAClB,eAAe,EACf,2BAA2B,EAC3B,mBAAmB,GACtB,MAAM,iBAAiB,CAAC;AACzB,YAAY,EAAE,iBAAiB,EAAE,sBAAsB,EAAE,qBAAqB,EAAE,MAAM,iBAAiB,CAAC;AACxG,OAAO,EAAE,iBAAiB,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAC5D,YAAY,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AACvD,OAAO,EAAE,yBAAyB,EAAE,MAAM,oBAAoB,CAAC;AAC/D,YAAY,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAC7D,OAAO,EAAE,uBAAuB,EAAE,MAAM,cAAc,CAAC;AACvD,YAAY,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAChE,OAAO,EAAE,iBAAiB,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,YAAY,CAAC;AAC3B,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAC3C,YAAY,EACV,KAAK,EACL,eAAe,EACf,gBAAgB,EAChB,qBAAqB,GACtB,MAAM,eAAe,CAAC;AACvB,OAAO,EACL,kBAAkB,EAClB,eAAe,EACf,aAAa,EACb,SAAS,EACT,YAAY,EACZ,iBAAiB,EACjB,iBAAiB,EACjB,eAAe,GAChB,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAC5D,OAAO,EACL,UAAU,EACV,gBAAgB,EAChB,WAAW,EACX,UAAU,EACV,UAAU,EACV,SAAS,EACT,SAAS,EACT,WAAW,GACZ,MAAM,WAAW,CAAC;AACnB,OAAO,EAAE,KAAK,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AACnD,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AACvC,OAAO,EAAE,iBAAiB,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AACnE,OAAO,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AAC3F,YAAY,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAClD,OAAO,EAAE,eAAe,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAChE,YAAY,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AACjE,OAAO,EAAE,mBAAmB,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,MAAM,wBAAwB,CAAC;AAC/H,OAAO,EAAE,wBAAwB,EAAE,MAAM,qBAAqB,CAAC;AAC/D,YAAY,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAC3E,OAAO,EACH,eAAe,EACf,6BAA6B,EAC7B,kBAAkB,EAClB,eAAe,EACf,2BAA2B,EAC3B,mBAAmB,GACtB,MAAM,iBAAiB,CAAC;AACzB,YAAY,EAAE,iBAAiB,EAAE,sBAAsB,EAAE,qBAAqB,EAAE,MAAM,iBAAiB,CAAC;AACxG,OAAO,EAAE,iBAAiB,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAC5D,YAAY,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AACvD,OAAO,EAAE,yBAAyB,EAAE,MAAM,oBAAoB,CAAC;AAC/D,YAAY,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAC7D,OAAO,EAAE,uBAAuB,EAAE,MAAM,cAAc,CAAC;AACvD,YAAY,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAChE,OAAO,EAAE,iBAAiB,EAAE,cAAc,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AAC3F,YAAY,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AACvD,OAAO,EAAE,YAAY,EAAE,iBAAiB,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AACtF,YAAY,EAAE,YAAY,EAAE,aAAa,EAAE,eAAe,EAAE,oBAAoB,EAAE,kBAAkB,EAAE,SAAS,EAAE,aAAa,EAAE,WAAW,EAAE,WAAW,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AACzM,OAAO,EAAE,iBAAiB,EAAE,oBAAoB,EAAE,uBAAuB,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AACzI,OAAO,EAAE,kBAAkB,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AACtE,OAAO,EACL,WAAW,EACX,MAAM,EACN,KAAK,YAAY,EACjB,KAAK,eAAe,EACpB,KAAK,MAAM,EACX,KAAK,WAAW,GACjB,MAAM,eAAe,CAAC;AACvB,cAAc,mBAAmB,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -506,14 +506,49 @@ function truncateLargeToolOutputs(messages, tokenCount, config, countTokens, opt
|
|
|
506
506
|
|
|
507
507
|
// src/hide-consumed.ts
|
|
508
508
|
var KEEP_LAST_ORPHANED = 0;
|
|
509
|
+
function rangeKey(startRef, endRef) {
|
|
510
|
+
return `${startRef}::${endRef}`;
|
|
511
|
+
}
|
|
512
|
+
function rewriteCompressText(text, liveKeys) {
|
|
513
|
+
let parsed;
|
|
514
|
+
try {
|
|
515
|
+
parsed = JSON.parse(text ?? "");
|
|
516
|
+
} catch {
|
|
517
|
+
return null;
|
|
518
|
+
}
|
|
519
|
+
if (!parsed || typeof parsed !== "object") return null;
|
|
520
|
+
const obj = parsed;
|
|
521
|
+
const content = obj.content;
|
|
522
|
+
if (!Array.isArray(content) || content.length === 0) return null;
|
|
523
|
+
const kept = content.filter((entry) => {
|
|
524
|
+
if (!entry || typeof entry !== "object") return false;
|
|
525
|
+
const s = typeof entry.startId === "string" ? entry.startId : typeof entry.messageId === "string" ? entry.messageId : "";
|
|
526
|
+
const e = typeof entry.endId === "string" ? entry.endId : typeof entry.messageId === "string" ? entry.messageId : "";
|
|
527
|
+
return liveKeys.has(rangeKey(s, e));
|
|
528
|
+
});
|
|
529
|
+
if (kept.length === content.length || kept.length === 0) return null;
|
|
530
|
+
return JSON.stringify({ ...obj, content: kept });
|
|
531
|
+
}
|
|
509
532
|
function hideConsumedCompressCalls(state, messages) {
|
|
510
|
-
const activeCallIds = /* @__PURE__ */ new Set();
|
|
511
533
|
const allBlockCallIds = /* @__PURE__ */ new Set();
|
|
534
|
+
const activeCallIds = /* @__PURE__ */ new Set();
|
|
535
|
+
const liveRangeKeysByCallId = /* @__PURE__ */ new Map();
|
|
536
|
+
const legacyLiveByCallId = /* @__PURE__ */ new Set();
|
|
512
537
|
for (const block of state.blocks) {
|
|
513
|
-
if (block.compressCallId)
|
|
514
|
-
|
|
515
|
-
|
|
538
|
+
if (!block.compressCallId) continue;
|
|
539
|
+
allBlockCallIds.add(block.compressCallId);
|
|
540
|
+
if (!block.active) continue;
|
|
541
|
+
activeCallIds.add(block.compressCallId);
|
|
542
|
+
if (block.startRef === void 0 || block.endRef === void 0) {
|
|
543
|
+
legacyLiveByCallId.add(block.compressCallId);
|
|
544
|
+
continue;
|
|
516
545
|
}
|
|
546
|
+
let keys = liveRangeKeysByCallId.get(block.compressCallId);
|
|
547
|
+
if (!keys) {
|
|
548
|
+
keys = /* @__PURE__ */ new Set();
|
|
549
|
+
liveRangeKeysByCallId.set(block.compressCallId, keys);
|
|
550
|
+
}
|
|
551
|
+
keys.add(rangeKey(block.startRef, block.endRef));
|
|
517
552
|
}
|
|
518
553
|
const lastOrphanedCallIds = [];
|
|
519
554
|
for (let i = messages.length - 1; i >= 0 && lastOrphanedCallIds.length < KEEP_LAST_ORPHANED; i--) {
|
|
@@ -542,6 +577,16 @@ function hideConsumedCompressCalls(state, messages) {
|
|
|
542
577
|
hidden++;
|
|
543
578
|
continue;
|
|
544
579
|
}
|
|
580
|
+
if (message.toolName === "compress" && message.contentType === "tool-call" && message.toolCallId && keepCallIds.has(message.toolCallId)) {
|
|
581
|
+
const liveKeys = liveRangeKeysByCallId.get(message.toolCallId);
|
|
582
|
+
if (liveKeys && liveKeys.size > 0 && !legacyLiveByCallId.has(message.toolCallId)) {
|
|
583
|
+
const rewritten = rewriteCompressText(message.text, liveKeys);
|
|
584
|
+
if (rewritten !== null) {
|
|
585
|
+
result.push({ ...message, text: rewritten });
|
|
586
|
+
continue;
|
|
587
|
+
}
|
|
588
|
+
}
|
|
589
|
+
}
|
|
545
590
|
result.push(message);
|
|
546
591
|
}
|
|
547
592
|
return { messages: result, hidden };
|
|
@@ -675,9 +720,13 @@ var TAG_CLOSE = LT + "/acp" + GT;
|
|
|
675
720
|
function acpTag(ref, tokens, type) {
|
|
676
721
|
return TAG_OPEN + 'tokens="' + formatTokens(tokens) + '" type="' + type + '"' + GT + ref + TAG_CLOSE;
|
|
677
722
|
}
|
|
678
|
-
function renderMessage(message, map, countTokens) {
|
|
723
|
+
function renderMessage(message, map, countTokens, strategy) {
|
|
679
724
|
const ref = refForRaw(map, message.id);
|
|
680
725
|
if (!ref || ref === BLOCKED_REF) return message;
|
|
726
|
+
if (strategy === "none") return message;
|
|
727
|
+
if (strategy === "text-only" && message.contentType !== "text") {
|
|
728
|
+
return message;
|
|
729
|
+
}
|
|
681
730
|
const ownTagRe = new RegExp(
|
|
682
731
|
"^" + escapeRegex(TAG_OPEN) + "[^>]*" + GT + escapeRegex(ref) + escapeRegex(TAG_CLOSE) + "\\n?"
|
|
683
732
|
);
|
|
@@ -688,21 +737,24 @@ function renderMessage(message, map, countTokens) {
|
|
|
688
737
|
if (!cleanText) return { ...message, text: prefix };
|
|
689
738
|
return { ...message, text: prefix + cleanText };
|
|
690
739
|
}
|
|
691
|
-
function renderVisibleRefs(messages, state, countTokens = (text) => Math.ceil(text.length / 4)) {
|
|
740
|
+
function renderVisibleRefs(messages, state, countTokens = (text) => Math.ceil(text.length / 4), strategy = "all") {
|
|
692
741
|
const map = state.messageRefs;
|
|
693
742
|
return messages.map(
|
|
694
|
-
(message) => renderMessage(message, map, countTokens)
|
|
743
|
+
(message) => renderMessage(message, map, countTokens, strategy)
|
|
695
744
|
);
|
|
696
745
|
}
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
}
|
|
746
|
+
function createRenderRefsNode(strategy) {
|
|
747
|
+
return {
|
|
748
|
+
name: "render-refs",
|
|
749
|
+
run(io, ctx) {
|
|
750
|
+
return {
|
|
751
|
+
...io,
|
|
752
|
+
messages: renderVisibleRefs(io.messages, io.state, ctx.countTokens, strategy)
|
|
753
|
+
};
|
|
754
|
+
}
|
|
755
|
+
};
|
|
756
|
+
}
|
|
757
|
+
var renderRefsNode = createRenderRefsNode("all");
|
|
706
758
|
|
|
707
759
|
// src/protected.ts
|
|
708
760
|
var ALWAYS_PROTECTED_TOOLS = ["compress"];
|
|
@@ -795,8 +847,8 @@ function refNum(ref) {
|
|
|
795
847
|
const n = parseInt(ref.slice(1), 10);
|
|
796
848
|
return Number.isNaN(n) ? -1 : n;
|
|
797
849
|
}
|
|
798
|
-
function
|
|
799
|
-
return Math.ceil(
|
|
850
|
+
function estimateTextTokens(text) {
|
|
851
|
+
return Math.ceil(text.length / 4);
|
|
800
852
|
}
|
|
801
853
|
function isToolMessage(message) {
|
|
802
854
|
return message.contentType === "tool-call" || message.contentType === "tool-result";
|
|
@@ -808,7 +860,7 @@ function isSyntheticOrPruned(message, state) {
|
|
|
808
860
|
}
|
|
809
861
|
return false;
|
|
810
862
|
}
|
|
811
|
-
function computeProtectedRefs(messages, state, config) {
|
|
863
|
+
function computeProtectedRefs(messages, state, config, countTokens = estimateTextTokens) {
|
|
812
864
|
const preserveN = config.preserveRecentMessages;
|
|
813
865
|
const preserveTokens = config.preserveRecentTokens;
|
|
814
866
|
const result = /* @__PURE__ */ new Set();
|
|
@@ -818,7 +870,7 @@ function computeProtectedRefs(messages, state, config) {
|
|
|
818
870
|
if (isNeverPreserveRecent(msg)) continue;
|
|
819
871
|
const ref = state.messageRefs.byRaw[msg.id];
|
|
820
872
|
if (!ref || ref === "BLOCKED") continue;
|
|
821
|
-
visible.push({ ref, tokens:
|
|
873
|
+
visible.push({ ref, tokens: countTokens(msg.text ?? "") });
|
|
822
874
|
}
|
|
823
875
|
if (preserveN > 0) {
|
|
824
876
|
for (const m of visible.slice(-preserveN)) {
|
|
@@ -843,7 +895,7 @@ function computeProtectedRefs(messages, state, config) {
|
|
|
843
895
|
}
|
|
844
896
|
return result;
|
|
845
897
|
}
|
|
846
|
-
function buildCompressibleRanges(messages, state, config, protectedZoneRefs) {
|
|
898
|
+
function buildCompressibleRanges(messages, state, config, protectedZoneRefs, countTokens = estimateTextTokens) {
|
|
847
899
|
const compressibleMsgs = [];
|
|
848
900
|
const protectedMsgs = [];
|
|
849
901
|
const protectedCallIds = collectProtectedToolCallIds(messages, config);
|
|
@@ -856,7 +908,7 @@ function buildCompressibleRanges(messages, state, config, protectedZoneRefs) {
|
|
|
856
908
|
protectedMsgs.push({
|
|
857
909
|
ref,
|
|
858
910
|
refNum: rn,
|
|
859
|
-
tokens:
|
|
911
|
+
tokens: countTokens(msg.text ?? ""),
|
|
860
912
|
tools: msg.toolName ? [msg.toolName] : []
|
|
861
913
|
});
|
|
862
914
|
continue;
|
|
@@ -867,7 +919,7 @@ function buildCompressibleRanges(messages, state, config, protectedZoneRefs) {
|
|
|
867
919
|
compressibleMsgs.push({
|
|
868
920
|
ref,
|
|
869
921
|
refNum: rn,
|
|
870
|
-
tokens:
|
|
922
|
+
tokens: countTokens(msg.text ?? ""),
|
|
871
923
|
isTool: isToolMessage(msg),
|
|
872
924
|
isUser: msg.role === "user"
|
|
873
925
|
});
|
|
@@ -961,7 +1013,7 @@ function createCore(ports = {}) {
|
|
|
961
1013
|
let tokensCompressed = 0;
|
|
962
1014
|
const errors = [];
|
|
963
1015
|
const warnings = [];
|
|
964
|
-
const protectedMessageIds = input.protectedMessageIds ?? computeProtectedRefs(input.messages, input.state, input.config);
|
|
1016
|
+
const protectedMessageIds = input.protectedMessageIds ?? computeProtectedRefs(input.messages, input.state, input.config, countTokens);
|
|
965
1017
|
const preExistingCoverage = collectCoverage(state);
|
|
966
1018
|
const rangeIndexSets = [];
|
|
967
1019
|
for (const spec of input.ranges) {
|
|
@@ -986,29 +1038,25 @@ function createCore(ports = {}) {
|
|
|
986
1038
|
const bMin = b.indices.length > 0 ? Math.min(...b.indices) : Infinity;
|
|
987
1039
|
return aMin - bMin;
|
|
988
1040
|
});
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
const
|
|
993
|
-
const
|
|
994
|
-
if (
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
errors: [
|
|
1001
|
-
`content: range (${prev.spec.startRef}..${prev.spec.endRef}) overlaps (${curr.spec.startRef}..${curr.spec.endRef}). Overlapping ranges cannot be compressed in the same batch.`
|
|
1002
|
-
],
|
|
1003
|
-
warnings: []
|
|
1004
|
-
}
|
|
1005
|
-
};
|
|
1041
|
+
const skipSpecs = /* @__PURE__ */ new Set();
|
|
1042
|
+
let acceptedMaxIndex = -1;
|
|
1043
|
+
for (const entry of sortedRanges) {
|
|
1044
|
+
const entryMax = entry.indices.length > 0 ? Math.max(...entry.indices) : -1;
|
|
1045
|
+
const entryMin = entry.indices.length > 0 ? Math.min(...entry.indices) : -1;
|
|
1046
|
+
if (entryMin >= 0 && entryMin <= acceptedMaxIndex) {
|
|
1047
|
+
skipSpecs.add(entry.spec);
|
|
1048
|
+
warnings.push(
|
|
1049
|
+
`Skipped range (${entry.spec.startRef}..${entry.spec.endRef}) \u2014 overlaps an earlier range in the batch; the earlier range takes precedence. Keep ranges disjoint.`
|
|
1050
|
+
);
|
|
1051
|
+
continue;
|
|
1006
1052
|
}
|
|
1053
|
+
if (entryMax > acceptedMaxIndex) acceptedMaxIndex = entryMax;
|
|
1007
1054
|
}
|
|
1008
1055
|
if (input.config.compress.minCompressRange > 0 && input.ranges.length > 0) {
|
|
1009
1056
|
let totalRangeChars = 0;
|
|
1010
1057
|
let hasBlockBoundaryRange = false;
|
|
1011
1058
|
for (const spec of input.ranges) {
|
|
1059
|
+
if (skipSpecs.has(spec)) continue;
|
|
1012
1060
|
let resolved;
|
|
1013
1061
|
try {
|
|
1014
1062
|
resolved = resolveBoundaries({
|
|
@@ -1044,6 +1092,7 @@ function createCore(ports = {}) {
|
|
|
1044
1092
|
}
|
|
1045
1093
|
}
|
|
1046
1094
|
for (const spec of input.ranges) {
|
|
1095
|
+
if (skipSpecs.has(spec)) continue;
|
|
1047
1096
|
try {
|
|
1048
1097
|
const outcome = applySingleRange({
|
|
1049
1098
|
spec,
|
|
@@ -1086,7 +1135,9 @@ function createCore(ports = {}) {
|
|
|
1086
1135
|
state: input.state,
|
|
1087
1136
|
effects: {}
|
|
1088
1137
|
};
|
|
1089
|
-
const
|
|
1138
|
+
const strategy = input.renderTags ?? "all";
|
|
1139
|
+
const nodes = buildNodes(strategy);
|
|
1140
|
+
const result = runPipeline(nodes, initial, ctx);
|
|
1090
1141
|
return {
|
|
1091
1142
|
messages: result.messages,
|
|
1092
1143
|
state: result.state,
|
|
@@ -1116,7 +1167,10 @@ function createCore(ports = {}) {
|
|
|
1116
1167
|
};
|
|
1117
1168
|
}
|
|
1118
1169
|
function defaultNodes() {
|
|
1119
|
-
return
|
|
1170
|
+
return buildNodes("all");
|
|
1171
|
+
}
|
|
1172
|
+
function buildNodes(strategy) {
|
|
1173
|
+
const base = [
|
|
1120
1174
|
assignRefsNode,
|
|
1121
1175
|
syncBlocksNode,
|
|
1122
1176
|
pruneNode,
|
|
@@ -1124,9 +1178,10 @@ function createCore(ports = {}) {
|
|
|
1124
1178
|
hideCompressCallsNode,
|
|
1125
1179
|
recommendNode,
|
|
1126
1180
|
nudgeNode,
|
|
1127
|
-
emergencyTruncateNode
|
|
1128
|
-
renderRefsNode
|
|
1181
|
+
emergencyTruncateNode
|
|
1129
1182
|
];
|
|
1183
|
+
if (strategy === "none") return base;
|
|
1184
|
+
return [...base, createRenderRefsNode(strategy)];
|
|
1130
1185
|
}
|
|
1131
1186
|
return { processTurn, applyCompression, defaultNodes, decompress, search, status };
|
|
1132
1187
|
}
|
|
@@ -1178,13 +1233,15 @@ var recommendNode = {
|
|
|
1178
1233
|
const protectedRefs = computeProtectedRefs(
|
|
1179
1234
|
io.messages,
|
|
1180
1235
|
io.state,
|
|
1181
|
-
ctx.config
|
|
1236
|
+
ctx.config,
|
|
1237
|
+
ctx.countTokens
|
|
1182
1238
|
);
|
|
1183
1239
|
const contextRanges = buildCompressibleRanges(
|
|
1184
1240
|
io.messages,
|
|
1185
1241
|
io.state,
|
|
1186
1242
|
ctx.config,
|
|
1187
|
-
protectedRefs
|
|
1243
|
+
protectedRefs,
|
|
1244
|
+
ctx.countTokens
|
|
1188
1245
|
);
|
|
1189
1246
|
const nothingToCompress = contextRanges.compressible.length === 0;
|
|
1190
1247
|
const recommendation = {
|
|
@@ -1362,7 +1419,9 @@ function applySingleRange(input) {
|
|
|
1362
1419
|
survivedCount: 0,
|
|
1363
1420
|
generation: "young",
|
|
1364
1421
|
active: true,
|
|
1365
|
-
compressCallId: input.spec.compressCallId
|
|
1422
|
+
compressCallId: input.spec.compressCallId,
|
|
1423
|
+
startRef: input.spec.startRef,
|
|
1424
|
+
endRef: input.spec.endRef
|
|
1366
1425
|
};
|
|
1367
1426
|
input.state.blocks.push(block);
|
|
1368
1427
|
for (const consumedId of consumedBlockIds) {
|
|
@@ -1857,7 +1916,7 @@ function renderNudgeText(decision) {
|
|
|
1857
1916
|
breakdownStr,
|
|
1858
1917
|
"",
|
|
1859
1918
|
`[TIER ${decision.tier} ${isT2 ? "DISTILLATION" : "CONDENSATION"} TRIGGER]`,
|
|
1860
|
-
isT2 ? `Your tier-1 compression summaries have accumulated. Distill them into a single denser tier-2 summary. Use block IDs as boundaries.` : `Your tier-2 compression summaries have accumulated. Condense them further into a tier-3 ultra-condensed summary. Use block IDs as boundaries.`,
|
|
1919
|
+
isT2 ? `Your tier-1 compression summaries have accumulated. Distill them into a single denser tier-2 summary. Use block IDs as boundaries (startId and endId as bN). Any raw (uncompressed) messages sitting between the boundary blocks are absorbed into the tier-2 block as well \u2014 apply HOW TO COMPRESS to those raw messages and the TIER 2 distillation rules to the existing summaries, so the whole span is covered and nothing is lost.` : `Your tier-2 compression summaries have accumulated. Condense them further into a tier-3 ultra-condensed summary. Use block IDs as boundaries (startId and endId as bN). Any raw (uncompressed) messages sitting between the boundary blocks are absorbed into the tier-3 block as well \u2014 apply HOW TO COMPRESS to those raw messages and the TIER 3 condensation rules to the existing summaries, so the whole span is covered and nothing is lost.`,
|
|
1861
1920
|
blockList,
|
|
1862
1921
|
`Example: compress({ content: [{ startId: "${startId}", endId: "${endId}", summary: "..." }] })`,
|
|
1863
1922
|
"",
|
|
@@ -2649,6 +2708,7 @@ export {
|
|
|
2649
2708
|
createBpeTokenizer,
|
|
2650
2709
|
createCore,
|
|
2651
2710
|
createInitialState,
|
|
2711
|
+
createRenderRefsNode,
|
|
2652
2712
|
deactivateBlock,
|
|
2653
2713
|
defaultConfig,
|
|
2654
2714
|
defaultCountTokens,
|