opencode-vcc-compaction 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +307 -2
- package/adapter.mjs +243 -0
- package/engine/compile.mjs +962 -0
- package/engine/recall.mjs +277 -0
- package/engine/search.mjs +124 -0
- package/engine/session-compat.mjs +98 -0
- package/engine/tokenize.mjs +202 -0
- package/index.ts +38 -0
- package/package.json +33 -4
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Tokenizer + token-budget primitives, ported from
|
|
3
|
+
* dsh-compaction-vcc@0.7.4 `src/compiler.js` (MIT:
|
|
4
|
+
* Copyright (c) 2026 TsFreddie, Copyright (c) 2026 Tingfeng).
|
|
5
|
+
*
|
|
6
|
+
* Faithful port with two documented deviations:
|
|
7
|
+
* - `tailTokens` is exported here (module-private upstream) so the
|
|
8
|
+
* sibling compile stage can import it across the file split.
|
|
9
|
+
* - No DSH imports (upstream module is already dependency-free).
|
|
10
|
+
*
|
|
11
|
+
* Note on style: this file keeps upstream's semicolons verbatim for
|
|
12
|
+
* line-for-line comparability with the MIT source (present and future
|
|
13
|
+
* re-sync). This coincides with the repo's own `.mjs` style
|
|
14
|
+
* (`compiler/selftest.mjs`, `scripts/check-engine-sync.mjs` both use
|
|
15
|
+
* semicolons), so there is no style conflict in practice.
|
|
16
|
+
*
|
|
17
|
+
* Everything below is deterministic and offline: same input text always
|
|
18
|
+
* yields the same tokens, counts, and slices.
|
|
19
|
+
*/
|
|
20
|
+
|
|
21
|
+
// ── tokenizer (VCC `_tokenize` / `_TOK_RE`) ────────────────────────────────
|
|
22
|
+
|
|
23
|
+
const TOKEN_RE = /[a-zA-Z]+|[0-9]+|[^\sa-zA-Z0-9]|\s+/g;
|
|
24
|
+
|
|
25
|
+
/** Whether a UTF-16 code unit can be an unpaired high surrogate. */
|
|
26
|
+
function isHighSurrogate(code) {
|
|
27
|
+
return code >= 0xd800 && code <= 0xdbff;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/** Whether a UTF-16 code unit can be an unpaired low surrogate. */
|
|
31
|
+
function isLowSurrogate(code) {
|
|
32
|
+
return code >= 0xdc00 && code <= 0xdfff;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* Move the end of a slice to a code-point boundary: a cut must never leave a
|
|
37
|
+
* high surrogate without its low half.
|
|
38
|
+
*/
|
|
39
|
+
function completeCodePointAtEnd(text) {
|
|
40
|
+
return isHighSurrogate(text.charCodeAt(text.length - 1)) ? text.slice(0, -1) : text;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* Move the start of a slice to a code-point boundary (a tail excerpt must
|
|
45
|
+
* never begin inside a surrogate pair).
|
|
46
|
+
*/
|
|
47
|
+
function completeCodePointAtStart(text) {
|
|
48
|
+
return isLowSurrogate(text.charCodeAt(0)) ? text.slice(1) : text;
|
|
49
|
+
}
|
|
50
|
+
const ANSI_RE = /\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])/g;
|
|
51
|
+
const CTRL_RE = /[\x00-\x08\x0b-\x1f\x7f-\x9f]/g;
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* Split text into fixed-density tokens: letter runs, digit runs, single
|
|
55
|
+
* punctuation chars, and whitespace runs. Whitespace tokens are included
|
|
56
|
+
* in the list but never counted by {@link countTokens}.
|
|
57
|
+
*/
|
|
58
|
+
export function tokenize(text) {
|
|
59
|
+
return text.match(TOKEN_RE) ?? [];
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* True when the character carries the 0.6 CJK weight. Name kept for API
|
|
64
|
+
* stability; it means "CJK-density character" (kana, hangul, Thai, Lao,
|
|
65
|
+
* Indic scripts included — same density class, not ideographs only).
|
|
66
|
+
*/
|
|
67
|
+
export function isCjkChar(char) {
|
|
68
|
+
const code = char.codePointAt(0);
|
|
69
|
+
return (
|
|
70
|
+
(code >= 0x3400 && code <= 0x4dbf) || // CJK Extension A
|
|
71
|
+
(code >= 0x4e00 && code <= 0x9fff) || // CJK Unified Ideographs
|
|
72
|
+
(code >= 0xf900 && code <= 0xfaff) || // CJK Compatibility Ideographs
|
|
73
|
+
(code >= 0x3000 && code <= 0x303f) || // CJK Symbols and Punctuation
|
|
74
|
+
(code >= 0xff00 && code <= 0xffef) || // Fullwidth Forms
|
|
75
|
+
(code >= 0x3040 && code <= 0x30ff) || // Hiragana + Katakana
|
|
76
|
+
(code >= 0x31f0 && code <= 0x31ff) || // Katakana Phonetic Extensions
|
|
77
|
+
(code >= 0xac00 && code <= 0xd7af) || // Hangul Syllables
|
|
78
|
+
(code >= 0x1100 && code <= 0x11ff) || // Hangul Jamo
|
|
79
|
+
(code >= 0x0e00 && code <= 0x0e7f) || // Thai
|
|
80
|
+
(code >= 0x0e80 && code <= 0x0eff) || // Lao
|
|
81
|
+
(code >= 0x0900 && code <= 0x097f) || // Devanagari
|
|
82
|
+
(code >= 0x0980 && code <= 0x09ff) || // Bengali
|
|
83
|
+
(code >= 0x0a00 && code <= 0x0a7f) || // Gurmukhi
|
|
84
|
+
(code >= 0x0a80 && code <= 0x0aff) || // Gujarati
|
|
85
|
+
(code >= 0x0b00 && code <= 0x0b7f) || // Oriya
|
|
86
|
+
(code >= 0x0b80 && code <= 0x0bff) || // Tamil
|
|
87
|
+
(code >= 0x0c00 && code <= 0x0c7f) || // Telugu
|
|
88
|
+
(code >= 0x0c80 && code <= 0x0cff) || // Kannada
|
|
89
|
+
(code >= 0x0d00 && code <= 0x0d7f) || // Malayalam
|
|
90
|
+
(code >= 0x0d80 && code <= 0x0dff) || // Sinhala
|
|
91
|
+
(code >= 0x1000 && code <= 0x109f) || // Myanmar
|
|
92
|
+
(code >= 0x1780 && code <= 0x17ff) // Khmer
|
|
93
|
+
);
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/**
|
|
97
|
+
* Estimate tokens under the character-density conversion (0.3 per
|
|
98
|
+
* non-CJK character, 0.6 per CJK character). Integer tenths, exact
|
|
99
|
+
* under ceil for long inputs.
|
|
100
|
+
*/
|
|
101
|
+
export function estimateDeepSeekTokens(text) {
|
|
102
|
+
let nonCjk = 0;
|
|
103
|
+
let cjk = 0;
|
|
104
|
+
for (const char of text) {
|
|
105
|
+
if (isCjkChar(char)) cjk += 1;
|
|
106
|
+
else nonCjk += 1;
|
|
107
|
+
}
|
|
108
|
+
return (nonCjk * 3 + cjk * 6) / 10;
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
/** Count the non-whitespace tokens of a text under {@link tokenize}. */
|
|
112
|
+
export function countTokens(text) {
|
|
113
|
+
let count = 0;
|
|
114
|
+
for (const token of tokenize(text)) if (token.trim().length > 0) count += 1;
|
|
115
|
+
return count;
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
/**
|
|
119
|
+
* Size one emitted entry the way every budget in this module is enforced:
|
|
120
|
+
* the word-class token count, floored at the character-density equivalent
|
|
121
|
+
* so the total cap also sees pathological long runs.
|
|
122
|
+
*/
|
|
123
|
+
export function estimateEntryTokens(text) {
|
|
124
|
+
return Math.max(countTokens(text), Math.ceil(estimateDeepSeekTokens(text)));
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
/**
|
|
128
|
+
* Strip carriage returns, ANSI escapes, and control bytes — deterministic
|
|
129
|
+
* cleanup, never token-changing rewrites.
|
|
130
|
+
*/
|
|
131
|
+
export function sanitize(text) {
|
|
132
|
+
if (typeof text !== "string") return "";
|
|
133
|
+
let out = text;
|
|
134
|
+
if (out.includes("\r")) out = out.replaceAll("\r", "");
|
|
135
|
+
if (out.includes("\x1b")) out = out.replace(ANSI_RE, "");
|
|
136
|
+
return out.replace(CTRL_RE, "");
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
// ── truncation (VCC `_trunc`) ──────────────────────────────────────────────
|
|
140
|
+
|
|
141
|
+
/**
|
|
142
|
+
* Truncate text to a non-whitespace-token budget, preserving exact original
|
|
143
|
+
* tokens and appending a provenance note on a cut. Every budget is enforced
|
|
144
|
+
* twice — by token count AND by a `limit * 4` character ceiling (floor 32),
|
|
145
|
+
* so pathological unbroken runs (base64 blobs, minified files) cannot pass.
|
|
146
|
+
*/
|
|
147
|
+
export function truncateTokens(text, limit, ref) {
|
|
148
|
+
const tokens = tokenize(text);
|
|
149
|
+
let count = 0;
|
|
150
|
+
let cut = tokens.length;
|
|
151
|
+
for (let index = 0; index < tokens.length; index += 1) {
|
|
152
|
+
const token = tokens[index];
|
|
153
|
+
if (token.trim().length === 0) continue;
|
|
154
|
+
count += 1;
|
|
155
|
+
if (count > limit) {
|
|
156
|
+
cut = index;
|
|
157
|
+
break;
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
let out = cut >= tokens.length ? text : tokens.slice(0, cut).join("");
|
|
161
|
+
let truncated = cut < tokens.length;
|
|
162
|
+
const maxChars = Math.max(32, limit * 4);
|
|
163
|
+
if (out.length > maxChars) {
|
|
164
|
+
out = completeCodePointAtEnd(out.slice(0, maxChars));
|
|
165
|
+
truncated = true;
|
|
166
|
+
} else if (truncated) {
|
|
167
|
+
out = completeCodePointAtEnd(out);
|
|
168
|
+
}
|
|
169
|
+
if (!truncated) return { text, truncated: false };
|
|
170
|
+
out = out.replace(/\s+$/u, "");
|
|
171
|
+
const note = ref === undefined ? "...(truncated)" : `...(truncated from ${ref})`;
|
|
172
|
+
return { text: out + note, truncated: true };
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
/**
|
|
176
|
+
* Keep the LAST `limit` non-whitespace tokens of a text, additionally capped
|
|
177
|
+
* at `limit * 4` characters. The end of the text is always preserved.
|
|
178
|
+
*/
|
|
179
|
+
export function tailTokens(text, limit, ref) {
|
|
180
|
+
const tokens = tokenize(text);
|
|
181
|
+
let count = 0;
|
|
182
|
+
let startIdx = tokens.length;
|
|
183
|
+
for (let index = tokens.length - 1; index >= 0; index -= 1) {
|
|
184
|
+
if (tokens[index].trim().length > 0) {
|
|
185
|
+
count += 1;
|
|
186
|
+
if (count > limit) break;
|
|
187
|
+
}
|
|
188
|
+
startIdx = index;
|
|
189
|
+
}
|
|
190
|
+
let out = startIdx === 0 ? text : tokens.slice(startIdx).join("");
|
|
191
|
+
let truncated = startIdx > 0;
|
|
192
|
+
const maxChars = Math.max(32, limit * 4);
|
|
193
|
+
if (out.length > maxChars) {
|
|
194
|
+
out = completeCodePointAtStart(out.slice(out.length - maxChars));
|
|
195
|
+
truncated = true;
|
|
196
|
+
} else if (truncated) {
|
|
197
|
+
out = completeCodePointAtStart(out);
|
|
198
|
+
}
|
|
199
|
+
if (!truncated) return { text, truncated: false };
|
|
200
|
+
const note = ref === undefined ? "...(elided before)" : `...(elided before ${ref})`;
|
|
201
|
+
return { text: note + out, truncated: true };
|
|
202
|
+
}
|
package/index.ts
ADDED
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import { Plugin } from "@opencode/plugin/effect"
|
|
2
|
+
import { Effect } from "effect"
|
|
3
|
+
import { buildConfig, compileCompaction } from "./adapter.mjs"
|
|
4
|
+
|
|
5
|
+
// opencode-vcc-compaction: deterministic LLM-free compaction summaries.
|
|
6
|
+
//
|
|
7
|
+
// The hook adapts the request messages through adapter.mjs (pure offline
|
|
8
|
+
// mapping + the VCC reference compiler) and sets event.result so the host
|
|
9
|
+
// takes its fromHook short-circuit: no model call, tail rebuilt from
|
|
10
|
+
// metadata.keepID. When the adapter abstains (no message id at/after the
|
|
11
|
+
// keep cut), result stays unset and the host falls back to model compaction.
|
|
12
|
+
//
|
|
13
|
+
// Effect-direct by owner decision: written against @opencode/plugin/effect
|
|
14
|
+
// from the start, so no convert-to-Effect task ever arises.
|
|
15
|
+
|
|
16
|
+
export default Plugin.define({
|
|
17
|
+
id: "opencode-vcc-compaction",
|
|
18
|
+
effect: Effect.fn(function* (ctx) {
|
|
19
|
+
const config = buildConfig(ctx.options)
|
|
20
|
+
|
|
21
|
+
yield* ctx.session.hook("compaction", (event) =>
|
|
22
|
+
Effect.sync(() => {
|
|
23
|
+
let outcome
|
|
24
|
+
try {
|
|
25
|
+
outcome = compileCompaction(event.messages, config)
|
|
26
|
+
} catch (error) {
|
|
27
|
+
console.warn("[opencode-vcc-compaction] adapter failed; leaving result unset:", error)
|
|
28
|
+
return
|
|
29
|
+
}
|
|
30
|
+
if (outcome === undefined) return
|
|
31
|
+
event.result = {
|
|
32
|
+
summary: outcome.summary,
|
|
33
|
+
metadata: { keepID: outcome.keepID },
|
|
34
|
+
}
|
|
35
|
+
}),
|
|
36
|
+
)
|
|
37
|
+
}),
|
|
38
|
+
})
|
package/package.json
CHANGED
|
@@ -1,6 +1,35 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "opencode-vcc-compaction",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"
|
|
5
|
-
"
|
|
6
|
-
|
|
3
|
+
"version": "0.1.0",
|
|
4
|
+
"description": "Deterministic LLM-free compaction summaries for OpenCode V2 (VCC reference engine, Effect-direct hook).",
|
|
5
|
+
"license": "MIT",
|
|
6
|
+
"author": "jwitmann <jerome_witmann@witmanco.net>",
|
|
7
|
+
"homepage": "https://github.com/jwitmann/opencode-vcc-compaction#readme",
|
|
8
|
+
"repository": {
|
|
9
|
+
"type": "git",
|
|
10
|
+
"url": "https://github.com/jwitmann/opencode-vcc-compaction.git",
|
|
11
|
+
"directory": "vcc-plugin"
|
|
12
|
+
},
|
|
13
|
+
"bugs": {
|
|
14
|
+
"url": "https://github.com/jwitmann/opencode-vcc-compaction/issues"
|
|
15
|
+
},
|
|
16
|
+
"keywords": [
|
|
17
|
+
"opencode",
|
|
18
|
+
"opencode-plugin",
|
|
19
|
+
"compaction",
|
|
20
|
+
"context-management",
|
|
21
|
+
"deterministic",
|
|
22
|
+
"llm-free",
|
|
23
|
+
"effect"
|
|
24
|
+
],
|
|
25
|
+
"type": "module",
|
|
26
|
+
"main": "./index.ts",
|
|
27
|
+
"files": ["index.ts", "adapter.mjs", "engine/", "README.md"],
|
|
28
|
+
"engines": {
|
|
29
|
+
"node": ">=18"
|
|
30
|
+
},
|
|
31
|
+
"peerDependencies": {
|
|
32
|
+
"@opencode/plugin": "^2.0.0",
|
|
33
|
+
"effect": "^4.0.0"
|
|
34
|
+
}
|
|
35
|
+
}
|