opencode-vcc-compaction 0.0.0-stage → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,202 @@
1
+ /**
2
+ * Tokenizer + token-budget primitives, ported from
3
+ * dsh-compaction-vcc@0.7.4 `src/compiler.js` (MIT:
4
+ * Copyright (c) 2026 TsFreddie, Copyright (c) 2026 Tingfeng).
5
+ *
6
+ * Faithful port with two documented deviations:
7
+ * - `tailTokens` is exported here (module-private upstream) so the
8
+ * sibling compile stage can import it across the file split.
9
+ * - No DSH imports (upstream module is already dependency-free).
10
+ *
11
+ * Note on style: this file keeps upstream's semicolons verbatim for
12
+ * line-for-line comparability with the MIT source (present and future
13
+ * re-sync). This coincides with the repo's own `.mjs` style
14
+ * (`compiler/selftest.mjs`, `scripts/check-engine-sync.mjs` both use
15
+ * semicolons), so there is no style conflict in practice.
16
+ *
17
+ * Everything below is deterministic and offline: same input text always
18
+ * yields the same tokens, counts, and slices.
19
+ */
20
+
21
+ // ── tokenizer (VCC `_tokenize` / `_TOK_RE`) ────────────────────────────────
22
+
23
+ const TOKEN_RE = /[a-zA-Z]+|[0-9]+|[^\sa-zA-Z0-9]|\s+/g;
24
+
25
+ /** Whether a UTF-16 code unit can be an unpaired high surrogate. */
26
+ function isHighSurrogate(code) {
27
+ return code >= 0xd800 && code <= 0xdbff;
28
+ }
29
+
30
+ /** Whether a UTF-16 code unit can be an unpaired low surrogate. */
31
+ function isLowSurrogate(code) {
32
+ return code >= 0xdc00 && code <= 0xdfff;
33
+ }
34
+
35
+ /**
36
+ * Move the end of a slice to a code-point boundary: a cut must never leave a
37
+ * high surrogate without its low half.
38
+ */
39
+ function completeCodePointAtEnd(text) {
40
+ return isHighSurrogate(text.charCodeAt(text.length - 1)) ? text.slice(0, -1) : text;
41
+ }
42
+
43
+ /**
44
+ * Move the start of a slice to a code-point boundary (a tail excerpt must
45
+ * never begin inside a surrogate pair).
46
+ */
47
+ function completeCodePointAtStart(text) {
48
+ return isLowSurrogate(text.charCodeAt(0)) ? text.slice(1) : text;
49
+ }
50
+ const ANSI_RE = /\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])/g;
51
+ const CTRL_RE = /[\x00-\x08\x0b-\x1f\x7f-\x9f]/g;
52
+
53
+ /**
54
+ * Split text into fixed-density tokens: letter runs, digit runs, single
55
+ * punctuation chars, and whitespace runs. Whitespace tokens are included
56
+ * in the list but never counted by {@link countTokens}.
57
+ */
58
+ export function tokenize(text) {
59
+ return text.match(TOKEN_RE) ?? [];
60
+ }
61
+
62
+ /**
63
+ * True when the character carries the 0.6 CJK weight. Name kept for API
64
+ * stability; it means "CJK-density character" (kana, hangul, Thai, Lao,
65
+ * Indic scripts included — same density class, not ideographs only).
66
+ */
67
+ export function isCjkChar(char) {
68
+ const code = char.codePointAt(0);
69
+ return (
70
+ (code >= 0x3400 && code <= 0x4dbf) || // CJK Extension A
71
+ (code >= 0x4e00 && code <= 0x9fff) || // CJK Unified Ideographs
72
+ (code >= 0xf900 && code <= 0xfaff) || // CJK Compatibility Ideographs
73
+ (code >= 0x3000 && code <= 0x303f) || // CJK Symbols and Punctuation
74
+ (code >= 0xff00 && code <= 0xffef) || // Fullwidth Forms
75
+ (code >= 0x3040 && code <= 0x30ff) || // Hiragana + Katakana
76
+ (code >= 0x31f0 && code <= 0x31ff) || // Katakana Phonetic Extensions
77
+ (code >= 0xac00 && code <= 0xd7af) || // Hangul Syllables
78
+ (code >= 0x1100 && code <= 0x11ff) || // Hangul Jamo
79
+ (code >= 0x0e00 && code <= 0x0e7f) || // Thai
80
+ (code >= 0x0e80 && code <= 0x0eff) || // Lao
81
+ (code >= 0x0900 && code <= 0x097f) || // Devanagari
82
+ (code >= 0x0980 && code <= 0x09ff) || // Bengali
83
+ (code >= 0x0a00 && code <= 0x0a7f) || // Gurmukhi
84
+ (code >= 0x0a80 && code <= 0x0aff) || // Gujarati
85
+ (code >= 0x0b00 && code <= 0x0b7f) || // Oriya
86
+ (code >= 0x0b80 && code <= 0x0bff) || // Tamil
87
+ (code >= 0x0c00 && code <= 0x0c7f) || // Telugu
88
+ (code >= 0x0c80 && code <= 0x0cff) || // Kannada
89
+ (code >= 0x0d00 && code <= 0x0d7f) || // Malayalam
90
+ (code >= 0x0d80 && code <= 0x0dff) || // Sinhala
91
+ (code >= 0x1000 && code <= 0x109f) || // Myanmar
92
+ (code >= 0x1780 && code <= 0x17ff) // Khmer
93
+ );
94
+ }
95
+
96
+ /**
97
+ * Estimate tokens under the character-density conversion (0.3 per
98
+ * non-CJK character, 0.6 per CJK character). Integer tenths, exact
99
+ * under ceil for long inputs.
100
+ */
101
+ export function estimateDeepSeekTokens(text) {
102
+ let nonCjk = 0;
103
+ let cjk = 0;
104
+ for (const char of text) {
105
+ if (isCjkChar(char)) cjk += 1;
106
+ else nonCjk += 1;
107
+ }
108
+ return (nonCjk * 3 + cjk * 6) / 10;
109
+ }
110
+
111
+ /** Count the non-whitespace tokens of a text under {@link tokenize}. */
112
+ export function countTokens(text) {
113
+ let count = 0;
114
+ for (const token of tokenize(text)) if (token.trim().length > 0) count += 1;
115
+ return count;
116
+ }
117
+
118
+ /**
119
+ * Size one emitted entry the way every budget in this module is enforced:
120
+ * the word-class token count, floored at the character-density equivalent
121
+ * so the total cap also sees pathological long runs.
122
+ */
123
+ export function estimateEntryTokens(text) {
124
+ return Math.max(countTokens(text), Math.ceil(estimateDeepSeekTokens(text)));
125
+ }
126
+
127
+ /**
128
+ * Strip carriage returns, ANSI escapes, and control bytes — deterministic
129
+ * cleanup, never token-changing rewrites.
130
+ */
131
+ export function sanitize(text) {
132
+ if (typeof text !== "string") return "";
133
+ let out = text;
134
+ if (out.includes("\r")) out = out.replaceAll("\r", "");
135
+ if (out.includes("\x1b")) out = out.replace(ANSI_RE, "");
136
+ return out.replace(CTRL_RE, "");
137
+ }
138
+
139
+ // ── truncation (VCC `_trunc`) ──────────────────────────────────────────────
140
+
141
+ /**
142
+ * Truncate text to a non-whitespace-token budget, preserving exact original
143
+ * tokens and appending a provenance note on a cut. Every budget is enforced
144
+ * twice — by token count AND by a `limit * 4` character ceiling (floor 32),
145
+ * so pathological unbroken runs (base64 blobs, minified files) cannot pass.
146
+ */
147
+ export function truncateTokens(text, limit, ref) {
148
+ const tokens = tokenize(text);
149
+ let count = 0;
150
+ let cut = tokens.length;
151
+ for (let index = 0; index < tokens.length; index += 1) {
152
+ const token = tokens[index];
153
+ if (token.trim().length === 0) continue;
154
+ count += 1;
155
+ if (count > limit) {
156
+ cut = index;
157
+ break;
158
+ }
159
+ }
160
+ let out = cut >= tokens.length ? text : tokens.slice(0, cut).join("");
161
+ let truncated = cut < tokens.length;
162
+ const maxChars = Math.max(32, limit * 4);
163
+ if (out.length > maxChars) {
164
+ out = completeCodePointAtEnd(out.slice(0, maxChars));
165
+ truncated = true;
166
+ } else if (truncated) {
167
+ out = completeCodePointAtEnd(out);
168
+ }
169
+ if (!truncated) return { text, truncated: false };
170
+ out = out.replace(/\s+$/u, "");
171
+ const note = ref === undefined ? "...(truncated)" : `...(truncated from ${ref})`;
172
+ return { text: out + note, truncated: true };
173
+ }
174
+
175
+ /**
176
+ * Keep the LAST `limit` non-whitespace tokens of a text, additionally capped
177
+ * at `limit * 4` characters. The end of the text is always preserved.
178
+ */
179
+ export function tailTokens(text, limit, ref) {
180
+ const tokens = tokenize(text);
181
+ let count = 0;
182
+ let startIdx = tokens.length;
183
+ for (let index = tokens.length - 1; index >= 0; index -= 1) {
184
+ if (tokens[index].trim().length > 0) {
185
+ count += 1;
186
+ if (count > limit) break;
187
+ }
188
+ startIdx = index;
189
+ }
190
+ let out = startIdx === 0 ? text : tokens.slice(startIdx).join("");
191
+ let truncated = startIdx > 0;
192
+ const maxChars = Math.max(32, limit * 4);
193
+ if (out.length > maxChars) {
194
+ out = completeCodePointAtStart(out.slice(out.length - maxChars));
195
+ truncated = true;
196
+ } else if (truncated) {
197
+ out = completeCodePointAtStart(out);
198
+ }
199
+ if (!truncated) return { text, truncated: false };
200
+ const note = ref === undefined ? "...(elided before)" : `...(elided before ${ref})`;
201
+ return { text: note + out, truncated: true };
202
+ }
package/index.ts ADDED
@@ -0,0 +1,286 @@
1
+ import { Plugin } from "@opencode/plugin/effect"
2
+ import { Effect } from "effect"
3
+ import { buildConfig, compileCompaction } from "./adapter.mjs"
4
+ import {
5
+ COMPILER_REV,
6
+ SNAPSHOT_KEY_PREFIX,
7
+ buildSnapshot,
8
+ classifySnapshotStore,
9
+ formatRecallReply,
10
+ formatSearchReply,
11
+ formatStatusText,
12
+ parseRecallArgs,
13
+ parseSearchArgs,
14
+ recallFromSnapshot,
15
+ resolveRecallSelections,
16
+ searchSnapshot,
17
+ snapshotNeedsWrite,
18
+ } from "./snapshot.mjs"
19
+
20
+ // opencode-vcc-compaction: deterministic LLM-free compaction summaries.
21
+ //
22
+ // The compaction hook adapts the request messages through adapter.mjs (pure
23
+ // offline mapping + the VCC reference compiler) and sets event.result so the
24
+ // host takes its fromHook short-circuit: no model call, tail rebuilt from
25
+ // metadata.keepID. When the adapter abstains (no message id at/after the
26
+ // keep cut), result stays unset and the host falls back to model compaction.
27
+ // Alongside, context + compaction hooks snapshot cleaned raw messages
28
+ // (snapshot.mjs, growth-guarded by version/count/lastID, warn-and-continue)
29
+ // so the vcc- commands can recall/search history the command ctx cannot
30
+ // read directly.
31
+ //
32
+ // Effect-direct by owner decision: written against @opencode/plugin/effect
33
+ // from the start, so no convert-to-Effect task ever arises.
34
+
35
+ export default Plugin.define({
36
+ id: "opencode-vcc-compaction",
37
+ effect: Effect.fn(function* (ctx) {
38
+ const config = buildConfig(ctx.options)
39
+ const seen = new Map<string, { version: number; count: number; lastID: string | undefined }>()
40
+
41
+ const remember = (sessionID: unknown, messages: unknown) =>
42
+ Effect.catch(
43
+ Effect.flatMap(
44
+ Effect.sync(() => {
45
+ if (typeof sessionID !== "string" || sessionID === "") {
46
+ console.warn("[opencode-vcc-compaction] snapshot skipped: missing sessionID")
47
+ return null
48
+ }
49
+ return buildSnapshot(messages as never)
50
+ }),
51
+ (snap) => {
52
+ if (snap === null) return Effect.void
53
+ const prev = seen.get(sessionID as string)
54
+ if (!snapshotNeedsWrite(prev, snap)) return Effect.void
55
+ return Effect.catch(
56
+ Effect.flatMap(
57
+ ctx.storage.set(`${SNAPSHOT_KEY_PREFIX}${sessionID}`, {
58
+ version: snap.version,
59
+ messages: snap.messages,
60
+ count: snap.count,
61
+ ...(snap.lastID === undefined ? {} : { lastID: snap.lastID }),
62
+ writtenAt: Date.now(),
63
+ }),
64
+ () =>
65
+ Effect.sync(() => {
66
+ seen.set(sessionID as string, {
67
+ version: snap.version,
68
+ count: snap.count,
69
+ lastID: snap.lastID,
70
+ })
71
+ }),
72
+ ),
73
+ (error) =>
74
+ Effect.sync(() => {
75
+ console.warn("[opencode-vcc-compaction] snapshot write failed:", error)
76
+ }),
77
+ )
78
+ },
79
+ ),
80
+ (error) =>
81
+ Effect.sync(() => {
82
+ console.warn("[opencode-vcc-compaction] snapshot build failed:", error)
83
+ }),
84
+ )
85
+
86
+ try {
87
+ yield* ctx.session.hook("context", (event) =>
88
+ remember((event as { sessionID: string }).sessionID, (event as { messages: unknown }).messages),
89
+ )
90
+ } catch (error) {
91
+ console.warn("[opencode-vcc-compaction] context hook unavailable; snapshots via compaction only:", error)
92
+ }
93
+
94
+ yield* ctx.session.hook("compaction", (event) =>
95
+ Effect.flatMap(
96
+ remember((event as { sessionID: string }).sessionID, (event as { messages: unknown }).messages),
97
+ () =>
98
+ Effect.sync(() => {
99
+ let outcome
100
+ try {
101
+ outcome = compileCompaction(event.messages, config)
102
+ } catch (error) {
103
+ console.warn("[opencode-vcc-compaction] adapter failed; leaving result unset:", error)
104
+ return
105
+ }
106
+ if (outcome === undefined) return
107
+ event.result = {
108
+ summary: outcome.summary,
109
+ metadata: { keepID: outcome.keepID },
110
+ }
111
+ }),
112
+ ),
113
+ )
114
+
115
+ try {
116
+ yield* ctx.command.transform((editor) => {
117
+ editor.add({
118
+ name: "vcc-recall",
119
+ description: "Restore exact original content from this session by (seq N), result, or checkpoint reference.",
120
+ execute: (input) =>
121
+ Effect.catchCause(
122
+ Effect.flatMap(
123
+ Effect.sync(() => parseRecallArgs((input.prompt as { text?: unknown })?.text ?? "")),
124
+ (parsed) => {
125
+ if ("error" in parsed) {
126
+ return ctx.session.prompt({
127
+ sessionID: input.sessionID,
128
+ text: `**/vcc-recall** — ${parsed.error}`,
129
+ delivery: input.delivery,
130
+ })
131
+ }
132
+ return Effect.flatMap(
133
+ Effect.catch(
134
+ ctx.storage.get(`${SNAPSHOT_KEY_PREFIX}${input.sessionID}`),
135
+ (error) =>
136
+ Effect.sync(() => {
137
+ console.warn("[opencode-vcc-compaction] vcc-recall: snapshot read failed:", error)
138
+ return { __vccReadFailed: true as const }
139
+ }),
140
+ ),
141
+ (stored) => {
142
+ const classified = classifySnapshotStore(stored as never)
143
+ if (classified.status === "failed") {
144
+ return ctx.session.prompt({
145
+ sessionID: input.sessionID,
146
+ text: "**/vcc-recall** — snapshot read failed, try again in a moment",
147
+ delivery: input.delivery,
148
+ })
149
+ }
150
+ if (classified.status === "empty") {
151
+ console.log("[opencode-vcc-compaction] vcc-recall: no snapshot yet for this session")
152
+ return Effect.void
153
+ }
154
+ const resolved = resolveRecallSelections(classified.messages as never, parsed.type, parsed.id)
155
+ if (resolved.errors.length > 0 || resolved.selections.length === 0) {
156
+ return ctx.session.prompt({
157
+ sessionID: input.sessionID,
158
+ text: `**/vcc-recall** — ${resolved.errors.join("; ") || "nothing to recall"}`,
159
+ delivery: input.delivery,
160
+ })
161
+ }
162
+ const recalled = recallFromSnapshot(classified.messages as never, resolved.selections)
163
+ return ctx.session.prompt({
164
+ sessionID: input.sessionID,
165
+ text: formatRecallReply(parsed.type, parsed.id, recalled as never),
166
+ delivery: input.delivery,
167
+ })
168
+ },
169
+ )
170
+ },
171
+ ),
172
+ (cause) =>
173
+ Effect.sync(() => {
174
+ console.warn("[opencode-vcc-compaction] vcc-recall failed:", cause)
175
+ }),
176
+ ),
177
+ })
178
+ editor.add({
179
+ name: "vcc-search",
180
+ description: "Search this session's history by keyword or regex; returns (seq N) hits for /vcc-recall.",
181
+ execute: (input) =>
182
+ Effect.catchCause(
183
+ Effect.flatMap(
184
+ Effect.sync(() => parseSearchArgs((input.prompt as { text?: unknown })?.text ?? "")),
185
+ (parsed) => {
186
+ if ("error" in parsed) {
187
+ return ctx.session.prompt({
188
+ sessionID: input.sessionID,
189
+ text: `**/vcc-search** — ${parsed.error}`,
190
+ delivery: input.delivery,
191
+ })
192
+ }
193
+ return Effect.flatMap(
194
+ Effect.catch(
195
+ ctx.storage.get(`${SNAPSHOT_KEY_PREFIX}${input.sessionID}`),
196
+ (error) =>
197
+ Effect.sync(() => {
198
+ console.warn("[opencode-vcc-compaction] vcc-search: snapshot read failed:", error)
199
+ return { __vccReadFailed: true as const }
200
+ }),
201
+ ),
202
+ (stored) => {
203
+ const classified = classifySnapshotStore(stored as never)
204
+ if (classified.status === "failed") {
205
+ return ctx.session.prompt({
206
+ sessionID: input.sessionID,
207
+ text: "**/vcc-search** — snapshot read failed, try again in a moment",
208
+ delivery: input.delivery,
209
+ })
210
+ }
211
+ if (classified.status === "empty") {
212
+ console.log("[opencode-vcc-compaction] vcc-search: no snapshot yet for this session")
213
+ return Effect.void
214
+ }
215
+ let found
216
+ try {
217
+ found = searchSnapshot(classified.messages as never, parsed.pattern)
218
+ } catch (error) {
219
+ return ctx.session.prompt({
220
+ sessionID: input.sessionID,
221
+ text: `**/vcc-search** — ${error instanceof Error ? error.message : String(error)}`,
222
+ delivery: input.delivery,
223
+ })
224
+ }
225
+ return ctx.session.prompt({
226
+ sessionID: input.sessionID,
227
+ text: formatSearchReply(parsed.pattern, found as never),
228
+ delivery: input.delivery,
229
+ })
230
+ },
231
+ )
232
+ },
233
+ ),
234
+ (cause) =>
235
+ Effect.sync(() => {
236
+ console.warn("[opencode-vcc-compaction] vcc-search failed:", cause)
237
+ }),
238
+ ),
239
+ })
240
+ editor.add({
241
+ name: "vcc-status",
242
+ description: "Show opencode-vcc-compaction engine revision, config, and snapshot freshness.",
243
+ execute: (input) =>
244
+ Effect.catchCause(
245
+ Effect.flatMap(
246
+ Effect.catch(
247
+ ctx.storage.get(`${SNAPSHOT_KEY_PREFIX}${input.sessionID}`),
248
+ (error) =>
249
+ Effect.sync(() => {
250
+ console.warn("[opencode-vcc-compaction] vcc-status: snapshot read failed:", error)
251
+ return { __vccReadFailed: true as const }
252
+ }),
253
+ ),
254
+ (stored) => {
255
+ const classified = classifySnapshotStore(stored as never)
256
+ if (classified.status === "failed") {
257
+ return ctx.session.prompt({
258
+ sessionID: input.sessionID,
259
+ text: "**/vcc-status** — snapshot read failed, try again in a moment",
260
+ delivery: input.delivery,
261
+ })
262
+ }
263
+ return ctx.session.prompt({
264
+ sessionID: input.sessionID,
265
+ text: formatStatusText(
266
+ classified.status === "ready" ? classified.stored as never : undefined,
267
+ config as never,
268
+ COMPILER_REV as string,
269
+ Date.now(),
270
+ ),
271
+ delivery: input.delivery,
272
+ })
273
+ },
274
+ ),
275
+ (cause) =>
276
+ Effect.sync(() => {
277
+ console.warn("[opencode-vcc-compaction] vcc-status failed:", cause)
278
+ }),
279
+ ),
280
+ })
281
+ })
282
+ } catch (error) {
283
+ console.warn("[opencode-vcc-compaction] command API unavailable; /vcc-* commands not registered:", error)
284
+ }
285
+ }),
286
+ })
package/package.json CHANGED
@@ -1,6 +1,35 @@
1
1
  {
2
2
  "name": "opencode-vcc-compaction",
3
- "version": "0.0.0-stage",
4
- "stub": true,
5
- "description": "Temporary package placeholder for staged publishing"
6
- }
3
+ "version": "0.2.0",
4
+ "description": "Deterministic LLM-free compaction summaries for OpenCode V2 (VCC reference engine, Effect-direct hook).",
5
+ "license": "MIT",
6
+ "author": "jwitmann <jerome_witmann@witmanco.net>",
7
+ "homepage": "https://github.com/jwitmann/opencode-vcc-compaction#readme",
8
+ "repository": {
9
+ "type": "git",
10
+ "url": "https://github.com/jwitmann/opencode-vcc-compaction.git",
11
+ "directory": "vcc-plugin"
12
+ },
13
+ "bugs": {
14
+ "url": "https://github.com/jwitmann/opencode-vcc-compaction/issues"
15
+ },
16
+ "keywords": [
17
+ "opencode",
18
+ "opencode-plugin",
19
+ "compaction",
20
+ "context-management",
21
+ "deterministic",
22
+ "llm-free",
23
+ "effect"
24
+ ],
25
+ "type": "module",
26
+ "main": "./index.ts",
27
+ "files": ["index.ts", "adapter.mjs", "snapshot.mjs", "engine/", "README.md"],
28
+ "engines": {
29
+ "node": ">=18"
30
+ },
31
+ "peerDependencies": {
32
+ "@opencode/plugin": "^2.0.0",
33
+ "effect": "^4.0.0"
34
+ }
35
+ }