@snylonue/dsh-context-policy 0.3.0 → 0.3.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,255 @@
1
+ import assert from "node:assert/strict";
2
+ import { spawnSync } from "node:child_process";
3
+ import test from "node:test";
4
+ import { Context } from "@deepseek-ai/cordis";
5
+ import { createUserMessage } from "@deepseek-ai/dsh-llm";
6
+ import type { Session, SessionEvent } from "@deepseek-ai/dsh-session";
7
+ import { SessionSeq } from "@deepseek-ai/dsh-session";
8
+ import Registry from "@deepseek-ai/dsh-session-projection";
9
+ import { Tokenizer } from "@huggingface/tokenizers";
10
+ import { createTokenizerCounter } from "../dist/builtin-tokenizers.js";
11
+ import { foldCompact, initialCompactState } from "../dist/compact.js";
12
+ import Meter from "../dist/token-meter.js";
13
+ import { registerTokenizer } from "../dist/tokenizers.js";
14
+
15
+ function user(seq: number, text: string): SessionEvent {
16
+ return {
17
+ type: "user/message",
18
+ seq: SessionSeq(seq),
19
+ time: seq,
20
+ data: createUserMessage({
21
+ source: { kind: "user" },
22
+ content: [{ type: "text", text }],
23
+ }),
24
+ surfaceOp: "append",
25
+ } as SessionEvent;
26
+ }
27
+ function session(events: SessionEvent[]): Session {
28
+ return {
29
+ seq: events.length,
30
+ header: {},
31
+ inheritedEventCount: 0,
32
+ snapshotEvents: () => events.slice(),
33
+ eventAt: (seq: number) => events[seq],
34
+ } as unknown as Session;
35
+ }
36
+
37
+ test("cold registry replay counts each event once across three projections beyond both text-cache limits", async () => {
38
+ let callsA = 0;
39
+ let callsB = 0;
40
+ const unregisterA = registerTokenizer("replay-a", (text) => {
41
+ callsA++;
42
+ return text.length;
43
+ });
44
+ const unregisterB = registerTokenizer("replay-b", (text) => {
45
+ callsB++;
46
+ return text.length + 11;
47
+ });
48
+ const ctx = new Context();
49
+ try {
50
+ await ctx.plugin(Registry).await();
51
+ await ctx
52
+ .plugin(
53
+ Meter,
54
+ Meter.Config({
55
+ defaultTokenizer: "replay-a",
56
+ providerPolicies: [{ provider: "p", tokenizer: "replay-b" }],
57
+ }),
58
+ )
59
+ .await();
60
+ const tools = [{ name: "tool", parameters: {} }];
61
+ const events: SessionEvent[] = [
62
+ {
63
+ type: "request/header",
64
+ seq: SessionSeq(0),
65
+ time: 0,
66
+ data: { header: { config: { provider: "p", model: "m" }, tools } },
67
+ } as SessionEvent,
68
+ ];
69
+ for (let i = 0; i < 600; i++) {
70
+ events.push(
71
+ user(
72
+ events.length,
73
+ `${String(i).padStart(4, "0")}:${"x".repeat(4495)}`,
74
+ ),
75
+ );
76
+ }
77
+ const s = session(events);
78
+ const values = ctx.sessionProjections.snapshot(s).values;
79
+ const measurement = ctx.tokenMeter.measure(s);
80
+ assert.equal(callsA, 601);
81
+ assert.equal(callsB, 601);
82
+ assert.equal(measurement.surfaceTokens, 600 * (4500 + 11 + 3));
83
+ const toolsTokens = JSON.stringify(tools).length + 11 + 4;
84
+ assert.equal(
85
+ measurement.totalTokens,
86
+ measurement.surfaceTokens + toolsTokens + 3,
87
+ );
88
+ assert.equal(
89
+ values.contextPressure?.projectedTokens,
90
+ measurement.totalTokens,
91
+ );
92
+ assert.deepEqual(values.contextBreakdown, {
93
+ systemTokens: 0,
94
+ toolsTokens,
95
+ messageTokens: measurement.surfaceTokens,
96
+ });
97
+ const checkpoint = JSON.parse(
98
+ JSON.stringify(ctx.sessionProjections.checkpoint(s)),
99
+ );
100
+ assert.deepEqual(ctx.sessionProjections.viewCheckpoint(checkpoint), values);
101
+ assert.equal(
102
+ callsA,
103
+ 601,
104
+ "checkpoint/view reads must not recount evicted bodies",
105
+ );
106
+ assert.equal(callsB, 601);
107
+ } finally {
108
+ await ctx.fiber.dispose();
109
+ unregisterA();
110
+ unregisterB();
111
+ }
112
+ });
113
+
114
+ test("oversized uncached text is counted once, not once per projection", async () => {
115
+ let calls = 0;
116
+ const unregister = registerTokenizer("replay-large", (text) => {
117
+ calls++;
118
+ return text.length;
119
+ });
120
+ const ctx = new Context();
121
+ try {
122
+ await ctx.plugin(Registry).await();
123
+ await ctx
124
+ .plugin(Meter, Meter.Config({ defaultTokenizer: "replay-large" }))
125
+ .await();
126
+ const s = session([user(0, "x".repeat(2_000_001))]);
127
+ ctx.sessionProjections.snapshot(s);
128
+ assert.equal(ctx.tokenMeter.measure(s).surfaceTokens, 2_000_004);
129
+ assert.equal(calls, 1);
130
+ } finally {
131
+ await ctx.fiber.dispose();
132
+ unregister();
133
+ }
134
+ });
135
+
136
+ test("payload memo separates event identity and tokenizer sets, not sequence numbers", () => {
137
+ const unregisterA = registerTokenizer(
138
+ "replay-identity-a",
139
+ (text) => text.length,
140
+ );
141
+ const unregisterB = registerTokenizer(
142
+ "replay-identity-b",
143
+ (text) => text.length * 2,
144
+ );
145
+ try {
146
+ const a = Meter.Config({ defaultTokenizer: "replay-identity-a" });
147
+ const b = Meter.Config({ defaultTokenizer: "replay-identity-b" });
148
+ const first = user(0, "first");
149
+ const otherSession = user(0, "different session");
150
+ const fold = (event: SessionEvent, config = a) =>
151
+ foldCompact(initialCompactState(), event, config);
152
+ const firstCounts = fold(first).nodes[0].counts;
153
+ assert.equal(firstCounts.text["replay-identity-a"], 5);
154
+ assert.strictEqual(fold(first).nodes[0].counts, firstCounts);
155
+ assert(Object.isFrozen(firstCounts));
156
+ assert(Object.isFrozen(firstCounts.text));
157
+ assert.throws(() => {
158
+ firstCounts.text["replay-identity-a"] = 99;
159
+ }, TypeError);
160
+ assert.equal(
161
+ fold(otherSession).nodes[0].counts.text["replay-identity-a"],
162
+ 17,
163
+ );
164
+ assert.equal(fold(first, b).nodes[0].counts.text["replay-identity-b"], 10);
165
+ assert.equal(fold(first).nodes[0].counts.text["replay-identity-a"], 5);
166
+ } finally {
167
+ unregisterA();
168
+ unregisterB();
169
+ }
170
+ });
171
+
172
+ test("re-registering a custom tokenizer cannot reuse old event counts", () => {
173
+ const config = Meter.Config({ defaultTokenizer: "replay-replaced" });
174
+ const event = user(0, "payload");
175
+ const unregister = registerTokenizer(
176
+ "replay-replaced",
177
+ (text) => text.length,
178
+ );
179
+ try {
180
+ assert.equal(
181
+ foldCompact(initialCompactState(), event, config).nodes[0].counts.text[
182
+ "replay-replaced"
183
+ ],
184
+ 7,
185
+ );
186
+ } finally {
187
+ unregister();
188
+ }
189
+ assert.throws(
190
+ () => foldCompact(initialCompactState(), event, config),
191
+ /Unknown tokenizer/,
192
+ );
193
+ const unregisterAgain = registerTokenizer(
194
+ "replay-replaced",
195
+ (text) => text.length * 2,
196
+ );
197
+ try {
198
+ assert.equal(
199
+ foldCompact(initialCompactState(), event, config).nodes[0].counts.text[
200
+ "replay-replaced"
201
+ ],
202
+ 14,
203
+ );
204
+ } finally {
205
+ unregisterAgain();
206
+ }
207
+ });
208
+
209
+ test("numeric payload memos do not keep discarded event objects alive", () => {
210
+ const compact = JSON.stringify(
211
+ new URL("../dist/compact.js", import.meta.url).href,
212
+ );
213
+ const tokenizers = JSON.stringify(
214
+ new URL("../dist/tokenizers.js", import.meta.url).href,
215
+ );
216
+ const script = [
217
+ `import {foldCompact, initialCompactState} from ${compact};`,
218
+ `import {registerTokenizer} from ${tokenizers};`,
219
+ "registerTokenizer('gc-counter', text => text.length);",
220
+ "let ref;",
221
+ "(() => {",
222
+ " const event = {type:'user/message', seq:0, time:0, surfaceOp:'append', data:{role:'user', id:'gc', source:{kind:'user'}, content:[{type:'text', text:'discarded body'}]}};",
223
+ " foldCompact(initialCompactState(), event, {defaultTokenizer:'gc-counter', modelPolicies:[]});",
224
+ " ref = new WeakRef(event);",
225
+ "})();",
226
+ "for (let i=0; i<16; i++) {",
227
+ " await new Promise(resolve => setImmediate(resolve));",
228
+ " globalThis.gc();",
229
+ " if (!ref.deref()) process.exit(0);",
230
+ "}",
231
+ "throw new Error('payload memo retained the event');",
232
+ ].join("\n");
233
+ // GC runs only in this disposable child, never the host or main test process.
234
+ const result = spawnSync(
235
+ process.execPath,
236
+ ["--expose-gc", "--input-type=module", "-e", script],
237
+ {
238
+ encoding: "utf8",
239
+ timeout: 10_000,
240
+ },
241
+ );
242
+ assert.ifError(result.error);
243
+ assert.equal(result.status, 0, result.stderr || result.stdout);
244
+ });
245
+
246
+ test("builtin token counts do not allocate token IDs or attention masks", (t) => {
247
+ t.mock.method(Tokenizer.prototype, "encode", () => {
248
+ throw new Error("counting must use tokenize rather than encode");
249
+ });
250
+ for (const name of ["deepseek-v4", "o200k-base"]) {
251
+ const count = createTokenizerCounter(name);
252
+ assert.equal(count(""), 0);
253
+ assert(count("hello 世界 <|endoftext|>") > 0);
254
+ }
255
+ });