@snylonue/dsh-context-policy 0.0.0-stage → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +223 -2
  3. package/assets/deepseek-v4/LICENSE +21 -0
  4. package/assets/deepseek-v4/README.md +19 -0
  5. package/assets/deepseek-v4/tokenizer.json.gz +0 -0
  6. package/assets/o200k-base/LICENSE +21 -0
  7. package/assets/o200k-base/README.md +20 -0
  8. package/assets/o200k-base/tokenizer.json.gz +0 -0
  9. package/cordis.patch.yml +19 -0
  10. package/dist/builtin-tokenizers.d.ts +2 -0
  11. package/dist/builtin-tokenizers.d.ts.map +1 -0
  12. package/dist/builtin-tokenizers.js +15 -0
  13. package/dist/builtin-tokenizers.js.map +1 -0
  14. package/dist/compaction.d.ts +43 -0
  15. package/dist/compaction.d.ts.map +1 -0
  16. package/dist/compaction.js +198 -0
  17. package/dist/compaction.js.map +1 -0
  18. package/dist/core.d.ts +49 -0
  19. package/dist/core.d.ts.map +1 -0
  20. package/dist/core.js +213 -0
  21. package/dist/core.js.map +1 -0
  22. package/dist/projection.d.ts +103 -0
  23. package/dist/projection.d.ts.map +1 -0
  24. package/dist/projection.js +137 -0
  25. package/dist/projection.js.map +1 -0
  26. package/dist/token-meter.d.ts +46 -0
  27. package/dist/token-meter.d.ts.map +1 -0
  28. package/dist/token-meter.js +55 -0
  29. package/dist/token-meter.js.map +1 -0
  30. package/dist/tokenizers.d.ts +6 -0
  31. package/dist/tokenizers.d.ts.map +1 -0
  32. package/dist/tokenizers.js +64 -0
  33. package/dist/tokenizers.js.map +1 -0
  34. package/dist/types.d.ts +62 -0
  35. package/dist/types.d.ts.map +1 -0
  36. package/dist/types.js +2 -0
  37. package/dist/types.js.map +1 -0
  38. package/dist/usage.d.ts +67 -0
  39. package/dist/usage.d.ts.map +1 -0
  40. package/dist/usage.js +81 -0
  41. package/dist/usage.js.map +1 -0
  42. package/examples/preset-compaction.yml +13 -0
  43. package/locale/en.json +6 -0
  44. package/locale/zh.json +6 -0
  45. package/package.json +92 -5
  46. package/scripts/audit.ts +105 -0
  47. package/scripts/check.ts +19 -0
  48. package/scripts/prepare-o200k.ts +40 -0
  49. package/src/builtin-tokenizers.ts +24 -0
  50. package/src/compaction.ts +293 -0
  51. package/src/core.ts +277 -0
  52. package/src/projection.ts +173 -0
  53. package/src/token-meter.ts +81 -0
  54. package/src/tokenizers.ts +78 -0
  55. package/src/types.ts +63 -0
  56. package/src/usage.ts +96 -0
  57. package/tests/bundle.test.ts +43 -0
  58. package/tests/deepseek-v4-fixtures.json +125 -0
  59. package/tests/o200k-base-fixtures.json +184 -0
  60. package/tests/policy.test.ts +773 -0
  61. package/tests/tokenizers.test.ts +51 -0
  62. package/tsconfig.check.json +15 -0
  63. package/tsconfig.json +17 -0
package/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 DeepSeek
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
package/README.md CHANGED
@@ -1,3 +1,224 @@
1
- # Temporary Holding Version
1
+ # DSH Context Policy
2
2
 
3
- This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
3
+ Percentage-based automatic compaction and model-configurable token accounting for DeepSeek Harness (DSH). The package provides Cordis services, compiled ESM modules, and TypeScript declarations.
4
+
5
+ ## Requirements
6
+
7
+ - Node.js 24 or later.
8
+ - DSH 0.2.0-rc.2.
9
+
10
+ ## Installation
11
+
12
+ Install the npm package into a DSH profile after the release is published:
13
+
14
+ ```sh
15
+ dsh plugin --profile web add @snylonue/dsh-context-policy # or your profile
16
+ ```
17
+
18
+ The bundle applies [cordis.patch.yml](./cordis.patch.yml), which replaces the global `token-meter` and `compaction-basic` entries supplied by DSH's base bundle. Its defaults apply to all model routes; it contains no provider or model bindings. DSH's presets also declare their own isolated compaction service; replace those children as described below.
19
+
20
+ Edit each affected preset's compaction plugin entry in place:
21
+
22
+ ```yaml
23
+ # Before
24
+ - id: compaction-basic
25
+ name: '@deepseek-ai/dsh-compaction-basic'
26
+
27
+ # After
28
+ - id: compaction-basic
29
+ name: '@snylonue/dsh-context-policy/compaction'
30
+ ```
31
+
32
+ Inspect the composed configuration before starting the profile:
33
+
34
+ ```sh
35
+ dsh --profile web --dump-config # or your profile
36
+ ```
37
+
38
+ ## Default behavior
39
+
40
+ Automatic pressure compaction starts when projected input tokens reach the configured fraction of the model's context window. At the default `thresholdRatio` of 0.8, a hypothetical 100000-token window triggers at 80000 tokens. Output limits and the summary token budget do not reduce the threshold.
41
+
42
+ The numeric compaction defaults match DSH's default: `thresholdRatio: 0.8`, `retainRatio: 0.16`, and `maxTokens: 65536`. This plugin calculates pressure and retention against the full context window and ignores upstream headroom deductions. The pressure retention budget defaults to 16% of the full context window. A fixed `retainTokens` budget can replace the ratio. Range selection preserves the leading system message and complete tool call/result pairs. Compaction rechecks pressure after pruning and summarization, with a configurable retry limit.
43
+
44
+ The meter uses tokenizer for accurate esitimation. `deepseek_v4` tokenizer is used by default. Provider defaults and model policies can select a different tokenizer. A successful request's provider-reported input usage calibrates subsequent custom measurements. Zero-usage failures preserve those measurements. Billing totals follow the upstream provider-usage accounting rules. The existing client pressure and composition views display the custom counts.
45
+
46
+ ## Compaction configuration
47
+
48
+ The global entry has the ID percentage-compaction. The following is a profile patch for that entry:
49
+
50
+ ```yaml
51
+ - id: percentage-compaction
52
+ config:
53
+ thresholdRatio: 0.8
54
+ retainRatio: 0.16
55
+ # retainTokens: 12000 # Use instead of retainRatio; remove retainRatio first.
56
+ maxTokens: 65536
57
+ compactionRetries: 1
58
+ maxOverflowRetries: 1
59
+ auto: true
60
+ # summarizationProvider: example-provider # Optional summarization route override.
61
+ # summarizationModel: example-model
62
+ modelPolicies: []
63
+ headroomTokens: 0 # Accepted for migration; ignored by this plugin.
64
+ ```
65
+
66
+ For an isolated preset, place the configuration on its compaction child. Values below describe the automatic pressure policy unless otherwise stated.
67
+
68
+ | Option | Default | Description |
69
+ | --- | --- | --- |
70
+ | `thresholdRatio` | 0.8 | Trigger fraction of the full context window; greater than 0 and at most 1. |
71
+ | `retainRatio` | 0.16 | Retention fraction of the full window; nonnegative and below thresholdRatio. |
72
+ | `retainTokens` | Unset | Fixed nonnegative retention budget. Configure this instead of retainRatio. |
73
+ | `maxTokens` | 65536 | Positive output budget for the summarization request. |
74
+ | `compactionRetries` | 1 | Additional pressure compaction attempts if pressure remains above the threshold. |
75
+ | `maxOverflowRetries` | 1 | Retry limit used by the inherited context-overflow handler. |
76
+ | `auto` | true | Enables automatic compaction listeners. |
77
+ | `summarizationProvider` | Unset | Optional provider override for summarization. |
78
+ | `summarizationModel` | Unset | Optional model override for summarization. |
79
+ | `modelPolicies` | [] | Partial overrides for exact provider/model routes. Both identifiers are required. |
80
+ | `headroomTokens` | Ignored | Accepted for migration from the upstream configuration. |
81
+
82
+ A route override can adjust pressure behavior independently:
83
+
84
+ ```yaml
85
+ modelPolicies:
86
+ - provider: example-provider
87
+ model: example-model
88
+ thresholdRatio: 0.85
89
+ retainTokens: 12000
90
+ # retainRatio: 0.16 # Use instead of retainTokens; remove retainTokens first.
91
+ maxTokens: 4096
92
+ compactionRetries: 1
93
+ maxOverflowRetries: 1
94
+ # summarizationProvider: example-summary-provider
95
+ # summarizationModel: example-summary-model
96
+ headroomTokens: 0 # Accepted for migration; ignored by this plugin.
97
+ ```
98
+
99
+ Use the provider identifier and model name registered in the target DSH installation. Fields omitted from a route override inherit the service configuration. `auto` is a service-wide option and cannot be overridden per route. The LLM adapter must supply `contextWindow` for pressure compaction.
100
+
101
+ ## Token-meter configuration
102
+
103
+ The global entry has the ID model-token-meter. A profile patch can select a tokenizer and accounting policy per route:
104
+
105
+ ```yaml
106
+ - id: model-token-meter
107
+ config:
108
+ defaultTokenizer: deepseek_v4
109
+ providerTokenizers:
110
+ example-provider: o200k_base
111
+ modelPolicies:
112
+ - provider: example-provider
113
+ model: example-model
114
+ tokenizer: o200k_base
115
+ tokensPerMessage: 3
116
+ toolsOverhead: 4
117
+ replyPrimingTokens: 3
118
+ includeReasoning: false
119
+ - model: example-model
120
+ tokenizer: o200k_base
121
+ ```
122
+
123
+ The identifiers in this example are placeholders. An exact provider/model policy takes precedence over a model-only policy. The matched policy's explicit tokenizer overrides `providerTokenizers[provider]`, which overrides `defaultTokenizer`. A matched policy that omits tokenizer inherits the provider default or the global default. With no request header, the global default applies.
124
+
125
+ | Option | Default | Description |
126
+ | --- | --- | --- |
127
+ | `defaultTokenizer` | `deepseek_v4` | Registered tokenizer used when no model or provider tokenizer applies. |
128
+ | `providerTokenizers` | {} | Provider identifier to default tokenizer mapping, applied to all models on that provider. |
129
+ | `modelPolicies` | [] | Route-specific tokenizer and accounting settings. |
130
+ | `tokensPerMessage` | 3 | Per-message framing allowance. |
131
+ | `toolsOverhead` | 4 | Allowance added when tool definitions are present. |
132
+ | `replyPrimingTokens` | 3 | Request-level reply framing allowance. |
133
+ | `includeReasoning` | false | Includes visible reasoning text in the estimated message surface. |
134
+
135
+ The last four options belong to model policy entries. Token allowances must be nonnegative integers. For duplicate routes at the same specificity, the first matching entry is used. Unknown tokenizer names fail when text is counted.
136
+
137
+ ### Builtin tokenizers
138
+
139
+ | Name | Vocabulary |
140
+ | --- | --- |
141
+ | `o200k_base` | Modern GPT vocabulary. |
142
+ | `deepseek_v4` | Official shared DeepSeek V4 Pro / Flash vocabulary and pre-tokenization rules; the default. |
143
+
144
+ DeepSeek V4 is the global default. Select GPT token counting for all models on a provider with a provider default:
145
+
146
+ ```yaml
147
+ - id: model-token-meter
148
+ config:
149
+ defaultTokenizer: deepseek_v4
150
+ providerTokenizers:
151
+ openai: o200k_base
152
+ ```
153
+
154
+ Use the provider identifiers configured in your DSH profile; `openai` is an example.
155
+
156
+ ### Custom tokenizers
157
+
158
+ Register a synchronous counter before loading a meter that selects it:
159
+
160
+ ```ts
161
+ import { registerTokenizer } from '@snylonue/dsh-context-policy/tokenizers';
162
+
163
+ export function registerModelTokenizer(encode: (text: string) => readonly number[]) {
164
+ return registerTokenizer('custom-tokenizer', text => encode(text).length);
165
+ }
166
+ ```
167
+
168
+ Call this function with the model's tokenizer implementation, then select `custom-tokenizer` in your profile. The returned function unregisters the counter; dispose dependent meters before calling it. Counters must return nonnegative safe integers. Text counts use a bounded cache, and special-token-looking input is treated as ordinary text by both builtin counters.
169
+
170
+ ## Accounting and accuracy
171
+
172
+ Compared with DSH 0.2.0-rc.2's official `dsh-token-meter` and `dsh-compaction-basic`, this package changes the following behavior:
173
+
174
+ | Area | Official DSH | This plugin |
175
+ | --- | --- | --- |
176
+ | Text and tool counting | Fixed heuristic of roughly four characters per token, with fixed overheads. | Use a configurable tokenizer. |
177
+ | Calibration baseline | Uses reported input + cache + output when the request header matches and that total is at least the local estimate for the original request and response. | Uses positive reported input + cache from a successful request when the request header matches, even if it is below the local estimate. |
178
+ | Next-request estimate | Starts from the request and response baseline, then adjusts for subsequent content changes. | Starts from the input baseline, then counts retained reply text and later content changes with the configured tokenizer. Reported output usage, which may include hidden reasoning, is not added. |
179
+ | UI estimates | Context composition uses the fixed heuristic; displayed pressure is calculated separately from route-aware measurement. | Context composition uses the configured tokenizer and adapter attachment pricing. Displayed `projectedTokens` equals the measurement used by automatic compaction. |
180
+ | Measurement details | Returns pressure and surface counts with per-node prices. | Also reports the selected `tokenizer`, `toolsTokens`, and `unpricedImages` / `unpricedFiles` flags for attachments lacking adapter pricing. |
181
+ | Automatic compaction budgets | Trigger: `floor(min(W × thresholdRatio, W − O − B))`. Ratio retention: `floor((W − O) × retainRatio)`. | Trigger: `floor(W × thresholdRatio)`. Ratio retention: `floor(W × retainRatio)`. |
182
+ | Summary output budget | When `maxTokens` is omitted, it follows the resolved `headroomTokens`. | `maxTokens` defaults to 65536 independently of `headroomTokens`. |
183
+
184
+ `W` is the model's context window, `O` is the effective request output reservation, and `B` is the official compaction headroom.
185
+
186
+ ## Module exports
187
+
188
+ | Import path | API |
189
+ | --- | --- |
190
+ | `@snylonue/dsh-context-policy` | Default Cordis plugin (`Config`, `inject`, `apply`) and named ModelTokenMeter service class. |
191
+ | `@snylonue/dsh-context-policy/token-meter` | Same plugin entry and named ModelTokenMeter service class. |
192
+ | `@snylonue/dsh-context-policy/compaction` | PercentageCompactionEngine and its default service export. |
193
+ | `@snylonue/dsh-context-policy/tokenizers` | Tokenizer registration, text counting, and policy resolution. |
194
+ | `@snylonue/dsh-context-policy/core` | Custom counting replay, measurement, and provider-input helpers. |
195
+
196
+ ## Compatibility
197
+
198
+ The compaction service extends `BasicCompactionEngine` and replaces automatic pressure range selection. Summarization, durable transactions, manual commands, and overflow listeners use the upstream implementation.
199
+
200
+ The meter uses the public Cordis service lifecycle and projection registration API. Its billing fold is maintained locally because the upstream definition is not exported at runtime. The public fixed-density estimator remains the source of durable shadow prices.
201
+
202
+ ## Development and release
203
+
204
+ From a source checkout:
205
+
206
+
207
+ ```sh
208
+ pnpm install --frozen-lockfile
209
+ pnpm run check
210
+ npm pack
211
+ ```
212
+
213
+ To audit local session logs after building:
214
+
215
+
216
+ ```sh
217
+ node scripts/audit.ts /absolute/path/to/session.v4.jsonl.zstd > audit.json
218
+ ```
219
+
220
+ The audit reports prediction errors and zero-usage occupancy preservation without including message content. It uses the default tokenizer policy and runs without live adapter pricing, so attachment results have the same estimation limits described above.
221
+
222
+ ## License
223
+
224
+ MIT. See [LICENSE](./LICENSE).
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2023 DeepSeek
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,19 @@
1
+ # DeepSeek V4 tokenizer asset
2
+
3
+ `tokenizer.json.gz` is the unmodified official tokenizer JSON compressed with gzip (level 9). Both DeepSeek V4 Pro and Flash publish identical tokenizer bytes:
4
+
5
+ - [DeepSeek-V4-Pro](https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro/blob/b5968e9190ef611bbf34a7229255be88a0e937c1/tokenizer.json), revision `b5968e9190ef611bbf34a7229255be88a0e937c1`.
6
+ - [DeepSeek-V4-Flash](https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash/blob/60d8d70770c6776ff598c94bb586a859a38244f1/tokenizer.json), revision `60d8d70770c6776ff598c94bb586a859a38244f1`.
7
+ - SHA-256 of the decompressed JSON: `8f9f37ca37fdc4f5fd36d5cf4d3b0e8392edb4e894fd10cc0d70b4957c8633cf`.
8
+ - The upstream MIT license is preserved in [LICENSE](./LICENSE).
9
+
10
+ To reproduce the asset from the repository root:
11
+
12
+ ```sh
13
+ curl -fL https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro/resolve/b5968e9190ef611bbf34a7229255be88a0e937c1/tokenizer.json -o .cache/deepseek-v4-tokenizer.json
14
+ node --input-type=module -e 'import {readFileSync,writeFileSync} from "node:fs"; import {gzipSync} from "node:zlib"; writeFileSync("assets/deepseek-v4/tokenizer.json.gz",gzipSync(readFileSync(".cache/deepseek-v4-tokenizer.json"),{level:9}));'
15
+ ```
16
+
17
+ The runtime lazily decompresses this local asset and runs the official sequence of Split and ByteLevel pre-tokenizers and BPE merges using `@huggingface/tokenizers`. It removes `added_tokens` and disables added special tokens to count literal message content as ordinary text. Message framing remains the meter's separate allowance.
18
+
19
+ The golden counts in [deepseek-v4-fixtures.json](../../tests/deepseek-v4-fixtures.json) were generated by Hugging Face Rust `tokenizers` 0.23.2: decompress the asset, set `data["added_tokens"] = []`, load `Tokenizer.from_str(json.dumps(data))`, and count `len(tokenizer.encode(text, add_special_tokens=False).ids)` for each fixture text.
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2022 OpenAI, Shantanu Jain
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,20 @@
1
+ # o200k_base tokenizer asset
2
+
3
+ Both builtin counters use `@huggingface/tokenizers`. This GPT asset is based on [Xenova/gpt-4o](https://huggingface.co/Xenova/gpt-4o/blob/7956d98f2a83b2751a98ea7136fdf7fe6cf54e69/tokenizer.json), pinned to revision `7956d98f2a83b2751a98ea7136fdf7fe6cf54e69`.
4
+
5
+ - Source JSON SHA-256: `43a3ad4618a6a938f8c2614154ea10c31ad53a62d5683ae0b0e6133575cef07e`.
6
+ - Bundled decompressed JSON SHA-256: `d807dc8b57ba3e3dbddb94324dcb4e359e20d83a8c8d68a73d2e8b01e70a1ace`.
7
+ - All 199998 mergeable vocabulary entries and IDs were checked against [OpenAI's official o200k_base ranks](https://openaipublic.blob.core.windows.net/encodings/o200k_base.tiktoken), SHA-256 `446a9538cb6c348e3516120d7c08b09f57c36495e2acfffe59a5bf8b0cfb1a2d`.
8
+ - The source model card specifies MIT licensing; the upstream OpenAI tiktoken license is preserved in [LICENSE](./LICENSE).
9
+
10
+ [prepare-o200k.ts](../../scripts/prepare-o200k.ts) validates the pinned source checksum, removes added tokens, enables whole-piece vocabulary lookup (`ignore_merges`), and expands whitespace shorthands into explicit ECMAScript character classes. This preserves the previous `js-tiktoken` counter's treatment of U+FEFF and U+0085 while using Hugging Face's BPE engine. No BOS/EOS tokens are added.
11
+
12
+ Reproduce the gzip asset from the repository root:
13
+
14
+ ```sh
15
+ mkdir -p .cache
16
+ curl -fL https://huggingface.co/Xenova/gpt-4o/resolve/7956d98f2a83b2751a98ea7136fdf7fe6cf54e69/tokenizer.json -o .cache/o200k-tokenizer.json
17
+ node scripts/prepare-o200k.ts .cache/o200k-tokenizer.json
18
+ ```
19
+
20
+ The golden counts in [o200k-base-fixtures.json](../../tests/o200k-base-fixtures.json) were captured from `js-tiktoken` 1.0.21 using `new Tiktoken(ranks).encode(text, [], []).length` before removing that dependency. They include multilingual text, code, literal control markers, Unicode whitespace, and lone UTF-16 surrogates. Tests use these stored reference counts and require only the current tokenizer dependency.
@@ -0,0 +1,19 @@
1
+ # Replaces the global meter and compaction service.
2
+ # Scoped preset compaction must also be replaced.
3
+ - id: token-meter
4
+ name: '@deepseek-ai/dsh-token-meter'
5
+ disabled: true
6
+ - id: compaction-basic
7
+ name: '@deepseek-ai/dsh-compaction-basic'
8
+ disabled: true
9
+ - insert:
10
+ - id: model-token-meter
11
+ name: '@snylonue/dsh-context-policy/token-meter'
12
+ config:
13
+ defaultTokenizer: deepseek_v4
14
+ - id: percentage-compaction
15
+ name: '@snylonue/dsh-context-policy/compaction'
16
+ config:
17
+ thresholdRatio: 0.8
18
+ retainRatio: 0.16
19
+ maxTokens: 65536
@@ -0,0 +1,2 @@
1
+ export declare function createTokenizerCounter(asset: string): (text: string) => number;
2
+ //# sourceMappingURL=builtin-tokenizers.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"builtin-tokenizers.d.ts","sourceRoot":"","sources":["../src/builtin-tokenizers.ts"],"names":[],"mappings":"AAKA,wBAAgB,sBAAsB,CACrC,KAAK,EAAE,MAAM,GACX,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM,CAgB1B"}
@@ -0,0 +1,15 @@
1
+ import { readFileSync } from "node:fs";
2
+ import { gunzipSync } from "node:zlib";
3
+ import { Tokenizer } from "@huggingface/tokenizers";
4
+ import { util } from "zod";
5
+ export function createTokenizerCounter(asset) {
6
+ const tokenizer = util.cached(() => {
7
+ const data = JSON.parse(gunzipSync(readFileSync(new URL(`../assets/${asset}/tokenizer.json.gz`, import.meta.url))).toString("utf8"));
8
+ // Count literal message content as ordinary text. Request framing is
9
+ // accounted for separately by the meter.
10
+ data.added_tokens = [];
11
+ return new Tokenizer(data, {});
12
+ });
13
+ return (text) => tokenizer.value.encode(text, { add_special_tokens: false }).ids.length;
14
+ }
15
+ //# sourceMappingURL=builtin-tokenizers.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"builtin-tokenizers.js","sourceRoot":"","sources":["../src/builtin-tokenizers.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AACvC,OAAO,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAC;AACpD,OAAO,EAAE,IAAI,EAAE,MAAM,KAAK,CAAC;AAE3B,MAAM,UAAU,sBAAsB,CACrC,KAAa;IAEb,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE;QAClC,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CACtB,UAAU,CACT,YAAY,CACX,IAAI,GAAG,CAAC,aAAa,KAAK,oBAAoB,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAChE,CACD,CAAC,QAAQ,CAAC,MAAM,CAAC,CAClB,CAAC;QACF,qEAAqE;QACrE,yCAAyC;QACzC,IAAI,CAAC,YAAY,GAAG,EAAE,CAAC;QACvB,OAAO,IAAI,SAAS,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;IAChC,CAAC,CAAC,CAAC;IACH,OAAO,CAAC,IAAI,EAAE,EAAE,CACf,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,kBAAkB,EAAE,KAAK,EAAE,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC;AACzE,CAAC"}
@@ -0,0 +1,43 @@
1
+ import type { Context } from "@deepseek-ai/cordis";
2
+ import Schema from "@deepseek-ai/schemastery";
3
+ import BasicCompactionEngine from "@deepseek-ai/dsh-compaction-basic";
4
+ import type { BasicCompactionConfig, ResolvedTargetPolicy, ResolvedCompactSpec } from "@deepseek-ai/dsh-compaction-basic";
5
+ import type { CompactionResult } from "@deepseek-ai/dsh-compaction";
6
+ import type { Session } from "@deepseek-ai/dsh-session";
7
+ import type { TokenMeasurement } from "@deepseek-ai/dsh-token-meter";
8
+ export interface PercentageCompactionConfig extends BasicCompactionConfig {
9
+ thresholdRatio: number;
10
+ maxTokens: number;
11
+ modelPolicies: NonNullable<BasicCompactionConfig["modelPolicies"]>;
12
+ auto: boolean;
13
+ }
14
+ type Agent = Parameters<BasicCompactionEngine["compactIfNeeded"]>[0];
15
+ type Trigger = Parameters<BasicCompactionEngine["compactIfNeeded"]>[1];
16
+ /** Both budgets are percentages of the full context window. */
17
+ export declare function resolveCompactSpec(policy: ResolvedTargetPolicy, contextWindow: number): ResolvedCompactSpec;
18
+ /** Preserve system head and whole tool-call/result groups. */
19
+ export declare function selectCompactableRange(session: Session, measurement: TokenMeasurement, retainTokens: number, systemHead: boolean): {
20
+ start: import("@deepseek-ai/dsh-session").SessionSeq;
21
+ end: import("@deepseek-ai/dsh-session").SessionSeq;
22
+ } | null;
23
+ interface CompactionEntryState {
24
+ active: boolean;
25
+ headSeq: number | null;
26
+ systemHead: boolean;
27
+ }
28
+ declare module "@deepseek-ai/dsh-session-projection" {
29
+ interface SessionProjectionStateMap {
30
+ contextPolicyCompaction: CompactionEntryState;
31
+ }
32
+ }
33
+ /** Reuse the official summarizer, transactions, commands and overflow listeners. */
34
+ export declare class PercentageCompactionEngine extends BasicCompactionEngine {
35
+ static inject: string[];
36
+ static Config: Schema<BasicCompactionConfig, PercentageCompactionConfig>;
37
+ constructor(ctx: Context, config: PercentageCompactionConfig);
38
+ private entryState;
39
+ private policy;
40
+ compactIfNeeded(agent: Agent, trigger: Trigger, signal?: AbortSignal): Promise<CompactionResult | null>;
41
+ }
42
+ export default PercentageCompactionEngine;
43
+ //# sourceMappingURL=compaction.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"compaction.d.ts","sourceRoot":"","sources":["../src/compaction.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,MAAM,MAAM,0BAA0B,CAAC;AAC9C,OAAO,qBAAqB,MAAM,mCAAmC,CAAC;AACtE,OAAO,KAAK,EACX,qBAAqB,EACrB,oBAAoB,EACpB,mBAAmB,EAEnB,MAAM,mCAAmC,CAAC;AAK3C,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,6BAA6B,CAAC;AAIpE,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAC;AACxD,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,8BAA8B,CAAC;AAErE,MAAM,WAAW,0BAA2B,SAAQ,qBAAqB;IACxE,cAAc,EAAE,MAAM,CAAC;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,aAAa,EAAE,WAAW,CAAC,qBAAqB,CAAC,eAAe,CAAC,CAAC,CAAC;IACnE,IAAI,EAAE,OAAO,CAAC;CACd;AAED,KAAK,KAAK,GAAG,UAAU,CAAC,qBAAqB,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AACrE,KAAK,OAAO,GAAG,UAAU,CAAC,qBAAqB,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAavE,+DAA+D;AAC/D,wBAAgB,kBAAkB,CACjC,MAAM,EAAE,oBAAoB,EAC5B,aAAa,EAAE,MAAM,GACnB,mBAAmB,CA0BrB;AAED,8DAA8D;AAC9D,wBAAgB,sBAAsB,CACrC,OAAO,EAAE,OAAO,EAChB,WAAW,EAAE,gBAAgB,EAC7B,YAAY,EAAE,MAAM,EACpB,UAAU,EAAE,OAAO;;;SAyBnB;AAED,UAAU,oBAAoB;IAC7B,MAAM,EAAE,OAAO,CAAC;IAChB,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,UAAU,EAAE,OAAO,CAAC;CACpB;AAED,OAAO,QAAQ,qCAAqC,CAAC;IACpD,UAAU,yBAAyB;QAClC,uBAAuB,EAAE,oBAAoB,CAAC;KAC9C;CACD;AA8BD,oFAAoF;AACpF,qBAAa,0BAA2B,SAAQ,qBAAqB;IACpE,OAAgB,MAAM,WAGpB;IACF,OAAgB,MAAM,EAAE,MAAM,CAC7B,qBAAqB,EACrB,0BAA0B,CAC1B,CAYE;gBACS,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,0BAA0B;IAY5D,OAAO,CAAC,UAAU;IASlB,OAAO,CAAC,MAAM;IA+BC,eAAe,CAC7B,KAAK,EAAE,KAAK,EACZ,OAAO,EAAE,OAAO,EAChB,MAAM,CAAC,EAAE,WAAW,GAClB,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC;CAmEnC;AACD,eAAe,0BAA0B,CAAC"}
@@ -0,0 +1,198 @@
1
+ import Schema from "@deepseek-ai/schemastery";
2
+ import BasicCompactionEngine from "@deepseek-ai/dsh-compaction-basic";
3
+ import { ManualCompactionError, toolPairingBalancedBefore, } from "@deepseek-ai/dsh-compaction";
4
+ import { isSurfaceEvent } from "@deepseek-ai/dsh-session";
5
+ import { z } from "zod";
6
+ const fields = {
7
+ thresholdRatio: Schema.number().min(0).max(1),
8
+ headroomTokens: Schema.natural(),
9
+ retainRatio: Schema.number().min(0).max(1),
10
+ retainTokens: Schema.natural(),
11
+ summarizationProvider: Schema.string(),
12
+ summarizationModel: Schema.string(),
13
+ maxTokens: Schema.natural(),
14
+ compactionRetries: Schema.natural(),
15
+ maxOverflowRetries: Schema.natural(),
16
+ };
17
+ /** Both budgets are percentages of the full context window. */
18
+ export function resolveCompactSpec(policy, contextWindow) {
19
+ if (!Number.isSafeInteger(contextWindow) || contextWindow <= 0)
20
+ throw new Error("compaction: contextWindow must be a positive safe integer");
21
+ const thresholdTokens = Math.floor(contextWindow * policy.thresholdRatio);
22
+ const retainTokens = policy.retainTokens !== undefined
23
+ ? policy.retainTokens
24
+ : Math.floor(contextWindow * policy.retainRatio);
25
+ if (thresholdTokens <= 0 || retainTokens >= thresholdTokens)
26
+ throw new Error("compaction: retain budget must be below the percentage threshold");
27
+ return Object.freeze({
28
+ target: policy.target,
29
+ contextWindow,
30
+ thresholdRatio: policy.thresholdRatio,
31
+ thresholdTokens,
32
+ retainTokens,
33
+ summarizationProvider: policy.summarizationProvider,
34
+ summarizationModel: policy.summarizationModel,
35
+ maxTokens: policy.maxTokens,
36
+ compactionRetries: policy.compactionRetries,
37
+ maxOverflowRetries: policy.maxOverflowRetries,
38
+ });
39
+ }
40
+ /** Preserve system head and whole tool-call/result groups. */
41
+ export function selectCompactableRange(session, measurement, retainTokens, systemHead) {
42
+ const nodes = measurement.nodes;
43
+ if (!nodes.length)
44
+ return null;
45
+ const surface = session.surface.nodes;
46
+ if (surface.length !== nodes.length ||
47
+ surface.some((seq, i) => seq !== nodes[i].seq))
48
+ throw new Error("compaction: token-meter surface does not match the session surface");
49
+ const first = systemHead ? 1 : 0;
50
+ let total = 0;
51
+ let keep = nodes.length;
52
+ for (let i = nodes.length - 1; i >= 0; i--) {
53
+ total += nodes[i].tokens;
54
+ keep = i;
55
+ if (total >= retainTokens)
56
+ break;
57
+ }
58
+ while (keep > first && !toolPairingBalancedBefore(session, surface[keep]))
59
+ keep--;
60
+ return keep <= first
61
+ ? null
62
+ : { start: surface[first], end: surface[keep - 1] };
63
+ }
64
+ const compactionEntry = {
65
+ key: "contextPolicyCompaction",
66
+ stateVersion: 1,
67
+ stateSchema: z.object({
68
+ active: z.boolean(),
69
+ headSeq: z.number().int().nonnegative().nullable(),
70
+ systemHead: z.boolean(),
71
+ }),
72
+ init: () => ({ active: false, headSeq: null, systemHead: false }),
73
+ apply: (state, event) => {
74
+ if (event.type === "compaction/start")
75
+ return { ...state, active: true };
76
+ if (event.type === "compaction/end" || event.type === "session/end-seed")
77
+ return state.active ? { ...state, active: false } : state;
78
+ if (isSurfaceEvent(event) &&
79
+ (state.headSeq === null ||
80
+ (event.surfaceOp !== "append" &&
81
+ event.surfaceOp.startSeq === state.headSeq)))
82
+ return {
83
+ ...state,
84
+ headSeq: event.seq,
85
+ systemHead: event.type === "system/message",
86
+ };
87
+ return state;
88
+ },
89
+ };
90
+ /** Reuse the official summarizer, transactions, commands and overflow listeners. */
91
+ export class PercentageCompactionEngine extends BasicCompactionEngine {
92
+ static inject = [
93
+ ...BasicCompactionEngine.inject,
94
+ "sessionProjections",
95
+ ];
96
+ static Config = Schema.object({
97
+ ...fields,
98
+ thresholdRatio: Schema.number().min(0).max(1).default(0.8),
99
+ maxTokens: Schema.natural().default(65536),
100
+ auto: Schema.boolean().default(true),
101
+ modelPolicies: Schema.array(Schema.object({
102
+ ...fields,
103
+ provider: Schema.string().required(),
104
+ model: Schema.string().required(),
105
+ })).default([]),
106
+ });
107
+ constructor(ctx, config) {
108
+ super(ctx, {
109
+ ...config,
110
+ headroomTokens: 0,
111
+ modelPolicies: config.modelPolicies.map((p) => ({
112
+ ...p,
113
+ headroomTokens: 0,
114
+ maxTokens: p.maxTokens ?? config.maxTokens,
115
+ })),
116
+ });
117
+ ctx.sessionProjections.register(compactionEntry);
118
+ }
119
+ entryState(session) {
120
+ const state = this.ctx.sessionProjections.stateOf(session, "contextPolicyCompaction");
121
+ if (!state)
122
+ throw new Error("compaction: entry-state projection is unavailable");
123
+ return state;
124
+ }
125
+ policy(target) {
126
+ const match = this.config.modelPolicies.find((p) => p.provider === target.provider && p.model === target.model);
127
+ let retention;
128
+ if (match?.retainTokens !== undefined)
129
+ retention = { retainTokens: match.retainTokens };
130
+ else if (match?.retainRatio !== undefined)
131
+ retention = { retainRatio: match.retainRatio };
132
+ else if (this.config.retainTokens !== undefined)
133
+ retention = { retainTokens: this.config.retainTokens };
134
+ else
135
+ retention = { retainRatio: this.config.retainRatio };
136
+ return {
137
+ thresholdRatio: match?.thresholdRatio ?? this.config.thresholdRatio,
138
+ headroomTokens: 0,
139
+ summarizationProvider: match?.summarizationProvider ?? this.config.summarizationProvider,
140
+ summarizationModel: match?.summarizationModel ?? this.config.summarizationModel,
141
+ maxTokens: match?.maxTokens ?? this.config.maxTokens,
142
+ compactionRetries: match?.compactionRetries ?? this.config.compactionRetries,
143
+ maxOverflowRetries: match?.maxOverflowRetries ?? this.config.maxOverflowRetries,
144
+ target,
145
+ ...retention,
146
+ };
147
+ }
148
+ async compactIfNeeded(agent, trigger, signal) {
149
+ const header = agent.session.requestHeader();
150
+ if (!header?.config.provider || !header.config.model)
151
+ return null;
152
+ const target = {
153
+ provider: header.config.provider,
154
+ model: header.config.model,
155
+ };
156
+ const meter = this.ctx.tokenMeter;
157
+ let measurement = meter.measure(agent.session);
158
+ const prune = this.ctx.get("toolResultPruner");
159
+ if (trigger === "context-overflow") {
160
+ prune?.pruneSession(agent.session);
161
+ measurement = meter.measure(agent.session);
162
+ const range = selectCompactableRange(agent.session, measurement, 0, this.entryState(agent.session).systemHead);
163
+ return range
164
+ ? this.compactRegion(range.start, range.end, agent, signal)
165
+ : null;
166
+ }
167
+ const info = await this.ctx.llm.resolveModelInfo(target.provider, target.model, signal);
168
+ if (this.entryState(agent.session).active)
169
+ throw new ManualCompactionError("busy", "automatic pressure compaction: compaction already in progress");
170
+ if (!info.context)
171
+ throw new Error("compaction: configure contextWindow for " +
172
+ target.provider +
173
+ "/" +
174
+ target.model);
175
+ const spec = resolveCompactSpec(this.policy(target), info.context.contextWindow);
176
+ if (measurement.totalTokens < spec.thresholdTokens)
177
+ return null;
178
+ prune?.pruneSession(agent.session);
179
+ measurement = meter.measure(agent.session);
180
+ if (measurement.totalTokens < spec.thresholdTokens)
181
+ return null;
182
+ let result = null;
183
+ for (let attempt = 0; attempt <= spec.compactionRetries; attempt++) {
184
+ const range = selectCompactableRange(agent.session, measurement, spec.retainTokens, this.entryState(agent.session).systemHead);
185
+ if (!range)
186
+ return result;
187
+ result = await this.compactRegion(range.start, range.end, agent, signal);
188
+ measurement = meter.measure(agent.session);
189
+ if (measurement.totalTokens < spec.thresholdTokens)
190
+ return result;
191
+ }
192
+ throw new Error("compaction still above percentage threshold after " +
193
+ (spec.compactionRetries + 1) +
194
+ " attempts");
195
+ }
196
+ }
197
+ export default PercentageCompactionEngine;
198
+ //# sourceMappingURL=compaction.js.map