@snylonue/dsh-context-policy 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +223 -2
- package/assets/deepseek-v4/LICENSE +21 -0
- package/assets/deepseek-v4/README.md +19 -0
- package/assets/deepseek-v4/tokenizer.json.gz +0 -0
- package/assets/o200k-base/LICENSE +21 -0
- package/assets/o200k-base/README.md +20 -0
- package/assets/o200k-base/tokenizer.json.gz +0 -0
- package/cordis.patch.yml +19 -0
- package/dist/builtin-tokenizers.d.ts +2 -0
- package/dist/builtin-tokenizers.d.ts.map +1 -0
- package/dist/builtin-tokenizers.js +15 -0
- package/dist/builtin-tokenizers.js.map +1 -0
- package/dist/compaction.d.ts +43 -0
- package/dist/compaction.d.ts.map +1 -0
- package/dist/compaction.js +198 -0
- package/dist/compaction.js.map +1 -0
- package/dist/core.d.ts +49 -0
- package/dist/core.d.ts.map +1 -0
- package/dist/core.js +213 -0
- package/dist/core.js.map +1 -0
- package/dist/projection.d.ts +103 -0
- package/dist/projection.d.ts.map +1 -0
- package/dist/projection.js +137 -0
- package/dist/projection.js.map +1 -0
- package/dist/token-meter.d.ts +46 -0
- package/dist/token-meter.d.ts.map +1 -0
- package/dist/token-meter.js +55 -0
- package/dist/token-meter.js.map +1 -0
- package/dist/tokenizers.d.ts +6 -0
- package/dist/tokenizers.d.ts.map +1 -0
- package/dist/tokenizers.js +64 -0
- package/dist/tokenizers.js.map +1 -0
- package/dist/types.d.ts +62 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +2 -0
- package/dist/types.js.map +1 -0
- package/dist/usage.d.ts +67 -0
- package/dist/usage.d.ts.map +1 -0
- package/dist/usage.js +81 -0
- package/dist/usage.js.map +1 -0
- package/examples/preset-compaction.yml +13 -0
- package/locale/en.json +6 -0
- package/locale/zh.json +6 -0
- package/package.json +92 -5
- package/scripts/audit.ts +105 -0
- package/scripts/check.ts +19 -0
- package/scripts/prepare-o200k.ts +40 -0
- package/src/builtin-tokenizers.ts +24 -0
- package/src/compaction.ts +293 -0
- package/src/core.ts +277 -0
- package/src/projection.ts +173 -0
- package/src/token-meter.ts +81 -0
- package/src/tokenizers.ts +78 -0
- package/src/types.ts +63 -0
- package/src/usage.ts +96 -0
- package/tests/bundle.test.ts +43 -0
- package/tests/deepseek-v4-fixtures.json +125 -0
- package/tests/o200k-base-fixtures.json +184 -0
- package/tests/policy.test.ts +773 -0
- package/tests/tokenizers.test.ts +51 -0
- package/tsconfig.check.json +15 -0
- package/tsconfig.json +17 -0
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
{
|
|
2
|
+
"source": "js-tiktoken",
|
|
3
|
+
"version": "1.0.21",
|
|
4
|
+
"allowedSpecial": [],
|
|
5
|
+
"disallowedSpecial": [],
|
|
6
|
+
"cases": [
|
|
7
|
+
{
|
|
8
|
+
"text": "",
|
|
9
|
+
"tokens": 0
|
|
10
|
+
},
|
|
11
|
+
{
|
|
12
|
+
"text": "hello world",
|
|
13
|
+
"tokens": 2
|
|
14
|
+
},
|
|
15
|
+
{
|
|
16
|
+
"text": "你好世界",
|
|
17
|
+
"tokens": 2
|
|
18
|
+
},
|
|
19
|
+
{
|
|
20
|
+
"text": "给 dsh-context-policy 增加 deepseek 新模型的 tokenizer 支持",
|
|
21
|
+
"tokens": 16
|
|
22
|
+
},
|
|
23
|
+
{
|
|
24
|
+
"text": "DeepSeek V4 Pro / Flash",
|
|
25
|
+
"tokens": 7
|
|
26
|
+
},
|
|
27
|
+
{
|
|
28
|
+
"text": "12345678901234567890",
|
|
29
|
+
"tokens": 7
|
|
30
|
+
},
|
|
31
|
+
{
|
|
32
|
+
"text": "123456789",
|
|
33
|
+
"tokens": 8
|
|
34
|
+
},
|
|
35
|
+
{
|
|
36
|
+
"text": "١٢٣٤٥٦٧٨٩",
|
|
37
|
+
"tokens": 9
|
|
38
|
+
},
|
|
39
|
+
{
|
|
40
|
+
"text": "一二三こんにちはカタカナ世界",
|
|
41
|
+
"tokens": 9
|
|
42
|
+
},
|
|
43
|
+
{
|
|
44
|
+
"text": "café café Å Å",
|
|
45
|
+
"tokens": 7
|
|
46
|
+
},
|
|
47
|
+
{
|
|
48
|
+
"text": "🙂🚀👩🏽💻🏳️🌈",
|
|
49
|
+
"tokens": 16
|
|
50
|
+
},
|
|
51
|
+
{
|
|
52
|
+
"text": "\u0000\t\r\n\u0001",
|
|
53
|
+
"tokens": 3
|
|
54
|
+
},
|
|
55
|
+
{
|
|
56
|
+
"text": " \t\r\n hello\n\n world ",
|
|
57
|
+
"tokens": 7
|
|
58
|
+
},
|
|
59
|
+
{
|
|
60
|
+
"text": "a b
c
d
e f",
|
|
61
|
+
"tokens": 13
|
|
62
|
+
},
|
|
63
|
+
{
|
|
64
|
+
"text": "𠀀𐐷𓂀",
|
|
65
|
+
"tokens": 11
|
|
66
|
+
},
|
|
67
|
+
{
|
|
68
|
+
"text": "Привет мир! Ελληνικά العربية हिन्दी 한국어",
|
|
69
|
+
"tokens": 10
|
|
70
|
+
},
|
|
71
|
+
{
|
|
72
|
+
"text": "const x = foo.bar(123456); // 中文注释\n",
|
|
73
|
+
"tokens": 14
|
|
74
|
+
},
|
|
75
|
+
{
|
|
76
|
+
"text": "{\"name\":\"spawn\",\"arguments\":{\"command\":\"ls\",\"args\":[\"-la\"]}}",
|
|
77
|
+
"tokens": 17
|
|
78
|
+
},
|
|
79
|
+
{
|
|
80
|
+
"text": "https://api.deepseek.com/v1/chat/completions?q=你好",
|
|
81
|
+
"tokens": 15
|
|
82
|
+
},
|
|
83
|
+
{
|
|
84
|
+
"text": "'hello .world !hello _name /tmp/file --flag=value",
|
|
85
|
+
"tokens": 14
|
|
86
|
+
},
|
|
87
|
+
{
|
|
88
|
+
"text": "<|begin▁of▁sentence|>",
|
|
89
|
+
"tokens": 11
|
|
90
|
+
},
|
|
91
|
+
{
|
|
92
|
+
"text": "<|end▁of▁sentence|>",
|
|
93
|
+
"tokens": 11
|
|
94
|
+
},
|
|
95
|
+
{
|
|
96
|
+
"text": "<|User|>你好<|Assistant|>",
|
|
97
|
+
"tokens": 11
|
|
98
|
+
},
|
|
99
|
+
{
|
|
100
|
+
"text": "<think>thought</think>",
|
|
101
|
+
"tokens": 7
|
|
102
|
+
},
|
|
103
|
+
{
|
|
104
|
+
"text": "<|tool▁calls▁begin|>{\"x\":1}<|tool▁calls▁end|>",
|
|
105
|
+
"tokens": 25
|
|
106
|
+
},
|
|
107
|
+
{
|
|
108
|
+
"text": "<|endoftext|>",
|
|
109
|
+
"tokens": 7
|
|
110
|
+
},
|
|
111
|
+
{
|
|
112
|
+
"text": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
|
113
|
+
"tokens": 125
|
|
114
|
+
},
|
|
115
|
+
{
|
|
116
|
+
"text": " ",
|
|
117
|
+
"tokens": 9
|
|
118
|
+
},
|
|
119
|
+
{
|
|
120
|
+
"text": "世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n世界123abc\n",
|
|
121
|
+
"tokens": 800
|
|
122
|
+
},
|
|
123
|
+
{
|
|
124
|
+
"text": "
हिन्दीالعربية.testこんにちは.test你好\tالعربيةहिन्दी한국어🙂
1234567こんにちは\u0000 !?カナالعربية",
|
|
125
|
+
"tokens": 34
|
|
126
|
+
},
|
|
127
|
+
{
|
|
128
|
+
"text": "你好العربية<|Assistant|>
١٢٣٤٥٦٧𠀀$name
\t世界café\u0000$name<|Assistant|>\r\ncafé\t1234567
\r\n",
|
|
129
|
+
"tokens": 44
|
|
130
|
+
},
|
|
131
|
+
{
|
|
132
|
+
"text": "<|Assistant|>1234567𠀀
العربية世界\t🙂$name한국어 café👩🏽💻한국어👩🏽💻\u0000 helloहिन्दी1234567",
|
|
133
|
+
"tokens": 51
|
|
134
|
+
},
|
|
135
|
+
{
|
|
136
|
+
"text": "👩🏽💻🙂你好<|Assistant|>한국어
𠀀こんにちは
1234567🙂1234567 hellocafé<|Assistant|>
.test$name
你好",
|
|
137
|
+
"tokens": 48
|
|
138
|
+
},
|
|
139
|
+
{
|
|
140
|
+
"text": "हिन्दी\r\n
١٢٣٤٥٦٧हिन्दी abc
$name你好١٢٣٤٥٦٧العربية世界\t hello١٢٣٤٥٦٧世界 hello한국어\t",
|
|
141
|
+
"tokens": 46
|
|
142
|
+
},
|
|
143
|
+
{
|
|
144
|
+
"text": "👩🏽💻こんにちは
カナ🙂 hello\tहिन्दीabc
한국어👩🏽💻\r\n
helloこんにちは\t\r\n \r\n",
|
|
145
|
+
"tokens": 37
|
|
146
|
+
},
|
|
147
|
+
{
|
|
148
|
+
"text": " helloこんにちは𠀀1234567한국어👩🏽💻🙂\r\n
🙂abc
.test١٢٣٤٥٦٧
こんにちは\r\n
١٢٣٤٥٦٧\r\n",
|
|
149
|
+
"tokens": 46
|
|
150
|
+
},
|
|
151
|
+
{
|
|
152
|
+
"text": "foo.bar<|Assistant|> !?
𠀀 𠀀カナ
foo.bar !?1234567caféहिन्दीこんにちはabc🙂café
",
|
|
153
|
+
"tokens": 45
|
|
154
|
+
},
|
|
155
|
+
{
|
|
156
|
+
"text": "
$nameहिन्दीfoo.bar🙂foo.bar𠀀
<|Assistant|>abc
\r\nabc1234567.test\t
café",
|
|
157
|
+
"tokens": 40
|
|
158
|
+
},
|
|
159
|
+
{
|
|
160
|
+
"text": "\r\n
カナ
🙂foo.bar👩🏽💻カナ
1234567café
🙂🙂
١٢٣٤٥٦٧हिन्दी١٢٣٤٥٦٧カナ世界",
|
|
161
|
+
"tokens": 51
|
|
162
|
+
},
|
|
163
|
+
{
|
|
164
|
+
"text": "I’m WE’RE they've CAN'T it's",
|
|
165
|
+
"tokens": 9
|
|
166
|
+
},
|
|
167
|
+
{
|
|
168
|
+
"text": "?! hello",
|
|
169
|
+
"tokens": 4
|
|
170
|
+
},
|
|
171
|
+
{
|
|
172
|
+
"text": "
?! hello",
|
|
173
|
+
"tokens": 6
|
|
174
|
+
},
|
|
175
|
+
{
|
|
176
|
+
"text": "a b
c",
|
|
177
|
+
"tokens": 6
|
|
178
|
+
},
|
|
179
|
+
{
|
|
180
|
+
"text": "�x�",
|
|
181
|
+
"tokens": 3
|
|
182
|
+
}
|
|
183
|
+
]
|
|
184
|
+
}
|