@snylonue/dsh-context-policy 0.0.0-stage → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +223 -2
- package/assets/deepseek-v4/LICENSE +21 -0
- package/assets/deepseek-v4/README.md +19 -0
- package/assets/deepseek-v4/tokenizer.json.gz +0 -0
- package/assets/o200k-base/LICENSE +21 -0
- package/assets/o200k-base/README.md +20 -0
- package/assets/o200k-base/tokenizer.json.gz +0 -0
- package/cordis.patch.yml +19 -0
- package/dist/builtin-tokenizers.d.ts +2 -0
- package/dist/builtin-tokenizers.d.ts.map +1 -0
- package/dist/builtin-tokenizers.js +15 -0
- package/dist/builtin-tokenizers.js.map +1 -0
- package/dist/compaction.d.ts +43 -0
- package/dist/compaction.d.ts.map +1 -0
- package/dist/compaction.js +198 -0
- package/dist/compaction.js.map +1 -0
- package/dist/core.d.ts +49 -0
- package/dist/core.d.ts.map +1 -0
- package/dist/core.js +213 -0
- package/dist/core.js.map +1 -0
- package/dist/projection.d.ts +103 -0
- package/dist/projection.d.ts.map +1 -0
- package/dist/projection.js +137 -0
- package/dist/projection.js.map +1 -0
- package/dist/token-meter.d.ts +46 -0
- package/dist/token-meter.d.ts.map +1 -0
- package/dist/token-meter.js +55 -0
- package/dist/token-meter.js.map +1 -0
- package/dist/tokenizers.d.ts +6 -0
- package/dist/tokenizers.d.ts.map +1 -0
- package/dist/tokenizers.js +64 -0
- package/dist/tokenizers.js.map +1 -0
- package/dist/types.d.ts +62 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +2 -0
- package/dist/types.js.map +1 -0
- package/dist/usage.d.ts +67 -0
- package/dist/usage.d.ts.map +1 -0
- package/dist/usage.js +81 -0
- package/dist/usage.js.map +1 -0
- package/examples/preset-compaction.yml +13 -0
- package/locale/en.json +6 -0
- package/locale/zh.json +6 -0
- package/package.json +91 -3
- package/scripts/audit.ts +105 -0
- package/scripts/check.ts +19 -0
- package/scripts/prepare-o200k.ts +40 -0
- package/src/builtin-tokenizers.ts +24 -0
- package/src/compaction.ts +293 -0
- package/src/core.ts +277 -0
- package/src/projection.ts +173 -0
- package/src/token-meter.ts +81 -0
- package/src/tokenizers.ts +78 -0
- package/src/types.ts +63 -0
- package/src/usage.ts +96 -0
- package/tests/bundle.test.ts +43 -0
- package/tests/deepseek-v4-fixtures.json +125 -0
- package/tests/o200k-base-fixtures.json +184 -0
- package/tests/policy.test.ts +773 -0
- package/tests/tokenizers.test.ts +51 -0
- package/tsconfig.check.json +15 -0
- package/tsconfig.json +17 -0
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import test from "node:test";
|
|
2
|
+
import assert from "node:assert/strict";
|
|
3
|
+
import { readFileSync } from "node:fs";
|
|
4
|
+
import { createHash } from "node:crypto";
|
|
5
|
+
import { gunzipSync } from "node:zlib";
|
|
6
|
+
import { countText, registerTokenizer } from "../dist/tokenizers.js";
|
|
7
|
+
import fixtures from "./deepseek-v4-fixtures.json" with { type: "json" };
|
|
8
|
+
import gptFixtures from "./o200k-base-fixtures.json" with { type: "json" };
|
|
9
|
+
|
|
10
|
+
test("bundled DeepSeek V4 vocabulary matches the pinned official asset", () => {
|
|
11
|
+
const data = gunzipSync(
|
|
12
|
+
readFileSync(
|
|
13
|
+
new URL("../assets/deepseek-v4/tokenizer.json.gz", import.meta.url),
|
|
14
|
+
),
|
|
15
|
+
);
|
|
16
|
+
assert.equal(
|
|
17
|
+
createHash("sha256").update(data).digest("hex"),
|
|
18
|
+
"8f9f37ca37fdc4f5fd36d5cf4d3b0e8392edb4e894fd10cc0d70b4957c8633cf",
|
|
19
|
+
);
|
|
20
|
+
});
|
|
21
|
+
|
|
22
|
+
test("bundled o200k_base vocabulary matches the compatible converted asset", () => {
|
|
23
|
+
const data = gunzipSync(
|
|
24
|
+
readFileSync(
|
|
25
|
+
new URL("../assets/o200k-base/tokenizer.json.gz", import.meta.url),
|
|
26
|
+
),
|
|
27
|
+
);
|
|
28
|
+
assert.equal(
|
|
29
|
+
createHash("sha256").update(data).digest("hex"),
|
|
30
|
+
"d807dc8b57ba3e3dbddb94324dcb4e359e20d83a8c8d68a73d2e8b01e70a1ace",
|
|
31
|
+
);
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
// Counts captured from the previous js-tiktoken 1.0.21 implementation.
|
|
35
|
+
test("o200k_base preserves GPT counts for Unicode, code, literal markers and whitespace", () => {
|
|
36
|
+
for (const { text, tokens } of gptFixtures.cases) {
|
|
37
|
+
assert.equal(countText(text, "o200k_base"), tokens, JSON.stringify(text));
|
|
38
|
+
}
|
|
39
|
+
});
|
|
40
|
+
|
|
41
|
+
// Golden counts come from Hugging Face Rust tokenizers 0.23.2, with
|
|
42
|
+
// added_tokens removed and add_special_tokens=False (ordinary message text).
|
|
43
|
+
test("deepseek_v4 matches reference counts for multilingual text, code and control markers", () => {
|
|
44
|
+
for (const { text, tokens } of fixtures.cases) {
|
|
45
|
+
assert.equal(countText(text, "deepseek_v4"), tokens, JSON.stringify(text));
|
|
46
|
+
}
|
|
47
|
+
assert.throws(
|
|
48
|
+
() => registerTokenizer("deepseek_v4", () => 0),
|
|
49
|
+
/already registered/,
|
|
50
|
+
);
|
|
51
|
+
});
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
{
|
|
2
|
+
"extends": "./tsconfig.json",
|
|
3
|
+
"compilerOptions": {
|
|
4
|
+
"rootDir": ".",
|
|
5
|
+
"noEmit": true,
|
|
6
|
+
"resolveJsonModule": true
|
|
7
|
+
},
|
|
8
|
+
"include": [
|
|
9
|
+
"src/**/*.ts",
|
|
10
|
+
"tests/**/*.ts",
|
|
11
|
+
"scripts/check.ts",
|
|
12
|
+
"scripts/audit.ts",
|
|
13
|
+
"scripts/prepare-o200k.ts"
|
|
14
|
+
]
|
|
15
|
+
}
|
package/tsconfig.json
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
{
|
|
2
|
+
"compilerOptions": {
|
|
3
|
+
"target": "ES2023",
|
|
4
|
+
"module": "NodeNext",
|
|
5
|
+
"moduleResolution": "NodeNext",
|
|
6
|
+
"rootDir": "src",
|
|
7
|
+
"outDir": "dist",
|
|
8
|
+
"strict": true,
|
|
9
|
+
"skipLibCheck": true,
|
|
10
|
+
"declaration": true,
|
|
11
|
+
"sourceMap": true,
|
|
12
|
+
"declarationMap": true,
|
|
13
|
+
"types": ["node"],
|
|
14
|
+
"verbatimModuleSyntax": true
|
|
15
|
+
},
|
|
16
|
+
"include": ["src/**/*.ts"]
|
|
17
|
+
}
|