@trazum/tokenizer-openai 1.85.0-0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +74 -0
- package/dist/index.d.ts +112 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +116 -0
- package/dist/index.js.map +1 -0
- package/package.json +58 -0
- package/src/index.ts +209 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 David Muñoz Rey
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
# @trazum/tokenizer-openai
|
|
2
|
+
|
|
3
|
+
OpenAI's own tokenizer, as an optional counter for [Trazum](https://github.com/Davmunrey/Trazum).
|
|
4
|
+
|
|
5
|
+
Install it and Trazum counts OpenAI prompts **exactly** instead of estimating
|
|
6
|
+
them. Do not install it and nothing changes: `@trazum/core` declares no
|
|
7
|
+
dependencies, and every figure goes on saying which counter produced it.
|
|
8
|
+
|
|
9
|
+
```bash
|
|
10
|
+
npm install @trazum/tokenizer-openai
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
## Why it is a separate package
|
|
14
|
+
|
|
15
|
+
The rank tables are twenty-two megabytes. The zero-dependency promise in
|
|
16
|
+
`@trazum/core` is load-bearing for the CI case this product is built around --
|
|
17
|
+
**a gate that pulls a twenty-megabyte table into every build is a gate teams
|
|
18
|
+
turn off** -- so the only shape that improves the absolute figures without
|
|
19
|
+
spending that promise is a package somebody installs on purpose.
|
|
20
|
+
|
|
21
|
+
Nothing in `@trazum/core` imports this. The counter is passed in through the
|
|
22
|
+
`TokenCounter` seam that has been there since the beginning.
|
|
23
|
+
|
|
24
|
+
## What it is worth
|
|
25
|
+
|
|
26
|
+
The heuristic in `@trazum/core` is published under a ±33% band **measured
|
|
27
|
+
against Anthropic's counter**, and against OpenAI's it has been measured at
|
|
28
|
+
**112.4%** out at worst -- more than twice, on German prose. The estimator is
|
|
29
|
+
not broken; it is calibrated for a tokenizer that packs Latin text far less
|
|
30
|
+
densely than `o200k_base` does. This package removes the question.
|
|
31
|
+
|
|
32
|
+
Against the 47-sample corpus in this repository, measured through OpenAI's own
|
|
33
|
+
API with a real key, this package reproduces every count exactly: 47 of 47, to
|
|
34
|
+
the token.
|
|
35
|
+
|
|
36
|
+
## What it refuses
|
|
37
|
+
|
|
38
|
+
**A model it does not have the encoding for.** Not a nearby encoding, not the
|
|
39
|
+
newest one, not a default. `gpt-5-codex` and any model shipped since the rank
|
|
40
|
+
tables were written are refused, because a count produced by guessing which
|
|
41
|
+
table to use would go out labelled *exact*, and exact is the strongest word
|
|
42
|
+
this tool uses about a number.
|
|
43
|
+
|
|
44
|
+
**To claim it knows whose a model is.** A Claude model refuses here as
|
|
45
|
+
`unknown-encoding` rather than as *wrong family*. This package holds encodings,
|
|
46
|
+
not a catalogue.
|
|
47
|
+
|
|
48
|
+
## What the number is
|
|
49
|
+
|
|
50
|
+
The token count **of the text**. It is not the token count of a chat request:
|
|
51
|
+
an API call wraps each message in role markers and primes the reply, and those
|
|
52
|
+
tokens belong to the envelope rather than the prompt.
|
|
53
|
+
|
|
54
|
+
## Usage
|
|
55
|
+
|
|
56
|
+
```js
|
|
57
|
+
import { openaiCounter } from '@trazum/tokenizer-openai';
|
|
58
|
+
|
|
59
|
+
const counter = openaiCounter('gpt-5');
|
|
60
|
+
if (counter.ok) {
|
|
61
|
+
console.log(counter.encoding); // 'o200k_base'
|
|
62
|
+
console.log(counter.count('hello')); // 1
|
|
63
|
+
} else {
|
|
64
|
+
console.log(counter.refusal.reason); // 'unknown-encoding'
|
|
65
|
+
}
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
`canCount(model)` answers the same question without parsing a
|
|
69
|
+
two-hundred-thousand-rank table, for a caller deciding what to say before it
|
|
70
|
+
decides what to do.
|
|
71
|
+
|
|
72
|
+
## Licence
|
|
73
|
+
|
|
74
|
+
MIT. Wraps [`js-tiktoken`](https://github.com/dqbd/tiktoken), also MIT.
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* OpenAI's own tokenizer, as a counter Trazum can be handed.
|
|
3
|
+
*
|
|
4
|
+
* ## Why this is a separate package
|
|
5
|
+
*
|
|
6
|
+
* `@trazum/core` declares no dependencies and this one unpacks to twenty-two
|
|
7
|
+
* megabytes of byte-pair ranks. The zero-dependency promise is load-bearing for
|
|
8
|
+
* the CI case the whole product is built around -- **a gate that pulls a
|
|
9
|
+
* twenty-megabyte table into every build is a gate teams turn off** -- so the
|
|
10
|
+
* only shape that improves the absolute figures without spending that promise
|
|
11
|
+
* is a package somebody installs on purpose.
|
|
12
|
+
*
|
|
13
|
+
* Nothing in `@trazum/core` imports this. The core keeps working exactly as it
|
|
14
|
+
* does today when this is not installed, and the counter is passed in through
|
|
15
|
+
* the `TokenCounter` seam that has been there since the beginning.
|
|
16
|
+
*
|
|
17
|
+
* ## What it refuses
|
|
18
|
+
*
|
|
19
|
+
* **A model it does not have the encoding for.** Not a nearby encoding, not the
|
|
20
|
+
* newest one, not a default. `gpt-5-codex` and `gpt-5.5` are both refused today
|
|
21
|
+
* because the rank tables in this package's dependency do not name them, and a
|
|
22
|
+
* count produced by guessing which table to use would be the same fault as
|
|
23
|
+
* pricing a model nobody has read a rate for -- with the added harm that the
|
|
24
|
+
* figure would be labelled *exact*.
|
|
25
|
+
*
|
|
26
|
+
* **It does not claim to know whose a model is.** This package holds encodings,
|
|
27
|
+
* not a catalogue. A Claude model refuses here as `unknown-model` rather than
|
|
28
|
+
* as *wrong family*, because deciding which provider owns an id is the pricing
|
|
29
|
+
* catalogue's job and asserting it from a rank table would be a second source
|
|
30
|
+
* of truth about something already held in one place.
|
|
31
|
+
*
|
|
32
|
+
* ## Why `js-tiktoken/lite`, and not the package's main entry
|
|
33
|
+
*
|
|
34
|
+
* The main entry inlines every rank table into one file, which is 5.6 MB with a
|
|
35
|
+
* single line 2.3 million characters long. Socket's scanner reads that as
|
|
36
|
+
* **90% likely obfuscated** and flagged it on the pull request that added this
|
|
37
|
+
* package, which is a fair reading of the shape even though the content is a
|
|
38
|
+
* byte-pair vocabulary rather than hidden code.
|
|
39
|
+
*
|
|
40
|
+
* `lite` separates the two. The executable code it loads is `chunk-*.js`, whose
|
|
41
|
+
* longest line is 160 characters and which reads as ordinary source, and the
|
|
42
|
+
* ranks arrive as data from a separate module chosen by encoding. So the answer
|
|
43
|
+
* to *is this obfuscated* stops being a judgement about a heuristic and becomes
|
|
44
|
+
* a property of the file: `security.test.js` asserts the line length of what is
|
|
45
|
+
* loaded, and the vocabulary is data that no longer sits in the same file as
|
|
46
|
+
* the code.
|
|
47
|
+
*
|
|
48
|
+
* It is also simply less work: one table is parsed instead of six, and only the
|
|
49
|
+
* one the model actually uses.
|
|
50
|
+
*
|
|
51
|
+
* ## What the number is, exactly
|
|
52
|
+
*
|
|
53
|
+
* The token count **of the text**, which is what Trazum measures everywhere
|
|
54
|
+
* else and therefore what makes this comparable with the heuristic it replaces.
|
|
55
|
+
* It is not the token count of a chat request: an API call wraps each message
|
|
56
|
+
* in role markers and primes the reply, and those tokens are the envelope's
|
|
57
|
+
* rather than the prompt's. A reader billing from this should know the envelope
|
|
58
|
+
* is not in it, which is why the sentence is here rather than in a footnote.
|
|
59
|
+
*/
|
|
60
|
+
/** Why a model cannot be counted here. */
|
|
61
|
+
export type CounterRefusal = {
|
|
62
|
+
reason: 'unknown-encoding';
|
|
63
|
+
model: string;
|
|
64
|
+
/**
|
|
65
|
+
* The encodings this package does hold, so a refusal names what is missing
|
|
66
|
+
* rather than only that something is.
|
|
67
|
+
*/
|
|
68
|
+
known: readonly string[];
|
|
69
|
+
};
|
|
70
|
+
export type CounterResult = {
|
|
71
|
+
ok: true;
|
|
72
|
+
/** Tokens in the text, by OpenAI's own byte-pair encoding. */
|
|
73
|
+
count: (text: string) => number;
|
|
74
|
+
/** Which rank table produced the count, as provenance. */
|
|
75
|
+
encoding: string;
|
|
76
|
+
model: string;
|
|
77
|
+
} | {
|
|
78
|
+
ok: false;
|
|
79
|
+
refusal: CounterRefusal;
|
|
80
|
+
};
|
|
81
|
+
/**
|
|
82
|
+
* The rank tables this package can load.
|
|
83
|
+
*
|
|
84
|
+
* Listed rather than derived, because a refusal that cannot say what it does
|
|
85
|
+
* hold sends somebody to read this file to find out.
|
|
86
|
+
*/
|
|
87
|
+
export declare const KNOWN_ENCODINGS: readonly ["o200k_base", "cl100k_base", "p50k_base", "p50k_edit", "r50k_base", "gpt2"];
|
|
88
|
+
/**
|
|
89
|
+
* A counter for one model, or a refusal naming what is missing.
|
|
90
|
+
*
|
|
91
|
+
* Per model rather than one counter for everything, because the encoding is a
|
|
92
|
+
* property of the model and a counter that took the model per call would let a
|
|
93
|
+
* caller mix two families' counts into one total without noticing.
|
|
94
|
+
*
|
|
95
|
+
* Asynchronous because the rank table is loaded on demand: only the one the
|
|
96
|
+
* model actually uses is ever parsed, rather than six at import time. The
|
|
97
|
+
* counter it hands back is synchronous, which is what a caller counting a
|
|
98
|
+
* directory needs -- the cost is paid once, when the encoding is chosen.
|
|
99
|
+
*
|
|
100
|
+
* `canCount` stays synchronous for the same reason: answering *could this be
|
|
101
|
+
* counted* needs the name mapping and not the table.
|
|
102
|
+
*/
|
|
103
|
+
export declare function openaiCounter(model: string): Promise<CounterResult>;
|
|
104
|
+
/**
|
|
105
|
+
* Whether this package can count a model, without building the table.
|
|
106
|
+
*
|
|
107
|
+
* For a caller deciding what to say before it decides what to do -- a report
|
|
108
|
+
* naming which models it counted exactly, say -- so asking does not cost the
|
|
109
|
+
* parse of a two-hundred-thousand-rank table.
|
|
110
|
+
*/
|
|
111
|
+
export declare function canCount(model: string): boolean;
|
|
112
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAGA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0DG;AAEH,0CAA0C;AAC1C,MAAM,MAAM,cAAc,GAAG;IAC3B,MAAM,EAAE,kBAAkB,CAAC;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd;;;OAGG;IACH,KAAK,EAAE,SAAS,MAAM,EAAE,CAAC;CAC1B,CAAC;AAEF,MAAM,MAAM,aAAa,GACrB;IACE,EAAE,EAAE,IAAI,CAAC;IACT,8DAA8D;IAC9D,KAAK,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM,CAAC;IAChC,0DAA0D;IAC1D,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;CACf,GACD;IAAE,EAAE,EAAE,KAAK,CAAC;IAAC,OAAO,EAAE,cAAc,CAAA;CAAE,CAAC;AAE3C;;;;;GAKG;AACH,eAAO,MAAM,eAAe,uFAOjB,CAAC;AAoDZ;;;;;;;;;;;;;;GAcG;AACH,wBAAsB,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC,CA4BzE;AAED;;;;;;GAMG;AACH,wBAAgB,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAM/C"}
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
import { Tiktoken, getEncodingNameForModel } from 'js-tiktoken/lite';
|
|
2
|
+
/**
|
|
3
|
+
* The rank tables this package can load.
|
|
4
|
+
*
|
|
5
|
+
* Listed rather than derived, because a refusal that cannot say what it does
|
|
6
|
+
* hold sends somebody to read this file to find out.
|
|
7
|
+
*/
|
|
8
|
+
export const KNOWN_ENCODINGS = Object.freeze([
|
|
9
|
+
'o200k_base',
|
|
10
|
+
'cl100k_base',
|
|
11
|
+
'p50k_base',
|
|
12
|
+
'p50k_edit',
|
|
13
|
+
'r50k_base',
|
|
14
|
+
'gpt2',
|
|
15
|
+
]);
|
|
16
|
+
const isKnown = (name) => KNOWN_ENCODINGS.includes(name);
|
|
17
|
+
/*
|
|
18
|
+
* One table per encoding, built on first use and kept.
|
|
19
|
+
*
|
|
20
|
+
* Loading `o200k_base` parses a table of two hundred thousand ranks, and a
|
|
21
|
+
* caller counting a directory would otherwise pay that per file. The cache is
|
|
22
|
+
* keyed by encoding rather than by model, because several models share one.
|
|
23
|
+
*/
|
|
24
|
+
const loaded = new Map();
|
|
25
|
+
/**
|
|
26
|
+
* The rank tables, each behind its own module.
|
|
27
|
+
*
|
|
28
|
+
* Written as a literal map rather than a template string, because a dynamic
|
|
29
|
+
* specifier built at runtime is a specifier no bundler and no reader can
|
|
30
|
+
* follow -- and because it keeps `KNOWN_ENCODINGS` and what can actually be
|
|
31
|
+
* loaded in one place instead of two that agree by convention.
|
|
32
|
+
*/
|
|
33
|
+
const RANKS = Object.freeze({
|
|
34
|
+
o200k_base: () => import('js-tiktoken/ranks/o200k_base'),
|
|
35
|
+
cl100k_base: () => import('js-tiktoken/ranks/cl100k_base'),
|
|
36
|
+
p50k_base: () => import('js-tiktoken/ranks/p50k_base'),
|
|
37
|
+
p50k_edit: () => import('js-tiktoken/ranks/p50k_edit'),
|
|
38
|
+
r50k_base: () => import('js-tiktoken/ranks/r50k_base'),
|
|
39
|
+
gpt2: () => import('js-tiktoken/ranks/gpt2'),
|
|
40
|
+
});
|
|
41
|
+
/*
|
|
42
|
+
* The dependency types this argument as a union of the model ids it knows,
|
|
43
|
+
* which is narrower than what it does at runtime: it throws on anything else,
|
|
44
|
+
* and throwing is exactly the behaviour this package is built on. A caller here
|
|
45
|
+
* is passing a model id read out of somebody's usage log, so the cast is the
|
|
46
|
+
* honest shape -- the alternative is a type that says unknown models cannot
|
|
47
|
+
* reach a function whose entire purpose is refusing them.
|
|
48
|
+
*/
|
|
49
|
+
const nameFor = (model) => getEncodingNameForModel(model);
|
|
50
|
+
async function encodingFor(name) {
|
|
51
|
+
const held = loaded.get(name);
|
|
52
|
+
if (held !== undefined)
|
|
53
|
+
return held;
|
|
54
|
+
const built = new Tiktoken((await RANKS[name]()).default);
|
|
55
|
+
loaded.set(name, built);
|
|
56
|
+
return built;
|
|
57
|
+
}
|
|
58
|
+
/**
|
|
59
|
+
* A counter for one model, or a refusal naming what is missing.
|
|
60
|
+
*
|
|
61
|
+
* Per model rather than one counter for everything, because the encoding is a
|
|
62
|
+
* property of the model and a counter that took the model per call would let a
|
|
63
|
+
* caller mix two families' counts into one total without noticing.
|
|
64
|
+
*
|
|
65
|
+
* Asynchronous because the rank table is loaded on demand: only the one the
|
|
66
|
+
* model actually uses is ever parsed, rather than six at import time. The
|
|
67
|
+
* counter it hands back is synchronous, which is what a caller counting a
|
|
68
|
+
* directory needs -- the cost is paid once, when the encoding is chosen.
|
|
69
|
+
*
|
|
70
|
+
* `canCount` stays synchronous for the same reason: answering *could this be
|
|
71
|
+
* counted* needs the name mapping and not the table.
|
|
72
|
+
*/
|
|
73
|
+
export async function openaiCounter(model) {
|
|
74
|
+
let name;
|
|
75
|
+
try {
|
|
76
|
+
name = nameFor(model);
|
|
77
|
+
}
|
|
78
|
+
catch {
|
|
79
|
+
/*
|
|
80
|
+
* The dependency refuses a model it has no rule for, including models
|
|
81
|
+
* OpenAI has shipped since its rank tables were written. That refusal is
|
|
82
|
+
* kept rather than papered over with a default: the newest encoding is a
|
|
83
|
+
* reasonable guess and a guess is exactly what the word `exact` must never
|
|
84
|
+
* cover.
|
|
85
|
+
*/
|
|
86
|
+
return { ok: false, refusal: { reason: 'unknown-encoding', model, known: KNOWN_ENCODINGS } };
|
|
87
|
+
}
|
|
88
|
+
if (!isKnown(name)) {
|
|
89
|
+
// A rule pointing at a table this package does not list. Refused for the
|
|
90
|
+
// same reason, rather than attempted and thrown from inside `count`.
|
|
91
|
+
return { ok: false, refusal: { reason: 'unknown-encoding', model, known: KNOWN_ENCODINGS } };
|
|
92
|
+
}
|
|
93
|
+
const encoding = await encodingFor(name);
|
|
94
|
+
return {
|
|
95
|
+
ok: true,
|
|
96
|
+
count: (text) => encoding.encode(text).length,
|
|
97
|
+
encoding: name,
|
|
98
|
+
model,
|
|
99
|
+
};
|
|
100
|
+
}
|
|
101
|
+
/**
|
|
102
|
+
* Whether this package can count a model, without building the table.
|
|
103
|
+
*
|
|
104
|
+
* For a caller deciding what to say before it decides what to do -- a report
|
|
105
|
+
* naming which models it counted exactly, say -- so asking does not cost the
|
|
106
|
+
* parse of a two-hundred-thousand-rank table.
|
|
107
|
+
*/
|
|
108
|
+
export function canCount(model) {
|
|
109
|
+
try {
|
|
110
|
+
return isKnown(nameFor(model));
|
|
111
|
+
}
|
|
112
|
+
catch {
|
|
113
|
+
return false;
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,uBAAuB,EAAE,MAAM,kBAAkB,CAAC;AAqFrE;;;;;GAKG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG,MAAM,CAAC,MAAM,CAAC;IAC3C,YAAY;IACZ,aAAa;IACb,WAAW;IACX,WAAW;IACX,WAAW;IACX,MAAM;CACE,CAAC,CAAC;AAIZ,MAAM,OAAO,GAAG,CAAC,IAAY,EAAyB,EAAE,CACrD,eAAqC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;AAExD;;;;;;GAMG;AACH,MAAM,MAAM,GAAG,IAAI,GAAG,EAA2B,CAAC;AAElD;;;;;;;GAOG;AACH,MAAM,KAAK,GACT,MAAM,CAAC,MAAM,CAAC;IACZ,UAAU,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,8BAA8B,CAAC;IACxD,WAAW,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,+BAA+B,CAAC;IAC1D,SAAS,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,6BAA6B,CAAC;IACtD,SAAS,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,6BAA6B,CAAC;IACtD,SAAS,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,6BAA6B,CAAC;IACtD,IAAI,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,wBAAwB,CAAC;CAC7C,CAAC,CAAC;AAEL;;;;;;;GAOG;AACH,MAAM,OAAO,GAAG,CAAC,KAAa,EAAU,EAAE,CAAC,uBAAuB,CAAC,KAAsB,CAAC,CAAC;AAE3F,KAAK,UAAU,WAAW,CAAC,IAAmB;IAC5C,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAC9B,IAAI,IAAI,KAAK,SAAS;QAAE,OAAO,IAAI,CAAC;IACpC,MAAM,KAAK,GAAG,IAAI,QAAQ,CAAC,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC;IAC1D,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IACxB,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;;;;;;;;;;;GAcG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,KAAa;IAC/C,IAAI,IAAY,CAAC;IACjB,IAAI,CAAC;QACH,IAAI,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC;IACxB,CAAC;IAAC,MAAM,CAAC;QACP;;;;;;WAMG;QACH,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,MAAM,EAAE,kBAAkB,EAAE,KAAK,EAAE,KAAK,EAAE,eAAe,EAAE,EAAE,CAAC;IAC/F,CAAC;IAED,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;QACnB,yEAAyE;QACzE,qEAAqE;QACrE,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,MAAM,EAAE,kBAAkB,EAAE,KAAK,EAAE,KAAK,EAAE,eAAe,EAAE,EAAE,CAAC;IAC/F,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,WAAW,CAAC,IAAI,CAAC,CAAC;IACzC,OAAO;QACL,EAAE,EAAE,IAAI;QACR,KAAK,EAAE,CAAC,IAAY,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM;QACrD,QAAQ,EAAE,IAAI;QACd,KAAK;KACN,CAAC;AACJ,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,QAAQ,CAAC,KAAa;IACpC,IAAI,CAAC;QACH,OAAO,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;IACjC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC"}
|
package/package.json
ADDED
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@trazum/tokenizer-openai",
|
|
3
|
+
"version": "1.85.0-0",
|
|
4
|
+
"description": "OpenAI's own tokenizer as an optional counter for Trazum: exact counts for the models it knows, and a refusal for every model it does not.",
|
|
5
|
+
"license": "MIT",
|
|
6
|
+
"author": "David Muñoz Rey",
|
|
7
|
+
"repository": {
|
|
8
|
+
"type": "git",
|
|
9
|
+
"url": "git+https://github.com/Davmunrey/Trazum.git",
|
|
10
|
+
"directory": "packages/tokenizer-openai"
|
|
11
|
+
},
|
|
12
|
+
"homepage": "https://trazum.vercel.app",
|
|
13
|
+
"bugs": {
|
|
14
|
+
"url": "https://github.com/Davmunrey/Trazum/issues"
|
|
15
|
+
},
|
|
16
|
+
"keywords": [
|
|
17
|
+
"tokenizer",
|
|
18
|
+
"tiktoken",
|
|
19
|
+
"openai",
|
|
20
|
+
"tokens",
|
|
21
|
+
"cost",
|
|
22
|
+
"trazum"
|
|
23
|
+
],
|
|
24
|
+
"publishConfig": {
|
|
25
|
+
"access": "public"
|
|
26
|
+
},
|
|
27
|
+
"type": "module",
|
|
28
|
+
"main": "dist/index.js",
|
|
29
|
+
"types": "dist/index.d.ts",
|
|
30
|
+
"exports": {
|
|
31
|
+
".": {
|
|
32
|
+
"types": "./dist/index.d.ts",
|
|
33
|
+
"default": "./dist/index.js"
|
|
34
|
+
}
|
|
35
|
+
},
|
|
36
|
+
"files": [
|
|
37
|
+
"dist",
|
|
38
|
+
"src",
|
|
39
|
+
"LICENSE",
|
|
40
|
+
"README.md"
|
|
41
|
+
],
|
|
42
|
+
"scripts": {
|
|
43
|
+
"build": "tsc -p tsconfig.json",
|
|
44
|
+
"typecheck": "tsc -p tsconfig.json --noEmit",
|
|
45
|
+
"test": "npm run build && node --test test/*.test.js",
|
|
46
|
+
"prepublishOnly": "npm run build && npm test"
|
|
47
|
+
},
|
|
48
|
+
"dependencies": {
|
|
49
|
+
"js-tiktoken": "^1.0.21"
|
|
50
|
+
},
|
|
51
|
+
"devDependencies": {
|
|
52
|
+
"@types/node": "^26.2.0",
|
|
53
|
+
"typescript": "^7.0.2"
|
|
54
|
+
},
|
|
55
|
+
"engines": {
|
|
56
|
+
"node": ">=20"
|
|
57
|
+
}
|
|
58
|
+
}
|
package/src/index.ts
ADDED
|
@@ -0,0 +1,209 @@
|
|
|
1
|
+
import { Tiktoken, getEncodingNameForModel } from 'js-tiktoken/lite';
|
|
2
|
+
import type { TiktokenBPE, TiktokenModel } from 'js-tiktoken/lite';
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* OpenAI's own tokenizer, as a counter Trazum can be handed.
|
|
6
|
+
*
|
|
7
|
+
* ## Why this is a separate package
|
|
8
|
+
*
|
|
9
|
+
* `@trazum/core` declares no dependencies and this one unpacks to twenty-two
|
|
10
|
+
* megabytes of byte-pair ranks. The zero-dependency promise is load-bearing for
|
|
11
|
+
* the CI case the whole product is built around -- **a gate that pulls a
|
|
12
|
+
* twenty-megabyte table into every build is a gate teams turn off** -- so the
|
|
13
|
+
* only shape that improves the absolute figures without spending that promise
|
|
14
|
+
* is a package somebody installs on purpose.
|
|
15
|
+
*
|
|
16
|
+
* Nothing in `@trazum/core` imports this. The core keeps working exactly as it
|
|
17
|
+
* does today when this is not installed, and the counter is passed in through
|
|
18
|
+
* the `TokenCounter` seam that has been there since the beginning.
|
|
19
|
+
*
|
|
20
|
+
* ## What it refuses
|
|
21
|
+
*
|
|
22
|
+
* **A model it does not have the encoding for.** Not a nearby encoding, not the
|
|
23
|
+
* newest one, not a default. `gpt-5-codex` and `gpt-5.5` are both refused today
|
|
24
|
+
* because the rank tables in this package's dependency do not name them, and a
|
|
25
|
+
* count produced by guessing which table to use would be the same fault as
|
|
26
|
+
* pricing a model nobody has read a rate for -- with the added harm that the
|
|
27
|
+
* figure would be labelled *exact*.
|
|
28
|
+
*
|
|
29
|
+
* **It does not claim to know whose a model is.** This package holds encodings,
|
|
30
|
+
* not a catalogue. A Claude model refuses here as `unknown-model` rather than
|
|
31
|
+
* as *wrong family*, because deciding which provider owns an id is the pricing
|
|
32
|
+
* catalogue's job and asserting it from a rank table would be a second source
|
|
33
|
+
* of truth about something already held in one place.
|
|
34
|
+
*
|
|
35
|
+
* ## Why `js-tiktoken/lite`, and not the package's main entry
|
|
36
|
+
*
|
|
37
|
+
* The main entry inlines every rank table into one file, which is 5.6 MB with a
|
|
38
|
+
* single line 2.3 million characters long. Socket's scanner reads that as
|
|
39
|
+
* **90% likely obfuscated** and flagged it on the pull request that added this
|
|
40
|
+
* package, which is a fair reading of the shape even though the content is a
|
|
41
|
+
* byte-pair vocabulary rather than hidden code.
|
|
42
|
+
*
|
|
43
|
+
* `lite` separates the two. The executable code it loads is `chunk-*.js`, whose
|
|
44
|
+
* longest line is 160 characters and which reads as ordinary source, and the
|
|
45
|
+
* ranks arrive as data from a separate module chosen by encoding. So the answer
|
|
46
|
+
* to *is this obfuscated* stops being a judgement about a heuristic and becomes
|
|
47
|
+
* a property of the file: `security.test.js` asserts the line length of what is
|
|
48
|
+
* loaded, and the vocabulary is data that no longer sits in the same file as
|
|
49
|
+
* the code.
|
|
50
|
+
*
|
|
51
|
+
* It is also simply less work: one table is parsed instead of six, and only the
|
|
52
|
+
* one the model actually uses.
|
|
53
|
+
*
|
|
54
|
+
* ## What the number is, exactly
|
|
55
|
+
*
|
|
56
|
+
* The token count **of the text**, which is what Trazum measures everywhere
|
|
57
|
+
* else and therefore what makes this comparable with the heuristic it replaces.
|
|
58
|
+
* It is not the token count of a chat request: an API call wraps each message
|
|
59
|
+
* in role markers and primes the reply, and those tokens are the envelope's
|
|
60
|
+
* rather than the prompt's. A reader billing from this should know the envelope
|
|
61
|
+
* is not in it, which is why the sentence is here rather than in a footnote.
|
|
62
|
+
*/
|
|
63
|
+
|
|
64
|
+
/** Why a model cannot be counted here. */
|
|
65
|
+
export type CounterRefusal = {
|
|
66
|
+
reason: 'unknown-encoding';
|
|
67
|
+
model: string;
|
|
68
|
+
/**
|
|
69
|
+
* The encodings this package does hold, so a refusal names what is missing
|
|
70
|
+
* rather than only that something is.
|
|
71
|
+
*/
|
|
72
|
+
known: readonly string[];
|
|
73
|
+
};
|
|
74
|
+
|
|
75
|
+
export type CounterResult =
|
|
76
|
+
| {
|
|
77
|
+
ok: true;
|
|
78
|
+
/** Tokens in the text, by OpenAI's own byte-pair encoding. */
|
|
79
|
+
count: (text: string) => number;
|
|
80
|
+
/** Which rank table produced the count, as provenance. */
|
|
81
|
+
encoding: string;
|
|
82
|
+
model: string;
|
|
83
|
+
}
|
|
84
|
+
| { ok: false; refusal: CounterRefusal };
|
|
85
|
+
|
|
86
|
+
/**
|
|
87
|
+
* The rank tables this package can load.
|
|
88
|
+
*
|
|
89
|
+
* Listed rather than derived, because a refusal that cannot say what it does
|
|
90
|
+
* hold sends somebody to read this file to find out.
|
|
91
|
+
*/
|
|
92
|
+
export const KNOWN_ENCODINGS = Object.freeze([
|
|
93
|
+
'o200k_base',
|
|
94
|
+
'cl100k_base',
|
|
95
|
+
'p50k_base',
|
|
96
|
+
'p50k_edit',
|
|
97
|
+
'r50k_base',
|
|
98
|
+
'gpt2',
|
|
99
|
+
] as const);
|
|
100
|
+
|
|
101
|
+
type KnownEncoding = (typeof KNOWN_ENCODINGS)[number];
|
|
102
|
+
|
|
103
|
+
const isKnown = (name: string): name is KnownEncoding =>
|
|
104
|
+
(KNOWN_ENCODINGS as readonly string[]).includes(name);
|
|
105
|
+
|
|
106
|
+
/*
|
|
107
|
+
* One table per encoding, built on first use and kept.
|
|
108
|
+
*
|
|
109
|
+
* Loading `o200k_base` parses a table of two hundred thousand ranks, and a
|
|
110
|
+
* caller counting a directory would otherwise pay that per file. The cache is
|
|
111
|
+
* keyed by encoding rather than by model, because several models share one.
|
|
112
|
+
*/
|
|
113
|
+
const loaded = new Map<KnownEncoding, Tiktoken>();
|
|
114
|
+
|
|
115
|
+
/**
|
|
116
|
+
* The rank tables, each behind its own module.
|
|
117
|
+
*
|
|
118
|
+
* Written as a literal map rather than a template string, because a dynamic
|
|
119
|
+
* specifier built at runtime is a specifier no bundler and no reader can
|
|
120
|
+
* follow -- and because it keeps `KNOWN_ENCODINGS` and what can actually be
|
|
121
|
+
* loaded in one place instead of two that agree by convention.
|
|
122
|
+
*/
|
|
123
|
+
const RANKS: Readonly<Record<KnownEncoding, () => Promise<{ default: TiktokenBPE }>>> =
|
|
124
|
+
Object.freeze({
|
|
125
|
+
o200k_base: () => import('js-tiktoken/ranks/o200k_base'),
|
|
126
|
+
cl100k_base: () => import('js-tiktoken/ranks/cl100k_base'),
|
|
127
|
+
p50k_base: () => import('js-tiktoken/ranks/p50k_base'),
|
|
128
|
+
p50k_edit: () => import('js-tiktoken/ranks/p50k_edit'),
|
|
129
|
+
r50k_base: () => import('js-tiktoken/ranks/r50k_base'),
|
|
130
|
+
gpt2: () => import('js-tiktoken/ranks/gpt2'),
|
|
131
|
+
});
|
|
132
|
+
|
|
133
|
+
/*
|
|
134
|
+
* The dependency types this argument as a union of the model ids it knows,
|
|
135
|
+
* which is narrower than what it does at runtime: it throws on anything else,
|
|
136
|
+
* and throwing is exactly the behaviour this package is built on. A caller here
|
|
137
|
+
* is passing a model id read out of somebody's usage log, so the cast is the
|
|
138
|
+
* honest shape -- the alternative is a type that says unknown models cannot
|
|
139
|
+
* reach a function whose entire purpose is refusing them.
|
|
140
|
+
*/
|
|
141
|
+
const nameFor = (model: string): string => getEncodingNameForModel(model as TiktokenModel);
|
|
142
|
+
|
|
143
|
+
async function encodingFor(name: KnownEncoding): Promise<Tiktoken> {
|
|
144
|
+
const held = loaded.get(name);
|
|
145
|
+
if (held !== undefined) return held;
|
|
146
|
+
const built = new Tiktoken((await RANKS[name]()).default);
|
|
147
|
+
loaded.set(name, built);
|
|
148
|
+
return built;
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
/**
|
|
152
|
+
* A counter for one model, or a refusal naming what is missing.
|
|
153
|
+
*
|
|
154
|
+
* Per model rather than one counter for everything, because the encoding is a
|
|
155
|
+
* property of the model and a counter that took the model per call would let a
|
|
156
|
+
* caller mix two families' counts into one total without noticing.
|
|
157
|
+
*
|
|
158
|
+
* Asynchronous because the rank table is loaded on demand: only the one the
|
|
159
|
+
* model actually uses is ever parsed, rather than six at import time. The
|
|
160
|
+
* counter it hands back is synchronous, which is what a caller counting a
|
|
161
|
+
* directory needs -- the cost is paid once, when the encoding is chosen.
|
|
162
|
+
*
|
|
163
|
+
* `canCount` stays synchronous for the same reason: answering *could this be
|
|
164
|
+
* counted* needs the name mapping and not the table.
|
|
165
|
+
*/
|
|
166
|
+
export async function openaiCounter(model: string): Promise<CounterResult> {
|
|
167
|
+
let name: string;
|
|
168
|
+
try {
|
|
169
|
+
name = nameFor(model);
|
|
170
|
+
} catch {
|
|
171
|
+
/*
|
|
172
|
+
* The dependency refuses a model it has no rule for, including models
|
|
173
|
+
* OpenAI has shipped since its rank tables were written. That refusal is
|
|
174
|
+
* kept rather than papered over with a default: the newest encoding is a
|
|
175
|
+
* reasonable guess and a guess is exactly what the word `exact` must never
|
|
176
|
+
* cover.
|
|
177
|
+
*/
|
|
178
|
+
return { ok: false, refusal: { reason: 'unknown-encoding', model, known: KNOWN_ENCODINGS } };
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
if (!isKnown(name)) {
|
|
182
|
+
// A rule pointing at a table this package does not list. Refused for the
|
|
183
|
+
// same reason, rather than attempted and thrown from inside `count`.
|
|
184
|
+
return { ok: false, refusal: { reason: 'unknown-encoding', model, known: KNOWN_ENCODINGS } };
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
const encoding = await encodingFor(name);
|
|
188
|
+
return {
|
|
189
|
+
ok: true,
|
|
190
|
+
count: (text: string) => encoding.encode(text).length,
|
|
191
|
+
encoding: name,
|
|
192
|
+
model,
|
|
193
|
+
};
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
/**
|
|
197
|
+
* Whether this package can count a model, without building the table.
|
|
198
|
+
*
|
|
199
|
+
* For a caller deciding what to say before it decides what to do -- a report
|
|
200
|
+
* naming which models it counted exactly, say -- so asking does not cost the
|
|
201
|
+
* parse of a two-hundred-thousand-rank table.
|
|
202
|
+
*/
|
|
203
|
+
export function canCount(model: string): boolean {
|
|
204
|
+
try {
|
|
205
|
+
return isKnown(nameFor(model));
|
|
206
|
+
} catch {
|
|
207
|
+
return false;
|
|
208
|
+
}
|
|
209
|
+
}
|