gpt-tokenizer 1.0.0

package/esm/encoder.d.ts

@@ -0,0 +1,16 @@
+/**
+ * @returns {false | number} false if token limit is exceeded, otherwise the number of tokens
+ */
+export declare function isWithinTokenLimit(text: string, tokenLimit: number, cache?: Map<string, string>): false | number;
+export declare function encode(text: string, cache?: Map<string, string>): number[];
+export declare function decodeToken(token: number): string;
+export declare function endsWithIncompleteUtfPairSurrogate(string: string): boolean;
+export declare function decodeGenerator(tokens: Iterable<number>): Generator<string, void, undefined>;
+/**
+ * Decode tokens asynchronously and yield the decoded strings, one by one.
+ * Will not yield for tokens that include a high surrogate, but wait for the next token.
+ * @param {AsyncIterable<number>} tokens
+ * @returns {AsyncGenerator<string, void, undefined>}
+ */
+export declare function decodeAsyncGenerator(tokens: AsyncIterable<number>): AsyncGenerator<string, void, undefined>;
+export declare function decode(tokens: number[]): string;

package/esm/encoder.js

@@ -0,0 +1,188 @@
+/* eslint-disable unicorn/prefer-spread */
+/* eslint-disable unicorn/prefer-code-point */
+// eslint-disable-next-line @typescript-eslint/ban-ts-comment, @typescript-eslint/prefer-ts-expect-error
+// @ts-ignore
+import bpeRanksData from '../data/bpeRanks.json';
+// eslint-disable-next-line @typescript-eslint/ban-ts-comment, @typescript-eslint/prefer-ts-expect-error
+// @ts-ignore
+import encoderData from '../data/encoder.json';
+import { dictZip, getCharCode, getCharFromCode, getPairs, range, splitToken, } from './utils';
+const encoder = encoderData;
+const bpeRanks = bpeRanksData;
+const decoder = dictZip(Object.values(encoder), Object.keys(encoder));
+const textEncoder = new TextEncoder();
+const textDecoder = new TextDecoder();
+// eslint-disable-next-line no-magic-numbers
+const UPPER_BYTE_VALUE = 2 ** 8;
+function createByteToUnicodeMap() {
+    const asciiRange = range(getCharCode('!'), getCharCode('~') + 1);
+    const latin1Range1 = range(getCharCode('¡'), getCharCode('¬') + 1);
+    const latin1Range2 = range(getCharCode('®'), getCharCode('ÿ') + 1);
+    const initialCodePoints = [...asciiRange, ...latin1Range1, ...latin1Range2];
+    const mappedCodePoints = [...initialCodePoints];
+    let newCodePointOffset = 0;
+    for (let byteValue = 0; byteValue < UPPER_BYTE_VALUE; byteValue++) {
+        if (!initialCodePoints.includes(byteValue)) {
+            initialCodePoints.push(byteValue);
+            mappedCodePoints.push(UPPER_BYTE_VALUE + newCodePointOffset);
+            newCodePointOffset += 1;
+        }
+    }
+    const unicodeChars = mappedCodePoints.map(getCharFromCode);
+    return new Map(initialCodePoints.map((x, i) => [x, unicodeChars[i]]));
+}
+const pat = /'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+/gu;
+const byteEncoder = createByteToUnicodeMap();
+const byteDecoder = new Map([...byteEncoder].map(([k, v]) => [v, k]));
+function bpe(token, cache = new Map()) {
+    if (cache.has(token)) {
+        return cache.get(token);
+    }
+    let word = splitToken(token);
+    let pairs = getPairs(word);
+    if (pairs.length === 0) {
+        return token;
+    }
+    // eslint-disable-next-line no-constant-condition
+    while (true) {
+        const minPairs = new Map();
+        pairs.forEach((pair) => {
+            const rank = bpeRanks[pair.join(',')];
+            minPairs.set(
+            // eslint-disable-next-line no-magic-numbers
+            typeof rank === 'undefined' || Number.isNaN(rank) ? 10e10 : rank, pair);
+        });
+        const keys = [...minPairs.keys()];
+        const bigram = minPairs.get(Math.min(...keys));
+        if (!bigram || !(bigram.join(',') in bpeRanks)) {
+            break;
+        }
+        const [first, second] = bigram;
+        let newWord = [];
+        let i = 0;
+        while (i < word.length) {
+            const j = word.indexOf(first, i);
+            if (j === -1) {
+                newWord = newWord.concat(word.slice(i));
+                break;
+            }
+            newWord = newWord.concat(word.slice(i, j));
+            i = j;
+            if (word[i] === first && i < word.length - 1 && word[i + 1] === second) {
+                newWord.push(first + second);
+                i += 2;
+            }
+            else {
+                newWord.push(word[i]);
+                i += 1;
+            }
+        }
+        word = newWord;
+        if (word.length === 1) {
+            break;
+        }
+        else {
+            pairs = getPairs(word);
+        }
+    }
+    const result = word.join(' ');
+    cache.set(token, result);
+    return result;
+}
+function* encodeGenerator(text, cache = new Map()) {
+    for (let [token] of text.matchAll(pat)) {
+        token = [...textEncoder.encode(token)]
+            .map((x) => byteEncoder.get(x) ?? '')
+            .join('');
+        const newTokens = bpe(token, cache)
+            .split(' ')
+            .map((x) => encoder[x]);
+        yield newTokens;
+    }
+}
+/**
+ * @returns {false | number} false if token limit is exceeded, otherwise the number of tokens
+ */
+export function isWithinTokenLimit(text, tokenLimit, cache = new Map()) {
+    const tokenGenerator = encodeGenerator(text, cache);
+    let count = 0;
+    for (const tokens of tokenGenerator) {
+        count += tokens.length;
+        if (count > tokenLimit) {
+            return false;
+        }
+    }
+    return count;
+}
+export function encode(text, cache = new Map()) {
+    return [...encodeGenerator(text, cache)].flat(1);
+}
+export function decodeToken(token) {
+    const decodedToken = decoder[token];
+    if (typeof decodedToken === 'undefined') {
+        return '';
+    }
+    const decodedBytes = splitToken(decodedToken).map((x) => byteDecoder.get(x));
+    return textDecoder.decode(new Uint8Array(decodedBytes), {
+        stream: true,
+    });
+}
+const HIGH_SURROGATE_START = 55296;
+const HIGH_SURROGATE_END = 56319;
+export function endsWithIncompleteUtfPairSurrogate(string) {
+    if (string.length === 0)
+        return false;
+    // Check if the last character is a high surrogate
+    const lastCharCode = string.charCodeAt(string.length - 1);
+    return (lastCharCode >= HIGH_SURROGATE_START && lastCharCode <= HIGH_SURROGATE_END);
+}
+export function* decodeGenerator(tokens) {
+    let buffer = '';
+    for (const token of tokens) {
+        buffer += decodeToken(token);
+        if (buffer.length === 0 || endsWithIncompleteUtfPairSurrogate(buffer)) {
+            // Keep the high surrogate in the buffer and continue with the next token
+            // eslint-disable-next-line no-continue
+            continue;
+        }
+        else {
+            yield buffer;
+            // reset buffer
+            buffer = '';
+        }
+    }
+    // Yield any remaining characters in the buffer
+    if (buffer.length > 0) {
+        yield buffer;
+    }
+}
+/**
+ * Decode tokens asynchronously and yield the decoded strings, one by one.
+ * Will not yield for tokens that include a high surrogate, but wait for the next token.
+ * @param {AsyncIterable<number>} tokens
+ * @returns {AsyncGenerator<string, void, undefined>}
+ */
+export async function* decodeAsyncGenerator(tokens) {
+    let buffer = '';
+    for await (const token of tokens) {
+        buffer += decodeToken(token);
+        if (buffer.length === 0 || endsWithIncompleteUtfPairSurrogate(buffer)) {
+            // Keep the high surrogate in the buffer and continue with the next token
+            // eslint-disable-next-line no-continue
+            continue;
+        }
+        else {
+            yield buffer;
+            // reset buffer
+            buffer = '';
+        }
+    }
+    // Yield any remaining characters in the buffer
+    if (buffer.length > 0) {
+        yield buffer;
+    }
+}
+export function decode(tokens) {
+    return [...decodeGenerator(tokens)].join('');
+}
+//# sourceMappingURL=encoder.js.map

package/esm/encoder.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"encoder.js","sourceRoot":"","sources":["../src/encoder.ts"],"names":[],"mappings":"AAAA,0CAA0C;AAC1C,8CAA8C;AAE9C,wGAAwG;AACxG,aAAa;AACb,OAAO,YAAY,MAAM,uBAAuB,CAAA;AAChD,wGAAwG;AACxG,aAAa;AACb,OAAO,WAAW,MAAM,sBAAsB,CAAA;AAC9C,OAAO,EACL,OAAO,EACP,WAAW,EACX,eAAe,EACf,QAAQ,EACR,KAAK,EACL,UAAU,GACX,MAAM,SAAS,CAAA;AAEhB,MAAM,OAAO,GAA2B,WAAW,CAAA;AACnD,MAAM,QAAQ,GAA2B,YAAY,CAAA;AAErD,MAAM,OAAO,GAA2B,OAAO,CAC7C,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,EACtB,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CACrB,CAAA;AAED,MAAM,WAAW,GAAG,IAAI,WAAW,EAAE,CAAA;AACrC,MAAM,WAAW,GAAG,IAAI,WAAW,EAAE,CAAA;AAErC,4CAA4C;AAC5C,MAAM,gBAAgB,GAAG,CAAC,IAAI,CAAC,CAAA;AAE/B,SAAS,sBAAsB;IAC7B,MAAM,UAAU,GAAG,KAAK,CAAC,WAAW,CAAC,GAAG,CAAC,EAAE,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAA;IAChE,MAAM,YAAY,GAAG,KAAK,CAAC,WAAW,CAAC,GAAG,CAAC,EAAE,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAA;IAClE,MAAM,YAAY,GAAG,KAAK,CAAC,WAAW,CAAC,GAAG,CAAC,EAAE,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAA;IAElE,MAAM,iBAAiB,GAAG,CAAC,GAAG,UAAU,EAAE,GAAG,YAAY,EAAE,GAAG,YAAY,CAAC,CAAA;IAC3E,MAAM,gBAAgB,GAAG,CAAC,GAAG,iBAAiB,CAAC,CAAA;IAE/C,IAAI,kBAAkB,GAAG,CAAC,CAAA;IAC1B,KAAK,IAAI,SAAS,GAAG,CAAC,EAAE,SAAS,GAAG,gBAAgB,EAAE,SAAS,EAAE,EAAE;QACjE,IAAI,CAAC,iBAAiB,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE;YAC1C,iBAAiB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;YACjC,gBAAgB,CAAC,IAAI,CAAC,gBAAgB,GAAG,kBAAkB,CAAC,CAAA;YAC5D,kBAAkB,IAAI,CAAC,CAAA;SACxB;KACF;IAED,MAAM,YAAY,GAAG,gBAAgB,CAAC,GAAG,CAAC,eAAe,CAAC,CAAA;IAC1D,OAAO,IAAI,GAAG,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC,CAAE,CAAC,CAAC,CAAC,CAAA;AACxE,CAAC;AAED,MAAM,GAAG,GACP,8EAA8E,CAAA;AAEhF,MAAM,WAAW,GAAG,sBAAsB,EAAE,CAAA;AAC5C,MAAM,WAAW,GAAwB,IAAI,GAAG,CAC9C,CAAC,GAAG,WAAW,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CACzC,CAAA;AAED,SAAS,GAAG,CAAC,KAAa,EAAE,QAA6B,IAAI,GAAG,EAAE;IAChE,IAAI,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE;QACpB,OAAO,KAAK,CAAC,GAAG,CAAC,KAAK,CAAE,CAAA;KACzB;IAED,IAAI,IAAI,GAAG,UAAU,CAAC,KAAK,CAAC,CAAA;IAC5B,IAAI,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAA;IAE1B,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE;QACtB,OAAO,KAAK,CAAA;KACb;IAED,iDAAiD;IACjD,OAAO,IAAI,EAAE;QACX,MAAM,QAAQ,GAAkC,IAAI,GAAG,EAAE,CAAA;QACzD,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;YACrB,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAA;YACrC,QAAQ,CAAC,GAAG;YACV,4CAA4C;YAC5C,OAAO,IAAI,KAAK,WAAW,IAAI,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAChE,IAAI,CACL,CAAA;QACH,CAAC,CAAC,CAAA;QACF,MAAM,IAAI,GAAG,CAAC,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAA;QAEjC,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC,CAAA;QAE9C,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,QAAQ,CAAC,EAAE;YAC9C,MAAK;SACN;QAED,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,GAAG,MAAM,CAAA;QAC9B,IAAI,OAAO,GAAa,EAAE,CAAA;QAC1B,IAAI,CAAC,GAAG,CAAC,CAAA;QAET,OAAO,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE;YACtB,MAAM,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,CAAA;YAChC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE;gBACZ,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAA;gBACvC,MAAK;aACN;YACD,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;YAC1C,CAAC,GAAG,CAAC,CAAA;YAEL,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,MAAM,EAAE;gBACtE,OAAO,CAAC,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC,CAAA;gBAC5B,CAAC,IAAI,CAAC,CAAA;aACP;iBAAM;gBACL,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAE,CAAC,CAAA;gBACtB,CAAC,IAAI,CAAC,CAAA;aACP;SACF;QAED,IAAI,GAAG,OAAO,CAAA;QACd,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE;YACrB,MAAK;SACN;aAAM;YACL,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAA;SACvB;KACF;IAED,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;IAC7B,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,CAAA;IAExB,OAAO,MAAM,CAAA;AACf,CAAC;AAED,QAAQ,CAAC,CAAC,eAAe,CACvB,IAAY,EACZ,QAA6B,IAAI,GAAG,EAAE;IAEtC,KAAK,IAAI,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE;QACtC,KAAK,GAAG,CAAC,GAAG,WAAW,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;aACnC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;aACpC,IAAI,CAAC,EAAE,CAAC,CAAA;QAEX,MAAM,SAAS,GAAG,GAAG,CAAC,KAAK,EAAE,KAAK,CAAC;aAChC,KAAK,CAAC,GAAG,CAAC;aACV,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,CAAE,CAAC,CAAA;QAE1B,MAAM,SAAS,CAAA;KAChB;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,kBAAkB,CAChC,IAAY,EACZ,UAAkB,EAClB,QAA6B,IAAI,GAAG,EAAE;IAEtC,MAAM,cAAc,GAAG,eAAe,CAAC,IAAI,EAAE,KAAK,CAAC,CAAA;IACnD,IAAI,KAAK,GAAG,CAAC,CAAA;IACb,KAAK,MAAM,MAAM,IAAI,cAAc,EAAE;QACnC,KAAK,IAAI,MAAM,CAAC,MAAM,CAAA;QACtB,IAAI,KAAK,GAAG,UAAU,EAAE;YACtB,OAAO,KAAK,CAAA;SACb;KACF;IACD,OAAO,KAAK,CAAA;AACd,CAAC;AAED,MAAM,UAAU,MAAM,CACpB,IAAY,EACZ,QAA6B,IAAI,GAAG,EAAE;IAEtC,OAAO,CAAC,GAAG,eAAe,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;AAClD,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,KAAa;IACvC,MAAM,YAAY,GAAG,OAAO,CAAC,KAAK,CAAC,CAAA;IACnC,IAAI,OAAO,YAAY,KAAK,WAAW,EAAE;QACvC,OAAO,EAAE,CAAA;KACV;IACD,MAAM,YAAY,GAAG,UAAU,CAAC,YAAY,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC,CAAA;IAC7E,OAAO,WAAW,CAAC,MAAM,CAAC,IAAI,UAAU,CAAC,YAAY,CAAC,EAAE;QACtD,MAAM,EAAE,IAAI;KACb,CAAC,CAAA;AACJ,CAAC;AAED,MAAM,oBAAoB,GAAG,KAAM,CAAA;AACnC,MAAM,kBAAkB,GAAG,KAAM,CAAA;AACjC,MAAM,UAAU,kCAAkC,CAAC,MAAc;IAC/D,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,KAAK,CAAA;IACrC,kDAAkD;IAClD,MAAM,YAAY,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;IACzD,OAAO,CACL,YAAY,IAAI,oBAAoB,IAAI,YAAY,IAAI,kBAAkB,CAC3E,CAAA;AACH,CAAC;AAED,MAAM,SAAS,CAAC,CAAC,eAAe,CAC9B,MAAwB;IAExB,IAAI,MAAM,GAAG,EAAE,CAAA;IAEf,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE;QAC1B,MAAM,IAAI,WAAW,CAAC,KAAK,CAAC,CAAA;QAE5B,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,kCAAkC,CAAC,MAAM,CAAC,EAAE;YACrE,yEAAyE;YACzE,uCAAuC;YACvC,SAAQ;SACT;aAAM;YACL,MAAM,MAAM,CAAA;YACZ,eAAe;YACf,MAAM,GAAG,EAAE,CAAA;SACZ;KACF;IAED,+CAA+C;IAC/C,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE;QACrB,MAAM,MAAM,CAAA;KACb;AACH,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,oBAAoB,CACzC,MAA6B;IAE7B,IAAI,MAAM,GAAG,EAAE,CAAA;IAEf,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,MAAM,EAAE;QAChC,MAAM,IAAI,WAAW,CAAC,KAAK,CAAC,CAAA;QAE5B,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,kCAAkC,CAAC,MAAM,CAAC,EAAE;YACrE,yEAAyE;YACzE,uCAAuC;YACvC,SAAQ;SACT;aAAM;YACL,MAAM,MAAM,CAAA;YACZ,eAAe;YACf,MAAM,GAAG,EAAE,CAAA;SACZ;KACF;IAED,+CAA+C;IAC/C,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE;QACrB,MAAM,MAAM,CAAA;KACb;AACH,CAAC;AAED,MAAM,UAAU,MAAM,CAAC,MAAgB;IACrC,OAAO,CAAC,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;AAC9C,CAAC"}

package/esm/encoder.test.d.ts

@@ -0,0 +1 @@
+export {};

package/esm/encoder.test.js

@@ -0,0 +1,84 @@
+/* eslint-disable compat/compat */
+import { decode, decodeAsyncGenerator, decodeGenerator, encode, isWithinTokenLimit, } from './encoder';
+describe('basic functionality', () => {
+    it('empty string', () => {
+        const str = '';
+        expect(encode(str)).toEqual([]);
+        expect(decode(encode(str))).toEqual(str);
+        expect(isWithinTokenLimit(str, 0)).toBe(0);
+        expect(isWithinTokenLimit(str, 3)).toBe(0);
+    });
+    it('space', () => {
+        const str = ' ';
+        expect(encode(str)).toEqual([220]);
+        expect(decode(encode(str))).toEqual(str);
+        expect(isWithinTokenLimit(str, 3)).toBe(1);
+        expect(isWithinTokenLimit(str, 0)).toBe(false);
+    });
+    it('tab', () => {
+        const str = '\t';
+        expect(encode(str)).toEqual([197]);
+        expect(decode(encode(str))).toEqual(str);
+    });
+    it('simple text', () => {
+        const str = 'This is some text';
+        expect(encode(str)).toEqual([1212, 318, 617, 2420]);
+        expect(decode(encode(str))).toEqual(str);
+        expect(isWithinTokenLimit(str, 3)).toBe(false);
+        expect(isWithinTokenLimit(str, 5)).toBe(4);
+    });
+    it('multi-token word', () => {
+        const str = 'indivisible';
+        expect(encode(str)).toEqual([521, 452, 12843]);
+        expect(decode(encode(str))).toEqual(str);
+        expect(isWithinTokenLimit(str, 3)).toBe(3);
+    });
+    const helloWorldTokens = [31373, 50169, 233, 995, 12520, 234, 235];
+    it('emojis', () => {
+        const str = 'hello 👋 world 🌍';
+        expect(encode(str)).toEqual(helloWorldTokens);
+        expect(decode(encode(str))).toEqual(str);
+        expect(isWithinTokenLimit(str, 4)).toBe(false);
+        expect(isWithinTokenLimit(str, 400)).toBe(7);
+    });
+    it('decode token-by-token via generator', () => {
+        const generator = decodeGenerator(helloWorldTokens);
+        expect(generator.next().value).toBe('hello');
+        expect(generator.next().value).toBe(' ');
+        expect(generator.next().value).toBe('👋');
+        expect(generator.next().value).toBe(' world');
+        expect(generator.next().value).toBe(' ');
+        expect(generator.next().value).toBe('🌍');
+    });
+    async function* getHelloWorldTokensAsync() {
+        for (const token of helloWorldTokens) {
+            // eslint-disable-next-line no-await-in-loop
+            yield await Promise.resolve(token);
+        }
+    }
+    it('decode token-by-token via async generator', async () => {
+        const generator = decodeAsyncGenerator(getHelloWorldTokensAsync());
+        const decoded = ['hello', ' ', '👋', ' world', ' ', '🌍'];
+        for await (const value of generator) {
+            expect(value).toEqual(decoded.shift());
+        }
+    });
+    it('properties of Object', () => {
+        const str = 'toString constructor hasOwnProperty valueOf';
+        expect(encode(str)).toEqual([
+            1462, 10100, 23772, 468, 23858, 21746, 1988, 5189,
+        ]);
+        expect(decode(encode(str))).toEqual(str);
+    });
+    it('text with commas', () => {
+        const str = 'hello, I am a text, and I have commas. a,b,c';
+        expect(decode(encode(str))).toEqual(str);
+        expect(encode(str)).toStrictEqual([
+            31373, 11, 314, 716, 257, 2420, 11, 290, 314, 423, 725, 292, 13, 257,
+            11, 65, 11, 66,
+        ]);
+        expect(isWithinTokenLimit(str, 15)).toBe(false);
+        expect(isWithinTokenLimit(str, 300)).toBe(18);
+    });
+});
+//# sourceMappingURL=encoder.test.js.map

package/esm/encoder.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"encoder.test.js","sourceRoot":"","sources":["../src/encoder.test.ts"],"names":[],"mappings":"AAAA,kCAAkC;AAClC,OAAO,EACL,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,kBAAkB,GACnB,MAAM,WAAW,CAAA;AAElB,QAAQ,CAAC,qBAAqB,EAAE,GAAG,EAAE;IACnC,EAAE,CAAC,cAAc,EAAE,GAAG,EAAE;QACtB,MAAM,GAAG,GAAG,EAAE,CAAA;QACd,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;QAC/B,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;QACxC,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QAC1C,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IAC5C,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;QACf,MAAM,GAAG,GAAG,GAAG,CAAA;QACf,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;QAClC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;QACxC,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QAC1C,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;IAChD,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;QACb,MAAM,GAAG,GAAG,IAAI,CAAA;QAChB,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;QAClC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;IAC1C,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,aAAa,EAAE,GAAG,EAAE;QACrB,MAAM,GAAG,GAAG,mBAAmB,CAAA;QAC/B,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAK,EAAE,GAAG,EAAE,GAAG,EAAE,IAAK,CAAC,CAAC,CAAA;QACrD,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;QACxC,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QAC9C,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IAC5C,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,kBAAkB,EAAE,GAAG,EAAE;QAC1B,MAAM,GAAG,GAAG,aAAa,CAAA;QACzB,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,KAAM,CAAC,CAAC,CAAA;QAC/C,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;QACxC,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IAC5C,CAAC,CAAC,CAAA;IAEF,MAAM,gBAAgB,GAAG,CAAC,KAAM,EAAE,KAAM,EAAE,GAAG,EAAE,GAAG,EAAE,KAAM,EAAE,GAAG,EAAE,GAAG,CAAC,CAAA;IAErE,EAAE,CAAC,QAAQ,EAAE,GAAG,EAAE;QAChB,MAAM,GAAG,GAAG,mBAAmB,CAAA;QAC/B,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAA;QAC7C,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;QACxC,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QAC9C,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IAC9C,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,qCAAqC,EAAE,GAAG,EAAE;QAC7C,MAAM,SAAS,GAAG,eAAe,CAAC,gBAAgB,CAAC,CAAA;QACnD,MAAM,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;QAC5C,MAAM,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QACxC,MAAM,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACzC,MAAM,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;QAC7C,MAAM,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QACxC,MAAM,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IAC3C,CAAC,CAAC,CAAA;IAEF,KAAK,SAAS,CAAC,CAAC,wBAAwB;QACtC,KAAK,MAAM,KAAK,IAAI,gBAAgB,EAAE;YACpC,4CAA4C;YAC5C,MAAM,MAAM,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;SACnC;IACH,CAAC;IAED,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;QACzD,MAAM,SAAS,GAAG,oBAAoB,CAAC,wBAAwB,EAAE,CAAC,CAAA;QAClE,MAAM,OAAO,GAAG,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,EAAE,IAAI,CAAC,CAAA;QACzD,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,SAAS,EAAE;YACnC,MAAM,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,CAAA;SACvC;IACH,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,sBAAsB,EAAE,GAAG,EAAE;QAC9B,MAAM,GAAG,GAAG,6CAA6C,CAAA;QAEzD,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC;YAC1B,IAAK,EAAE,KAAM,EAAE,KAAM,EAAE,GAAG,EAAE,KAAM,EAAE,KAAM,EAAE,IAAK,EAAE,IAAK;SACzD,CAAC,CAAA;QACF,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;IAC1C,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,kBAAkB,EAAE,GAAG,EAAE;QAC1B,MAAM,GAAG,GAAG,8CAA8C,CAAA;QAC1D,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;QACxC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,aAAa,CAAC;YAChC,KAAM,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,IAAK,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG;YACtE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE;SACf,CAAC,CAAA;QACF,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QAC/C,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IAC/C,CAAC,CAAC,CAAA;AACJ,CAAC,CAAC,CAAA"}

package/esm/getBpe.d.ts

@@ -0,0 +1 @@
+export {};

package/esm/getBpe.js

@@ -0,0 +1,12 @@
+import fs from 'fs';
+import path from 'path';
+import { dictZip, range } from './utils';
+const bpeFile = fs.readFileSync(path.join(__dirname, '../data/vocab.bpe'), 'utf8');
+const lines = bpeFile.split('\n');
+const bpeMerges = lines.slice(1, -1).map((x) => x
+    .split(/(\s+)/)
+    .filter((e) => e.trim().length > 0)
+    .join(','));
+const bpeRanks = dictZip(bpeMerges, range(0, bpeMerges.length));
+fs.writeFileSync(path.join(__dirname, '../data/bpeRanks.json'), JSON.stringify(bpeRanks));
+//# sourceMappingURL=getBpe.js.map

package/esm/getBpe.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"getBpe.js","sourceRoot":"","sources":["../src/getBpe.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,IAAI,CAAA;AACnB,OAAO,IAAI,MAAM,MAAM,CAAA;AACvB,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,SAAS,CAAA;AAExC,MAAM,OAAO,GAAG,EAAE,CAAC,YAAY,CAC7B,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,mBAAmB,CAAC,EACzC,MAAM,CACP,CAAA;AACD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;AAEjC,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAC7C,CAAC;KACE,KAAK,CAAC,OAAO,CAAC;KACd,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC;KAClC,IAAI,CAAC,GAAG,CAAC,CACb,CAAA;AAED,MAAM,QAAQ,GAAG,OAAO,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC,CAAA;AAE/D,EAAE,CAAC,aAAa,CACd,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,uBAAuB,CAAC,EAC7C,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CACzB,CAAA"}

package/esm/main.d.ts

@@ -0,0 +1 @@
+export * from './encoder';

package/esm/main.js

@@ -0,0 +1,2 @@
+export * from './encoder';
+//# sourceMappingURL=main.js.map

package/esm/main.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"main.js","sourceRoot":"","sources":["../src/main.ts"],"names":[],"mappings":"AAAA,cAAc,WAAW,CAAA"}

package/esm/utils.d.ts

@@ -0,0 +1,6 @@
+export declare function dictZip<T extends PropertyKey, U>(x: T[], y: U[]): Record<T, U>;
+export declare function range(x: number, y: number): number[];
+export declare const getCharCode: (x: string) => number;
+export declare const getCharFromCode: (x: number) => string;
+export declare function getPairs(word: string[]): [string, string][];
+export declare const splitToken: (token: string) => string[];

package/esm/utils.js

@@ -0,0 +1,28 @@
+/* eslint-disable unicorn/prefer-spread */
+/* eslint-disable unicorn/prefer-code-point */
+export function dictZip(x, y) {
+    // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
+    const result = {};
+    x.forEach((k, i) => {
+        result[k] = y[i];
+    });
+    return result;
+}
+export function range(x, y) {
+    return Array.from({ length: y - x }, (_, i) => x + i);
+}
+export const getCharCode = (x) => x.charCodeAt(0);
+export const getCharFromCode = (x) => String.fromCharCode(x);
+export function getPairs(word) {
+    /** @type {[string, string][]} */
+    const pairs = [];
+    let prevChar = word[0];
+    for (let i = 1; i < word.length; i++) {
+        const char = word[i];
+        pairs.push([prevChar, char]);
+        prevChar = char;
+    }
+    return pairs;
+}
+export const splitToken = (token) => token.split('');
+//# sourceMappingURL=utils.js.map

package/esm/utils.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"utils.js","sourceRoot":"","sources":["../src/utils.ts"],"names":[],"mappings":"AAAA,0CAA0C;AAC1C,8CAA8C;AAE9C,MAAM,UAAU,OAAO,CACrB,CAAM,EACN,CAAM;IAEN,yEAAyE;IACzE,MAAM,MAAM,GAAG,EAAkB,CAAA;IACjC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACjB,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAE,CAAA;IACnB,CAAC,CAAC,CAAA;IACF,OAAO,MAAM,CAAA;AACf,CAAC;AAED,MAAM,UAAU,KAAK,CAAC,CAAS,EAAE,CAAS;IACxC,OAAO,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;AACvD,CAAC;AAED,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,CAAS,EAAU,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAA;AAEjE,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,CAAS,EAAU,EAAE,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAA;AAE5E,MAAM,UAAU,QAAQ,CAAC,IAAc;IACrC,iCAAiC;IACjC,MAAM,KAAK,GAAuB,EAAE,CAAA;IACpC,IAAI,QAAQ,GAAG,IAAI,CAAC,CAAC,CAAC,CAAA;IACtB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;QACpC,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,CAAA;QACpB,KAAK,CAAC,IAAI,CAAC,CAAC,QAAS,EAAE,IAAK,CAAC,CAAC,CAAA;QAC9B,QAAQ,GAAG,IAAI,CAAA;KAChB;IACD,OAAO,KAAK,CAAA;AACd,CAAC;AAED,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,KAAa,EAAY,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA"}

package/package.json

@@ -0,0 +1,116 @@
+{
+  "name": "gpt-tokenizer",
+  "version": "1.0.0",
+  "description": "BPE Encoder Decoder for GPT-2 / GPT-3",
+  "keywords": [
+    "BPE",
+    "encoder",
+    "decoder",
+    "tokenizer",
+    "GPT",
+    "GPT-2",
+    "GPT-3",
+    "GPT-3.5",
+    "GPT-4",
+    "NLP",
+    "Natural Language Processing",
+    "Text Generation",
+    "OpenAI",
+    "Machine Learning",
+    "ml"
+  ],
+  "homepage": "https://github.com/niieani/gpt-tokenizer#readme",
+  "bugs": {
+    "url": "https://github.com/niieani/gpt-tokenizer/issues"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/niieani/gpt-tokenizer.git"
+  },
+  "license": "MIT",
+  "author": "Bazyli Brzoska <npm@invent.life> (https://github.com/niieani)",
+  "contributors": [
+    {
+      "name": "Nick Walton",
+      "url": "https://github.com/nickwalton"
+    },
+    {
+      "name": "Bazyli Brzoska",
+      "email": "npm@invent.life",
+      "url": "https://github.com/niieani"
+    }
+  ],
+  "exports": {
+    ".": {
+      "import": "./esm/main.js",
+      "require": "./cjs/main.js"
+    },
+    "./*": {
+      "import": "./esm/*.js",
+      "require": "./cjs/*.js"
+    },
+    "./cjs": {
+      "require": "./cjs/main.js"
+    },
+    "./cjs/*": {
+      "require": "./cjs/*.js"
+    },
+    "./esm/*": {
+      "import": "./esm/*.js"
+    },
+    "./package.json": "./package.json"
+  },
+  "main": "esm/main.js",
+  "unpkg": "dist/main.js",
+  "module": "esm/main.js",
+  "source": "src/main.ts",
+  "files": [
+    "src",
+    "cjs",
+    "esm"
+  ],
+  "scripts": {
+    "build": "yarn build:cjs && yarn build:esm && yarn build:umd",
+    "build:cjs": "yarn rrun tsc --outDir cjs --module commonjs --target es2022",
+    "build:esm": "yarn rrun tsc --outDir esm --module esnext --target es2022",
+    "build:umd": "beemo webpack --entry='./src/main.ts' --env 'outDir=dist' --env 'moduleTarget=umd' --env 'engineTarget=web' --env 'codeTarget=es2022' --env 'name=GPT3Encoder' --env 'export=default' --env 'filename=gpt3encoder.js'",
+    "clean": "git clean -dfX --exclude=node_modules src && beemo typescript:sync-project-refs",
+    "format": "yarn rrun prettier --write \"./{src,tests,.config}/**/!(*.d).{.js,jsx,ts,tsx,json,md}\"",
+    "postinstallDev": "yarn prepare",
+    "prepare": "rrun husky install .config/husky && beemo create-config",
+    "release": "beemo run-script release",
+    "test": "yarn test:format && yarn test:types && yarn test:lint && yarn test:code",
+    "test:code": "rrun jest",
+    "test:format": "yarn rrun prettier --check \"./{src,tests,.config}/**/!(*.d).{.js,jsx,ts,tsx,json,md}\"",
+    "test:lint": "rrun eslint 'src/**/*.{js,jsx,ts,tsx}'",
+    "test:types": "yarn rrun tsc --noEmit"
+  },
+  "release": {
+    "branches": [
+      "+([0-9])?(.{+([0-9]),x}).x",
+      "master",
+      {
+        "name": "main",
+        "channel": false
+      },
+      "next",
+      {
+        "name": "beta",
+        "prerelease": true
+      },
+      {
+        "name": "alpha",
+        "prerelease": true
+      }
+    ],
+    "tagFormat": "${version}"
+  },
+  "devDependencies": {
+    "@niieani/scaffold": "^1.6.7",
+    "tsx": "^3.12.6"
+  },
+  "packageManager": "yarn@3.5.0",
+  "publishConfig": {
+    "access": "public"
+  }
+}

package/src/encoder.test.ts

@@ -0,0 +1,102 @@
+/* eslint-disable compat/compat */
+import {
+  decode,
+  decodeAsyncGenerator,
+  decodeGenerator,
+  encode,
+  isWithinTokenLimit,
+} from './encoder'
+
+describe('basic functionality', () => {
+  it('empty string', () => {
+    const str = ''
+    expect(encode(str)).toEqual([])
+    expect(decode(encode(str))).toEqual(str)
+    expect(isWithinTokenLimit(str, 0)).toBe(0)
+    expect(isWithinTokenLimit(str, 3)).toBe(0)
+  })
+
+  it('space', () => {
+    const str = ' '
+    expect(encode(str)).toEqual([220])
+    expect(decode(encode(str))).toEqual(str)
+    expect(isWithinTokenLimit(str, 3)).toBe(1)
+    expect(isWithinTokenLimit(str, 0)).toBe(false)
+  })
+
+  it('tab', () => {
+    const str = '\t'
+    expect(encode(str)).toEqual([197])
+    expect(decode(encode(str))).toEqual(str)
+  })
+
+  it('simple text', () => {
+    const str = 'This is some text'
+    expect(encode(str)).toEqual([1_212, 318, 617, 2_420])
+    expect(decode(encode(str))).toEqual(str)
+    expect(isWithinTokenLimit(str, 3)).toBe(false)
+    expect(isWithinTokenLimit(str, 5)).toBe(4)
+  })
+
+  it('multi-token word', () => {
+    const str = 'indivisible'
+    expect(encode(str)).toEqual([521, 452, 12_843])
+    expect(decode(encode(str))).toEqual(str)
+    expect(isWithinTokenLimit(str, 3)).toBe(3)
+  })
+
+  const helloWorldTokens = [31_373, 50_169, 233, 995, 12_520, 234, 235]
+
+  it('emojis', () => {
+    const str = 'hello 👋 world 🌍'
+    expect(encode(str)).toEqual(helloWorldTokens)
+    expect(decode(encode(str))).toEqual(str)
+    expect(isWithinTokenLimit(str, 4)).toBe(false)
+    expect(isWithinTokenLimit(str, 400)).toBe(7)
+  })
+
+  it('decode token-by-token via generator', () => {
+    const generator = decodeGenerator(helloWorldTokens)
+    expect(generator.next().value).toBe('hello')
+    expect(generator.next().value).toBe(' ')
+    expect(generator.next().value).toBe('👋')
+    expect(generator.next().value).toBe(' world')
+    expect(generator.next().value).toBe(' ')
+    expect(generator.next().value).toBe('🌍')
+  })
+
+  async function* getHelloWorldTokensAsync() {
+    for (const token of helloWorldTokens) {
+      // eslint-disable-next-line no-await-in-loop
+      yield await Promise.resolve(token)
+    }
+  }
+
+  it('decode token-by-token via async generator', async () => {
+    const generator = decodeAsyncGenerator(getHelloWorldTokensAsync())
+    const decoded = ['hello', ' ', '👋', ' world', ' ', '🌍']
+    for await (const value of generator) {
+      expect(value).toEqual(decoded.shift())
+    }
+  })
+
+  it('properties of Object', () => {
+    const str = 'toString constructor hasOwnProperty valueOf'
+
+    expect(encode(str)).toEqual([
+      1_462, 10_100, 23_772, 468, 23_858, 21_746, 1_988, 5_189,
+    ])
+    expect(decode(encode(str))).toEqual(str)
+  })
+
+  it('text with commas', () => {
+    const str = 'hello, I am a text, and I have commas. a,b,c'
+    expect(decode(encode(str))).toEqual(str)
+    expect(encode(str)).toStrictEqual([
+      31_373, 11, 314, 716, 257, 2_420, 11, 290, 314, 423, 725, 292, 13, 257,
+      11, 65, 11, 66,
+    ])
+    expect(isWithinTokenLimit(str, 15)).toBe(false)
+    expect(isWithinTokenLimit(str, 300)).toBe(18)
+  })
+})