gpt-tokenizer 1.0.3 → 1.0.4

package/esm/getData.js

@@ -0,0 +1,16 @@
+import fs from 'fs';
+import path from 'path';
+import { dictZip, range } from './utils';
+const bpeFile = fs.readFileSync(path.join(__dirname, '../data/vocab.bpe'), 'utf8');
+const lines = bpeFile.split('\n');
+const bpeMerges = lines.slice(1, -1).map((x) => x
+    .split(/(\s+)/)
+    .filter((e) => e.trim().length > 0)
+    .join(','));
+const bpeRanks = dictZip(bpeMerges, range(0, bpeMerges.length));
+// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
+const encoder = require('../data/encoder.json');
+fs.mkdirSync(path.join(__dirname, 'data'), { recursive: true });
+fs.writeFileSync(path.join(__dirname, 'data/bpeRanks.js'), `/* eslint-disable */\n// @ts-nocheck\n// prettier-ignore\nexport default ${JSON.stringify(bpeRanks)}`);
+fs.writeFileSync(path.join(__dirname, 'data/encoder.js'), `/* eslint-disable */\n// @ts-nocheck\n// prettier-ignore\nexport default ${JSON.stringify(encoder)}`);
+//# sourceMappingURL=getData.js.map

package/esm/getData.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"getData.js","sourceRoot":"","sources":["../src/getData.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,IAAI,CAAA;AACnB,OAAO,IAAI,MAAM,MAAM,CAAA;AACvB,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,SAAS,CAAA;AAExC,MAAM,OAAO,GAAG,EAAE,CAAC,YAAY,CAC7B,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,mBAAmB,CAAC,EACzC,MAAM,CACP,CAAA;AACD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;AAEjC,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAC7C,CAAC;KACE,KAAK,CAAC,OAAO,CAAC;KACd,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC;KAClC,IAAI,CAAC,GAAG,CAAC,CACb,CAAA;AAED,MAAM,QAAQ,GAAG,OAAO,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC,CAAA;AAC/D,mEAAmE;AACnE,MAAM,OAAO,GAAW,OAAO,CAAC,sBAAsB,CAAC,CAAA;AAEvD,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;AAC/D,EAAE,CAAC,aAAa,CACd,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,kBAAkB,CAAC,EACxC,4EAA4E,IAAI,CAAC,SAAS,CACxF,QAAQ,CACT,EAAE,CACJ,CAAA;AACD,EAAE,CAAC,aAAa,CACd,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,iBAAiB,CAAC,EACvC,4EAA4E,IAAI,CAAC,SAAS,CACxF,OAAO,CACR,EAAE,CACJ,CAAA"}

package/esm/main.d.ts

@@ -1 +1,28 @@
-export * from './encoder';
+export declare function bpe(token: string, cache?: Map<string, string>): string;
+export declare function encodeGenerator(text: string, cache?: Map<string, string>): Generator<number[], void, undefined>;
+/**
+ * @returns {false | number} false if token limit is exceeded, otherwise the number of tokens
+ */
+export declare function isWithinTokenLimit(text: string, tokenLimit: number, cache?: Map<string, string>): false | number;
+export declare function encode(text: string, cache?: Map<string, string>): number[];
+export declare function decodeToken(token: number): string;
+export declare function endsWithIncompleteUtfPairSurrogate(string: string): boolean;
+export declare function decodeGenerator(tokens: Iterable<number>): Generator<string, void, undefined>;
+/**
+ * Decode tokens asynchronously and yield the decoded strings, one by one.
+ * Will not yield for tokens that include a high surrogate, but wait for the next token.
+ * @param {AsyncIterable<number>} tokens
+ * @returns {AsyncGenerator<string, void, undefined>}
+ */
+export declare function decodeAsyncGenerator(tokens: AsyncIterable<number>): AsyncGenerator<string, void, undefined>;
+export declare function decode(tokens: number[]): string;
+declare const gptEncoder: {
+    encode: typeof encode;
+    decode: typeof decode;
+    encodeGenerator: typeof encodeGenerator;
+    decodeGenerator: typeof decodeGenerator;
+    decodeAsyncGenerator: typeof decodeAsyncGenerator;
+    isWithinTokenLimit: typeof isWithinTokenLimit;
+    bpe: typeof bpe;
+};
+export default gptEncoder;

package/esm/main.js

@@ -1,2 +1,197 @@
-export * from './encoder';
+/* eslint-disable import/extensions */
+/* eslint-disable unicorn/prefer-spread */
+/* eslint-disable unicorn/prefer-code-point */
+import bpeRanksData from './data/bpeRanks.js';
+import encoderData from './data/encoder.js';
+import { dictZip, getCharCode, getCharFromCode, getPairs, range, splitToken, } from './utils.js';
+const encoder = encoderData;
+const bpeRanks = bpeRanksData;
+const decoder = dictZip(Object.values(encoder), Object.keys(encoder));
+const textEncoder = new TextEncoder();
+const textDecoder = new TextDecoder();
+// eslint-disable-next-line no-magic-numbers
+const UPPER_BYTE_VALUE = 2 ** 8;
+function createByteToUnicodeMap() {
+    const asciiRange = range(getCharCode('!'), getCharCode('~') + 1);
+    const latin1Range1 = range(getCharCode('¡'), getCharCode('¬') + 1);
+    const latin1Range2 = range(getCharCode('®'), getCharCode('ÿ') + 1);
+    const initialCodePoints = [...asciiRange, ...latin1Range1, ...latin1Range2];
+    const mappedCodePoints = [...initialCodePoints];
+    let newCodePointOffset = 0;
+    for (let byteValue = 0; byteValue < UPPER_BYTE_VALUE; byteValue++) {
+        if (!initialCodePoints.includes(byteValue)) {
+            initialCodePoints.push(byteValue);
+            mappedCodePoints.push(UPPER_BYTE_VALUE + newCodePointOffset);
+            newCodePointOffset += 1;
+        }
+    }
+    const unicodeChars = mappedCodePoints.map(getCharFromCode);
+    return new Map(initialCodePoints.map((x, i) => [x, unicodeChars[i]]));
+}
+const pat = /'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+/gu;
+const byteEncoder = createByteToUnicodeMap();
+const byteDecoder = new Map([...byteEncoder].map(([k, v]) => [v, k]));
+export function bpe(token, cache = new Map()) {
+    if (cache.has(token)) {
+        return cache.get(token);
+    }
+    let word = splitToken(token);
+    let pairs = getPairs(word);
+    if (pairs.length === 0) {
+        return token;
+    }
+    // eslint-disable-next-line no-constant-condition
+    while (true) {
+        const minPairs = new Map();
+        pairs.forEach((pair) => {
+            const rank = bpeRanks[pair.join(',')];
+            minPairs.set(
+            // eslint-disable-next-line no-magic-numbers
+            typeof rank === 'undefined' || Number.isNaN(rank) ? 10e10 : rank, pair);
+        });
+        const keys = [...minPairs.keys()];
+        const bigram = minPairs.get(Math.min(...keys));
+        if (!bigram || !(bigram.join(',') in bpeRanks)) {
+            break;
+        }
+        const [first, second] = bigram;
+        let newWord = [];
+        let i = 0;
+        while (i < word.length) {
+            const j = word.indexOf(first, i);
+            if (j === -1) {
+                newWord = newWord.concat(word.slice(i));
+                break;
+            }
+            newWord = newWord.concat(word.slice(i, j));
+            i = j;
+            if (word[i] === first && i < word.length - 1 && word[i + 1] === second) {
+                newWord.push(first + second);
+                i += 2;
+            }
+            else {
+                newWord.push(word[i]);
+                i += 1;
+            }
+        }
+        word = newWord;
+        if (word.length === 1) {
+            break;
+        }
+        else {
+            pairs = getPairs(word);
+        }
+    }
+    const result = word.join(' ');
+    cache.set(token, result);
+    return result;
+}
+export function* encodeGenerator(text, cache = new Map()) {
+    for (let [token] of text.matchAll(pat)) {
+        token = [...textEncoder.encode(token)]
+            .map((x) => byteEncoder.get(x) ?? '')
+            .join('');
+        const newTokens = bpe(token, cache)
+            .split(' ')
+            .map((x) => encoder[x]);
+        yield newTokens;
+    }
+}
+/**
+ * @returns {false | number} false if token limit is exceeded, otherwise the number of tokens
+ */
+export function isWithinTokenLimit(text, tokenLimit, cache = new Map()) {
+    const tokenGenerator = encodeGenerator(text, cache);
+    let count = 0;
+    for (const tokens of tokenGenerator) {
+        count += tokens.length;
+        if (count > tokenLimit) {
+            return false;
+        }
+    }
+    return count;
+}
+export function encode(text, cache = new Map()) {
+    return [...encodeGenerator(text, cache)].flat(1);
+}
+export function decodeToken(token) {
+    const decodedToken = decoder[token];
+    if (typeof decodedToken === 'undefined') {
+        return '';
+    }
+    const decodedBytes = splitToken(decodedToken).map((x) => byteDecoder.get(x));
+    return textDecoder.decode(new Uint8Array(decodedBytes), {
+        stream: true,
+    });
+}
+const HIGH_SURROGATE_START = 55296;
+const HIGH_SURROGATE_END = 56319;
+export function endsWithIncompleteUtfPairSurrogate(string) {
+    if (string.length === 0)
+        return false;
+    // Check if the last character is a high surrogate
+    const lastCharCode = string.charCodeAt(string.length - 1);
+    return (lastCharCode >= HIGH_SURROGATE_START && lastCharCode <= HIGH_SURROGATE_END);
+}
+export function* decodeGenerator(tokens) {
+    let buffer = '';
+    for (const token of tokens) {
+        buffer += decodeToken(token);
+        if (buffer.length === 0 || endsWithIncompleteUtfPairSurrogate(buffer)) {
+            // Keep the high surrogate in the buffer and continue with the next token
+            // eslint-disable-next-line no-continue
+            continue;
+        }
+        else {
+            yield buffer;
+            // reset buffer
+            buffer = '';
+        }
+    }
+    // Yield any remaining characters in the buffer
+    if (buffer.length > 0) {
+        yield buffer;
+    }
+}
+/**
+ * Decode tokens asynchronously and yield the decoded strings, one by one.
+ * Will not yield for tokens that include a high surrogate, but wait for the next token.
+ * @param {AsyncIterable<number>} tokens
+ * @returns {AsyncGenerator<string, void, undefined>}
+ */
+export async function* decodeAsyncGenerator(tokens) {
+    let buffer = '';
+    for await (const token of tokens) {
+        buffer += decodeToken(token);
+        if (buffer.length === 0 || endsWithIncompleteUtfPairSurrogate(buffer)) {
+            // Keep the high surrogate in the buffer and continue with the next token
+            // eslint-disable-next-line no-continue
+            continue;
+        }
+        else {
+            yield buffer;
+            // reset buffer
+            buffer = '';
+        }
+    }
+    // Yield any remaining characters in the buffer
+    if (buffer.length > 0) {
+        yield buffer;
+    }
+}
+export function decode(tokens) {
+    return [...decodeGenerator(tokens)].join('');
+}
+const gptEncoder = {
+    encode,
+    decode,
+    encodeGenerator,
+    decodeGenerator,
+    decodeAsyncGenerator,
+    isWithinTokenLimit,
+    bpe,
+};
+// used by the UMD export
+// eslint-disable-next-line import/no-default-export
+export default gptEncoder;
 //# sourceMappingURL=main.js.map

package/esm/main.js.map

@@ -1 +1 @@
-{"version":3,"file":"main.js","sourceRoot":"","sources":["../src/main.ts"],"names":[],"mappings":"AAAA,cAAc,WAAW,CAAA"}
+{"version":3,"file":"main.js","sourceRoot":"","sources":["../src/main.ts"],"names":[],"mappings":"AAAA,sCAAsC;AACtC,0CAA0C;AAC1C,8CAA8C;AAE9C,OAAO,YAAY,MAAM,oBAAoB,CAAA;AAC7C,OAAO,WAAW,MAAM,mBAAmB,CAAA;AAC3C,OAAO,EACL,OAAO,EACP,WAAW,EACX,eAAe,EACf,QAAQ,EACR,KAAK,EACL,UAAU,GACX,MAAM,YAAY,CAAA;AAEnB,MAAM,OAAO,GAA2B,WAAW,CAAA;AACnD,MAAM,QAAQ,GAA2B,YAAY,CAAA;AAErD,MAAM,OAAO,GAA2B,OAAO,CAC7C,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,EACtB,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CACrB,CAAA;AAED,MAAM,WAAW,GAAG,IAAI,WAAW,EAAE,CAAA;AACrC,MAAM,WAAW,GAAG,IAAI,WAAW,EAAE,CAAA;AAErC,4CAA4C;AAC5C,MAAM,gBAAgB,GAAG,CAAC,IAAI,CAAC,CAAA;AAE/B,SAAS,sBAAsB;IAC7B,MAAM,UAAU,GAAG,KAAK,CAAC,WAAW,CAAC,GAAG,CAAC,EAAE,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAA;IAChE,MAAM,YAAY,GAAG,KAAK,CAAC,WAAW,CAAC,GAAG,CAAC,EAAE,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAA;IAClE,MAAM,YAAY,GAAG,KAAK,CAAC,WAAW,CAAC,GAAG,CAAC,EAAE,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAA;IAElE,MAAM,iBAAiB,GAAG,CAAC,GAAG,UAAU,EAAE,GAAG,YAAY,EAAE,GAAG,YAAY,CAAC,CAAA;IAC3E,MAAM,gBAAgB,GAAG,CAAC,GAAG,iBAAiB,CAAC,CAAA;IAE/C,IAAI,kBAAkB,GAAG,CAAC,CAAA;IAC1B,KAAK,IAAI,SAAS,GAAG,CAAC,EAAE,SAAS,GAAG,gBAAgB,EAAE,SAAS,EAAE,EAAE;QACjE,IAAI,CAAC,iBAAiB,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE;YAC1C,iBAAiB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;YACjC,gBAAgB,CAAC,IAAI,CAAC,gBAAgB,GAAG,kBAAkB,CAAC,CAAA;YAC5D,kBAAkB,IAAI,CAAC,CAAA;SACxB;KACF;IAED,MAAM,YAAY,GAAG,gBAAgB,CAAC,GAAG,CAAC,eAAe,CAAC,CAAA;IAC1D,OAAO,IAAI,GAAG,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC,CAAE,CAAC,CAAC,CAAC,CAAA;AACxE,CAAC;AAED,MAAM,GAAG,GACP,8EAA8E,CAAA;AAEhF,MAAM,WAAW,GAAG,sBAAsB,EAAE,CAAA;AAC5C,MAAM,WAAW,GAAwB,IAAI,GAAG,CAC9C,CAAC,GAAG,WAAW,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CACzC,CAAA;AAED,MAAM,UAAU,GAAG,CACjB,KAAa,EACb,QAA6B,IAAI,GAAG,EAAE;IAEtC,IAAI,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE;QACpB,OAAO,KAAK,CAAC,GAAG,CAAC,KAAK,CAAE,CAAA;KACzB;IAED,IAAI,IAAI,GAAG,UAAU,CAAC,KAAK,CAAC,CAAA;IAC5B,IAAI,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAA;IAE1B,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE;QACtB,OAAO,KAAK,CAAA;KACb;IAED,iDAAiD;IACjD,OAAO,IAAI,EAAE;QACX,MAAM,QAAQ,GAAkC,IAAI,GAAG,EAAE,CAAA;QACzD,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;YACrB,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAA;YACrC,QAAQ,CAAC,GAAG;YACV,4CAA4C;YAC5C,OAAO,IAAI,KAAK,WAAW,IAAI,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAChE,IAAI,CACL,CAAA;QACH,CAAC,CAAC,CAAA;QACF,MAAM,IAAI,GAAG,CAAC,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAA;QAEjC,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC,CAAA;QAE9C,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,QAAQ,CAAC,EAAE;YAC9C,MAAK;SACN;QAED,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,GAAG,MAAM,CAAA;QAC9B,IAAI,OAAO,GAAa,EAAE,CAAA;QAC1B,IAAI,CAAC,GAAG,CAAC,CAAA;QAET,OAAO,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE;YACtB,MAAM,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,CAAA;YAChC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE;gBACZ,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAA;gBACvC,MAAK;aACN;YACD,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;YAC1C,CAAC,GAAG,CAAC,CAAA;YAEL,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,MAAM,EAAE;gBACtE,OAAO,CAAC,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC,CAAA;gBAC5B,CAAC,IAAI,CAAC,CAAA;aACP;iBAAM;gBACL,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAE,CAAC,CAAA;gBACtB,CAAC,IAAI,CAAC,CAAA;aACP;SACF;QAED,IAAI,GAAG,OAAO,CAAA;QACd,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE;YACrB,MAAK;SACN;aAAM;YACL,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAA;SACvB;KACF;IAED,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;IAC7B,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,CAAA;IAExB,OAAO,MAAM,CAAA;AACf,CAAC;AAED,MAAM,SAAS,CAAC,CAAC,eAAe,CAC9B,IAAY,EACZ,QAA6B,IAAI,GAAG,EAAE;IAEtC,KAAK,IAAI,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE;QACtC,KAAK,GAAG,CAAC,GAAG,WAAW,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;aACnC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;aACpC,IAAI,CAAC,EAAE,CAAC,CAAA;QAEX,MAAM,SAAS,GAAG,GAAG,CAAC,KAAK,EAAE,KAAK,CAAC;aAChC,KAAK,CAAC,GAAG,CAAC;aACV,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,CAAE,CAAC,CAAA;QAE1B,MAAM,SAAS,CAAA;KAChB;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,kBAAkB,CAChC,IAAY,EACZ,UAAkB,EAClB,QAA6B,IAAI,GAAG,EAAE;IAEtC,MAAM,cAAc,GAAG,eAAe,CAAC,IAAI,EAAE,KAAK,CAAC,CAAA;IACnD,IAAI,KAAK,GAAG,CAAC,CAAA;IACb,KAAK,MAAM,MAAM,IAAI,cAAc,EAAE;QACnC,KAAK,IAAI,MAAM,CAAC,MAAM,CAAA;QACtB,IAAI,KAAK,GAAG,UAAU,EAAE;YACtB,OAAO,KAAK,CAAA;SACb;KACF;IACD,OAAO,KAAK,CAAA;AACd,CAAC;AAED,MAAM,UAAU,MAAM,CACpB,IAAY,EACZ,QAA6B,IAAI,GAAG,EAAE;IAEtC,OAAO,CAAC,GAAG,eAAe,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;AAClD,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,KAAa;IACvC,MAAM,YAAY,GAAG,OAAO,CAAC,KAAK,CAAC,CAAA;IACnC,IAAI,OAAO,YAAY,KAAK,WAAW,EAAE;QACvC,OAAO,EAAE,CAAA;KACV;IACD,MAAM,YAAY,GAAG,UAAU,CAAC,YAAY,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC,CAAA;IAC7E,OAAO,WAAW,CAAC,MAAM,CAAC,IAAI,UAAU,CAAC,YAAY,CAAC,EAAE;QACtD,MAAM,EAAE,IAAI;KACb,CAAC,CAAA;AACJ,CAAC;AAED,MAAM,oBAAoB,GAAG,KAAM,CAAA;AACnC,MAAM,kBAAkB,GAAG,KAAM,CAAA;AACjC,MAAM,UAAU,kCAAkC,CAAC,MAAc;IAC/D,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,KAAK,CAAA;IACrC,kDAAkD;IAClD,MAAM,YAAY,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;IACzD,OAAO,CACL,YAAY,IAAI,oBAAoB,IAAI,YAAY,IAAI,kBAAkB,CAC3E,CAAA;AACH,CAAC;AAED,MAAM,SAAS,CAAC,CAAC,eAAe,CAC9B,MAAwB;IAExB,IAAI,MAAM,GAAG,EAAE,CAAA;IAEf,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE;QAC1B,MAAM,IAAI,WAAW,CAAC,KAAK,CAAC,CAAA;QAE5B,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,kCAAkC,CAAC,MAAM,CAAC,EAAE;YACrE,yEAAyE;YACzE,uCAAuC;YACvC,SAAQ;SACT;aAAM;YACL,MAAM,MAAM,CAAA;YACZ,eAAe;YACf,MAAM,GAAG,EAAE,CAAA;SACZ;KACF;IAED,+CAA+C;IAC/C,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE;QACrB,MAAM,MAAM,CAAA;KACb;AACH,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,oBAAoB,CACzC,MAA6B;IAE7B,IAAI,MAAM,GAAG,EAAE,CAAA;IAEf,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,MAAM,EAAE;QAChC,MAAM,IAAI,WAAW,CAAC,KAAK,CAAC,CAAA;QAE5B,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,kCAAkC,CAAC,MAAM,CAAC,EAAE;YACrE,yEAAyE;YACzE,uCAAuC;YACvC,SAAQ;SACT;aAAM;YACL,MAAM,MAAM,CAAA;YACZ,eAAe;YACf,MAAM,GAAG,EAAE,CAAA;SACZ;KACF;IAED,+CAA+C;IAC/C,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE;QACrB,MAAM,MAAM,CAAA;KACb;AACH,CAAC;AAED,MAAM,UAAU,MAAM,CAAC,MAAgB;IACrC,OAAO,CAAC,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;AAC9C,CAAC;AAED,MAAM,UAAU,GAAG;IACjB,MAAM;IACN,MAAM;IACN,eAAe;IACf,eAAe;IACf,oBAAoB;IACpB,kBAAkB;IAClB,GAAG;CACJ,CAAA;AAED,yBAAyB;AACzB,oDAAoD;AACpD,eAAe,UAAU,CAAA"}

package/esm/{encoder.test.js → main.test.js}

@@ -1,5 +1,5 @@
 /* eslint-disable compat/compat */
-import { decode, decodeAsyncGenerator, decodeGenerator, encode, isWithinTokenLimit, } from './encoder';
+import { decode, decodeAsyncGenerator, decodeGenerator, encode, isWithinTokenLimit, } from './main';
 describe('basic functionality', () => {
     it('empty string', () => {
         const str = '';
@@ -81,4 +81,4 @@ describe('basic functionality', () => {
         expect(isWithinTokenLimit(str, 300)).toBe(18);
     });
 });
-//# sourceMappingURL=encoder.test.js.map
+//# sourceMappingURL=main.test.js.map

package/esm/main.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"main.test.js","sourceRoot":"","sources":["../src/main.test.ts"],"names":[],"mappings":"AAAA,kCAAkC;AAClC,OAAO,EACL,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,kBAAkB,GACnB,MAAM,QAAQ,CAAA;AAEf,QAAQ,CAAC,qBAAqB,EAAE,GAAG,EAAE;IACnC,EAAE,CAAC,cAAc,EAAE,GAAG,EAAE;QACtB,MAAM,GAAG,GAAG,EAAE,CAAA;QACd,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;QAC/B,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;QACxC,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QAC1C,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IAC5C,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;QACf,MAAM,GAAG,GAAG,GAAG,CAAA;QACf,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;QAClC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;QACxC,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QAC1C,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;IAChD,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;QACb,MAAM,GAAG,GAAG,IAAI,CAAA;QAChB,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;QAClC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;IAC1C,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,aAAa,EAAE,GAAG,EAAE;QACrB,MAAM,GAAG,GAAG,mBAAmB,CAAA;QAC/B,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAK,EAAE,GAAG,EAAE,GAAG,EAAE,IAAK,CAAC,CAAC,CAAA;QACrD,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;QACxC,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QAC9C,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IAC5C,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,kBAAkB,EAAE,GAAG,EAAE;QAC1B,MAAM,GAAG,GAAG,aAAa,CAAA;QACzB,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,KAAM,CAAC,CAAC,CAAA;QAC/C,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;QACxC,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IAC5C,CAAC,CAAC,CAAA;IAEF,MAAM,gBAAgB,GAAG,CAAC,KAAM,EAAE,KAAM,EAAE,GAAG,EAAE,GAAG,EAAE,KAAM,EAAE,GAAG,EAAE,GAAG,CAAC,CAAA;IAErE,EAAE,CAAC,QAAQ,EAAE,GAAG,EAAE;QAChB,MAAM,GAAG,GAAG,mBAAmB,CAAA;QAC/B,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAA;QAC7C,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;QACxC,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QAC9C,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IAC9C,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,qCAAqC,EAAE,GAAG,EAAE;QAC7C,MAAM,SAAS,GAAG,eAAe,CAAC,gBAAgB,CAAC,CAAA;QACnD,MAAM,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;QAC5C,MAAM,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QACxC,MAAM,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACzC,MAAM,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;QAC7C,MAAM,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QACxC,MAAM,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IAC3C,CAAC,CAAC,CAAA;IAEF,KAAK,SAAS,CAAC,CAAC,wBAAwB;QACtC,KAAK,MAAM,KAAK,IAAI,gBAAgB,EAAE;YACpC,4CAA4C;YAC5C,MAAM,MAAM,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;SACnC;IACH,CAAC;IAED,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;QACzD,MAAM,SAAS,GAAG,oBAAoB,CAAC,wBAAwB,EAAE,CAAC,CAAA;QAClE,MAAM,OAAO,GAAG,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,EAAE,IAAI,CAAC,CAAA;QACzD,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,SAAS,EAAE;YACnC,MAAM,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,CAAA;SACvC;IACH,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,sBAAsB,EAAE,GAAG,EAAE;QAC9B,MAAM,GAAG,GAAG,6CAA6C,CAAA;QAEzD,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC;YAC1B,IAAK,EAAE,KAAM,EAAE,KAAM,EAAE,GAAG,EAAE,KAAM,EAAE,KAAM,EAAE,IAAK,EAAE,IAAK;SACzD,CAAC,CAAA;QACF,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;IAC1C,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,kBAAkB,EAAE,GAAG,EAAE;QAC1B,MAAM,GAAG,GAAG,8CAA8C,CAAA;QAC1D,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;QACxC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,aAAa,CAAC;YAChC,KAAM,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,IAAK,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG;YACtE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE;SACf,CAAC,CAAA;QACF,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QAC/C,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IAC/C,CAAC,CAAC,CAAA;AACJ,CAAC,CAAC,CAAA"}

package/esm/package.json

@@ -0,0 +1 @@
+{"name": "gpt-tokenizer", "type": "module"}

package/esm/utils.d.ts

@@ -1,6 +1,22 @@
-export declare function dictZip<T extends PropertyKey, U>(x: T[], y: U[]): Record<T, U>;
-export declare function range(x: number, y: number): number[];
-export declare const getCharCode: (x: string) => number;
-export declare const getCharFromCode: (x: number) => string;
-export declare function getPairs(word: string[]): [string, string][];
-export declare const splitToken: (token: string) => string[];
+/**
+ * @template T
+ * @template U
+ * @param {T[]} x
+ * @param {U[]} y
+ * @returns {Record<T, U>}
+ */
+export function dictZip<T, U>(x: T[], y: U[]): Record<T, U>;
+/**
+ * @param {number} x
+ * @param {number} y
+ * @returns {number[]}
+ */
+export function range(x: number, y: number): number[];
+/**
+ * @param {string[]} word
+ * @returns {[string, string][]}
+ */
+export function getPairs(word: string[]): [string, string][];
+export function getCharCode(x: string): number;
+export function getCharFromCode(x: number): string;
+export function splitToken(token: string): string[];

package/esm/utils.js

@@ -1,18 +1,42 @@
 /* eslint-disable unicorn/prefer-spread */
 /* eslint-disable unicorn/prefer-code-point */
+/**
+ * @template T
+ * @template U
+ * @param {T[]} x
+ * @param {U[]} y
+ * @returns {Record<T, U>}
+ */
 export function dictZip(x, y) {
-    // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
+    /** @type {Record<T, U>} */
     const result = {};
     x.forEach((k, i) => {
         result[k] = y[i];
     });
     return result;
 }
+/**
+ * @param {number} x
+ * @param {number} y
+ * @returns {number[]}
+ */
 export function range(x, y) {
     return Array.from({ length: y - x }, (_, i) => x + i);
 }
+/**
+ * @param {string} x
+ * @returns {number}
+ */
 export const getCharCode = (x) => x.charCodeAt(0);
+/**
+ * @param {number} x
+ * @returns {string}
+ */
 export const getCharFromCode = (x) => String.fromCharCode(x);
+/**
+ * @param {string[]} word
+ * @returns {[string, string][]}
+ */
 export function getPairs(word) {
     /** @type {[string, string][]} */
     const pairs = [];
@@ -24,5 +48,9 @@ export function getPairs(word) {
     }
     return pairs;
 }
+/**
+ * @param {string} token
+ * @returns {string[]}
+ */
 export const splitToken = (token) => token.split('');
 //# sourceMappingURL=utils.js.map

package/esm/utils.js.map

@@ -1 +1 @@
-{"version":3,"file":"utils.js","sourceRoot":"","sources":["../src/utils.ts"],"names":[],"mappings":"AAAA,0CAA0C;AAC1C,8CAA8C;AAE9C,MAAM,UAAU,OAAO,CACrB,CAAM,EACN,CAAM;IAEN,yEAAyE;IACzE,MAAM,MAAM,GAAG,EAAkB,CAAA;IACjC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACjB,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAE,CAAA;IACnB,CAAC,CAAC,CAAA;IACF,OAAO,MAAM,CAAA;AACf,CAAC;AAED,MAAM,UAAU,KAAK,CAAC,CAAS,EAAE,CAAS;IACxC,OAAO,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;AACvD,CAAC;AAED,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,CAAS,EAAU,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAA;AAEjE,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,CAAS,EAAU,EAAE,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAA;AAE5E,MAAM,UAAU,QAAQ,CAAC,IAAc;IACrC,iCAAiC;IACjC,MAAM,KAAK,GAAuB,EAAE,CAAA;IACpC,IAAI,QAAQ,GAAG,IAAI,CAAC,CAAC,CAAC,CAAA;IACtB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;QACpC,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,CAAA;QACpB,KAAK,CAAC,IAAI,CAAC,CAAC,QAAS,EAAE,IAAK,CAAC,CAAC,CAAA;QAC9B,QAAQ,GAAG,IAAI,CAAA;KAChB;IACD,OAAO,KAAK,CAAA;AACd,CAAC;AAED,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,KAAa,EAAY,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA"}
+{"version":3,"file":"utils.js","sourceRoot":"","sources":["../src/utils.js"],"names":[],"mappings":"AAAA,0CAA0C;AAC1C,8CAA8C;AAE9C;;;;;;GAMG;AACH,MAAM,UAAU,OAAO,CAAC,CAAC,EAAE,CAAC;IAC1B,2BAA2B;IAC3B,MAAM,MAAM,GAAG,EAAE,CAAA;IACjB,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACjB,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAA;IAClB,CAAC,CAAC,CAAA;IACF,OAAO,MAAM,CAAA;AACf,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,KAAK,CAAC,CAAC,EAAE,CAAC;IACxB,OAAO,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;AACvD,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAA;AAEjD;;;GAGG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAA;AAE5D;;;GAGG;AACH,MAAM,UAAU,QAAQ,CAAC,IAAI;IAC3B,iCAAiC;IACjC,MAAM,KAAK,GAAG,EAAE,CAAA;IAChB,IAAI,QAAQ,GAAG,IAAI,CAAC,CAAC,CAAC,CAAA;IACtB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;QACpC,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,CAAA;QACpB,KAAK,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,CAAA;QAC5B,QAAQ,GAAG,IAAI,CAAA;KAChB;IACD,OAAO,KAAK,CAAA;AACd,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "gpt-tokenizer",
-  "version": "1.0.3",
+  "version": "1.0.4",
   "description": "BPE Encoder Decoder for GPT-2 / GPT-3",
   "keywords": [
     "BPE",
@@ -58,6 +58,10 @@
     "./esm/*": {
       "import": "./esm/*.js"
     },
+    "./data/*": {
+      "import": "./data/*",
+      "require": "./data/*"
+    },
     "./package.json": "./package.json"
   },
   "main": "esm/main.js",
@@ -73,7 +77,7 @@
   "scripts": {
     "build": "yarn build:cjs && yarn build:esm && yarn build:umd",
     "build:cjs": "yarn rrun tsc --outDir cjs --module commonjs --target es2022",
-    "build:esm": "yarn rrun tsc --outDir esm --module esnext --target es2022",
+    "build:esm": "yarn rrun tsc --outDir esm --module esnext --target es2022 && echo '{\"name\": \"gpt-tokenizer\", \"type\": \"module\"}' > ./esm/package.json",
     "build:umd": "beemo webpack --entry='./src/main.ts' --env 'outDir=dist' --env 'moduleTarget=umd' --env 'engineTarget=web' --env 'codeTarget=es2022' --env 'name=GPT3Encoder' --env 'export=default' --env 'filename=gpt3encoder.js'",
     "clean": "git clean -dfX --exclude=node_modules src && beemo typescript:sync-project-refs",
     "format": "yarn rrun prettier --write \"./{src,tests,.config}/**/!(*.d).{.js,jsx,ts,tsx,json,md}\"",
@@ -83,7 +87,7 @@
     "test": "yarn test:format && yarn test:types && yarn test:lint && yarn test:code",
     "test:code": "rrun jest",
     "test:format": "yarn rrun prettier --check \"./{src,tests,.config}/**/!(*.d).{.js,jsx,ts,tsx,json,md}\"",
-    "test:lint": "rrun eslint 'src/**/*.{js,jsx,ts,tsx}'",
+    "test:lint": "rrun eslint 'src/*.{js,jsx,ts,tsx}'",
     "test:types": "yarn rrun tsc --noEmit"
   },
   "release": {
@@ -108,7 +112,7 @@
   },
   "devDependencies": {
     "@niieani/scaffold": "^1.6.7",
-    "tsx": "^3.12.6"
+    "tsx": "^3.12.7"
   },
   "packageManager": "yarn@3.5.0",
   "publishConfig": {