gpt-tokenizer 1.0.2 → 1.0.4

package/cjs/{getBpe.js → getData.js}

@@ -13,5 +13,9 @@ const bpeMerges = lines.slice(1, -1).map((x) => x
     .filter((e) => e.trim().length > 0)
     .join(','));
 const bpeRanks = (0, utils_1.dictZip)(bpeMerges, (0, utils_1.range)(0, bpeMerges.length));
-fs_1.default.writeFileSync(path_1.default.join(__dirname, '../data/bpeRanks.json'), JSON.stringify(bpeRanks));
-//# sourceMappingURL=getBpe.js.map
+// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
+const encoder = require('../data/encoder.json');
+fs_1.default.mkdirSync(path_1.default.join(__dirname, 'data'), { recursive: true });
+fs_1.default.writeFileSync(path_1.default.join(__dirname, 'data/bpeRanks.js'), `/* eslint-disable */\n// @ts-nocheck\n// prettier-ignore\nexport default ${JSON.stringify(bpeRanks)}`);
+fs_1.default.writeFileSync(path_1.default.join(__dirname, 'data/encoder.js'), `/* eslint-disable */\n// @ts-nocheck\n// prettier-ignore\nexport default ${JSON.stringify(encoder)}`);
+//# sourceMappingURL=getData.js.map

package/cjs/getData.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"getData.js","sourceRoot":"","sources":["../src/getData.ts"],"names":[],"mappings":";;;;;AAAA,4CAAmB;AACnB,gDAAuB;AACvB,mCAAwC;AAExC,MAAM,OAAO,GAAG,YAAE,CAAC,YAAY,CAC7B,cAAI,CAAC,IAAI,CAAC,SAAS,EAAE,mBAAmB,CAAC,EACzC,MAAM,CACP,CAAA;AACD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;AAEjC,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAC7C,CAAC;KACE,KAAK,CAAC,OAAO,CAAC;KACd,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC;KAClC,IAAI,CAAC,GAAG,CAAC,CACb,CAAA;AAED,MAAM,QAAQ,GAAG,IAAA,eAAO,EAAC,SAAS,EAAE,IAAA,aAAK,EAAC,CAAC,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC,CAAA;AAC/D,mEAAmE;AACnE,MAAM,OAAO,GAAW,OAAO,CAAC,sBAAsB,CAAC,CAAA;AAEvD,YAAE,CAAC,SAAS,CAAC,cAAI,CAAC,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;AAC/D,YAAE,CAAC,aAAa,CACd,cAAI,CAAC,IAAI,CAAC,SAAS,EAAE,kBAAkB,CAAC,EACxC,4EAA4E,IAAI,CAAC,SAAS,CACxF,QAAQ,CACT,EAAE,CACJ,CAAA;AACD,YAAE,CAAC,aAAa,CACd,cAAI,CAAC,IAAI,CAAC,SAAS,EAAE,iBAAiB,CAAC,EACvC,4EAA4E,IAAI,CAAC,SAAS,CACxF,OAAO,CACR,EAAE,CACJ,CAAA"}

package/cjs/main.d.ts

@@ -1 +1,28 @@
-export * from './encoder';
+export declare function bpe(token: string, cache?: Map<string, string>): string;
+export declare function encodeGenerator(text: string, cache?: Map<string, string>): Generator<number[], void, undefined>;
+/**
+ * @returns {false | number} false if token limit is exceeded, otherwise the number of tokens
+ */
+export declare function isWithinTokenLimit(text: string, tokenLimit: number, cache?: Map<string, string>): false | number;
+export declare function encode(text: string, cache?: Map<string, string>): number[];
+export declare function decodeToken(token: number): string;
+export declare function endsWithIncompleteUtfPairSurrogate(string: string): boolean;
+export declare function decodeGenerator(tokens: Iterable<number>): Generator<string, void, undefined>;
+/**
+ * Decode tokens asynchronously and yield the decoded strings, one by one.
+ * Will not yield for tokens that include a high surrogate, but wait for the next token.
+ * @param {AsyncIterable<number>} tokens
+ * @returns {AsyncGenerator<string, void, undefined>}
+ */
+export declare function decodeAsyncGenerator(tokens: AsyncIterable<number>): AsyncGenerator<string, void, undefined>;
+export declare function decode(tokens: number[]): string;
+declare const gptEncoder: {
+    encode: typeof encode;
+    decode: typeof decode;
+    encodeGenerator: typeof encodeGenerator;
+    decodeGenerator: typeof decodeGenerator;
+    decodeAsyncGenerator: typeof decodeAsyncGenerator;
+    isWithinTokenLimit: typeof isWithinTokenLimit;
+    bpe: typeof bpe;
+};
+export default gptEncoder;

package/cjs/main.js

@@ -1,18 +1,212 @@
 "use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __exportStar = (this && this.__exportStar) || function(m, exports) {
-    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+/* eslint-disable import/extensions */
+/* eslint-disable unicorn/prefer-spread */
+/* eslint-disable unicorn/prefer-code-point */
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-__exportStar(require("./encoder"), exports);
+exports.decode = exports.decodeAsyncGenerator = exports.decodeGenerator = exports.endsWithIncompleteUtfPairSurrogate = exports.decodeToken = exports.encode = exports.isWithinTokenLimit = exports.encodeGenerator = exports.bpe = void 0;
+const bpeRanks_js_1 = __importDefault(require("./data/bpeRanks.js"));
+const encoder_js_1 = __importDefault(require("./data/encoder.js"));
+const utils_js_1 = require("./utils.js");
+const encoder = encoder_js_1.default;
+const bpeRanks = bpeRanks_js_1.default;
+const decoder = (0, utils_js_1.dictZip)(Object.values(encoder), Object.keys(encoder));
+const textEncoder = new TextEncoder();
+const textDecoder = new TextDecoder();
+// eslint-disable-next-line no-magic-numbers
+const UPPER_BYTE_VALUE = 2 ** 8;
+function createByteToUnicodeMap() {
+    const asciiRange = (0, utils_js_1.range)((0, utils_js_1.getCharCode)('!'), (0, utils_js_1.getCharCode)('~') + 1);
+    const latin1Range1 = (0, utils_js_1.range)((0, utils_js_1.getCharCode)('¡'), (0, utils_js_1.getCharCode)('¬') + 1);
+    const latin1Range2 = (0, utils_js_1.range)((0, utils_js_1.getCharCode)('®'), (0, utils_js_1.getCharCode)('ÿ') + 1);
+    const initialCodePoints = [...asciiRange, ...latin1Range1, ...latin1Range2];
+    const mappedCodePoints = [...initialCodePoints];
+    let newCodePointOffset = 0;
+    for (let byteValue = 0; byteValue < UPPER_BYTE_VALUE; byteValue++) {
+        if (!initialCodePoints.includes(byteValue)) {
+            initialCodePoints.push(byteValue);
+            mappedCodePoints.push(UPPER_BYTE_VALUE + newCodePointOffset);
+            newCodePointOffset += 1;
+        }
+    }
+    const unicodeChars = mappedCodePoints.map(utils_js_1.getCharFromCode);
+    return new Map(initialCodePoints.map((x, i) => [x, unicodeChars[i]]));
+}
+const pat = /'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+/gu;
+const byteEncoder = createByteToUnicodeMap();
+const byteDecoder = new Map([...byteEncoder].map(([k, v]) => [v, k]));
+function bpe(token, cache = new Map()) {
+    if (cache.has(token)) {
+        return cache.get(token);
+    }
+    let word = (0, utils_js_1.splitToken)(token);
+    let pairs = (0, utils_js_1.getPairs)(word);
+    if (pairs.length === 0) {
+        return token;
+    }
+    // eslint-disable-next-line no-constant-condition
+    while (true) {
+        const minPairs = new Map();
+        pairs.forEach((pair) => {
+            const rank = bpeRanks[pair.join(',')];
+            minPairs.set(
+            // eslint-disable-next-line no-magic-numbers
+            typeof rank === 'undefined' || Number.isNaN(rank) ? 10e10 : rank, pair);
+        });
+        const keys = [...minPairs.keys()];
+        const bigram = minPairs.get(Math.min(...keys));
+        if (!bigram || !(bigram.join(',') in bpeRanks)) {
+            break;
+        }
+        const [first, second] = bigram;
+        let newWord = [];
+        let i = 0;
+        while (i < word.length) {
+            const j = word.indexOf(first, i);
+            if (j === -1) {
+                newWord = newWord.concat(word.slice(i));
+                break;
+            }
+            newWord = newWord.concat(word.slice(i, j));
+            i = j;
+            if (word[i] === first && i < word.length - 1 && word[i + 1] === second) {
+                newWord.push(first + second);
+                i += 2;
+            }
+            else {
+                newWord.push(word[i]);
+                i += 1;
+            }
+        }
+        word = newWord;
+        if (word.length === 1) {
+            break;
+        }
+        else {
+            pairs = (0, utils_js_1.getPairs)(word);
+        }
+    }
+    const result = word.join(' ');
+    cache.set(token, result);
+    return result;
+}
+exports.bpe = bpe;
+function* encodeGenerator(text, cache = new Map()) {
+    for (let [token] of text.matchAll(pat)) {
+        token = [...textEncoder.encode(token)]
+            .map((x) => byteEncoder.get(x) ?? '')
+            .join('');
+        const newTokens = bpe(token, cache)
+            .split(' ')
+            .map((x) => encoder[x]);
+        yield newTokens;
+    }
+}
+exports.encodeGenerator = encodeGenerator;
+/**
+ * @returns {false | number} false if token limit is exceeded, otherwise the number of tokens
+ */
+function isWithinTokenLimit(text, tokenLimit, cache = new Map()) {
+    const tokenGenerator = encodeGenerator(text, cache);
+    let count = 0;
+    for (const tokens of tokenGenerator) {
+        count += tokens.length;
+        if (count > tokenLimit) {
+            return false;
+        }
+    }
+    return count;
+}
+exports.isWithinTokenLimit = isWithinTokenLimit;
+function encode(text, cache = new Map()) {
+    return [...encodeGenerator(text, cache)].flat(1);
+}
+exports.encode = encode;
+function decodeToken(token) {
+    const decodedToken = decoder[token];
+    if (typeof decodedToken === 'undefined') {
+        return '';
+    }
+    const decodedBytes = (0, utils_js_1.splitToken)(decodedToken).map((x) => byteDecoder.get(x));
+    return textDecoder.decode(new Uint8Array(decodedBytes), {
+        stream: true,
+    });
+}
+exports.decodeToken = decodeToken;
+const HIGH_SURROGATE_START = 55296;
+const HIGH_SURROGATE_END = 56319;
+function endsWithIncompleteUtfPairSurrogate(string) {
+    if (string.length === 0)
+        return false;
+    // Check if the last character is a high surrogate
+    const lastCharCode = string.charCodeAt(string.length - 1);
+    return (lastCharCode >= HIGH_SURROGATE_START && lastCharCode <= HIGH_SURROGATE_END);
+}
+exports.endsWithIncompleteUtfPairSurrogate = endsWithIncompleteUtfPairSurrogate;
+function* decodeGenerator(tokens) {
+    let buffer = '';
+    for (const token of tokens) {
+        buffer += decodeToken(token);
+        if (buffer.length === 0 || endsWithIncompleteUtfPairSurrogate(buffer)) {
+            // Keep the high surrogate in the buffer and continue with the next token
+            // eslint-disable-next-line no-continue
+            continue;
+        }
+        else {
+            yield buffer;
+            // reset buffer
+            buffer = '';
+        }
+    }
+    // Yield any remaining characters in the buffer
+    if (buffer.length > 0) {
+        yield buffer;
+    }
+}
+exports.decodeGenerator = decodeGenerator;
+/**
+ * Decode tokens asynchronously and yield the decoded strings, one by one.
+ * Will not yield for tokens that include a high surrogate, but wait for the next token.
+ * @param {AsyncIterable<number>} tokens
+ * @returns {AsyncGenerator<string, void, undefined>}
+ */
+async function* decodeAsyncGenerator(tokens) {
+    let buffer = '';
+    for await (const token of tokens) {
+        buffer += decodeToken(token);
+        if (buffer.length === 0 || endsWithIncompleteUtfPairSurrogate(buffer)) {
+            // Keep the high surrogate in the buffer and continue with the next token
+            // eslint-disable-next-line no-continue
+            continue;
+        }
+        else {
+            yield buffer;
+            // reset buffer
+            buffer = '';
+        }
+    }
+    // Yield any remaining characters in the buffer
+    if (buffer.length > 0) {
+        yield buffer;
+    }
+}
+exports.decodeAsyncGenerator = decodeAsyncGenerator;
+function decode(tokens) {
+    return [...decodeGenerator(tokens)].join('');
+}
+exports.decode = decode;
+const gptEncoder = {
+    encode,
+    decode,
+    encodeGenerator,
+    decodeGenerator,
+    decodeAsyncGenerator,
+    isWithinTokenLimit,
+    bpe,
+};
+// used by the UMD export
+// eslint-disable-next-line import/no-default-export
+exports.default = gptEncoder;
 //# sourceMappingURL=main.js.map

package/cjs/main.js.map

@@ -1 +1 @@
-{"version":3,"file":"main.js","sourceRoot":"","sources":["../src/main.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,4CAAyB"}
+{"version":3,"file":"main.js","sourceRoot":"","sources":["../src/main.ts"],"names":[],"mappings":";AAAA,sCAAsC;AACtC,0CAA0C;AAC1C,8CAA8C;;;;;;AAE9C,qEAA6C;AAC7C,mEAA2C;AAC3C,yCAOmB;AAEnB,MAAM,OAAO,GAA2B,oBAAW,CAAA;AACnD,MAAM,QAAQ,GAA2B,qBAAY,CAAA;AAErD,MAAM,OAAO,GAA2B,IAAA,kBAAO,EAC7C,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,EACtB,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CACrB,CAAA;AAED,MAAM,WAAW,GAAG,IAAI,WAAW,EAAE,CAAA;AACrC,MAAM,WAAW,GAAG,IAAI,WAAW,EAAE,CAAA;AAErC,4CAA4C;AAC5C,MAAM,gBAAgB,GAAG,CAAC,IAAI,CAAC,CAAA;AAE/B,SAAS,sBAAsB;IAC7B,MAAM,UAAU,GAAG,IAAA,gBAAK,EAAC,IAAA,sBAAW,EAAC,GAAG,CAAC,EAAE,IAAA,sBAAW,EAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAA;IAChE,MAAM,YAAY,GAAG,IAAA,gBAAK,EAAC,IAAA,sBAAW,EAAC,GAAG,CAAC,EAAE,IAAA,sBAAW,EAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAA;IAClE,MAAM,YAAY,GAAG,IAAA,gBAAK,EAAC,IAAA,sBAAW,EAAC,GAAG,CAAC,EAAE,IAAA,sBAAW,EAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAA;IAElE,MAAM,iBAAiB,GAAG,CAAC,GAAG,UAAU,EAAE,GAAG,YAAY,EAAE,GAAG,YAAY,CAAC,CAAA;IAC3E,MAAM,gBAAgB,GAAG,CAAC,GAAG,iBAAiB,CAAC,CAAA;IAE/C,IAAI,kBAAkB,GAAG,CAAC,CAAA;IAC1B,KAAK,IAAI,SAAS,GAAG,CAAC,EAAE,SAAS,GAAG,gBAAgB,EAAE,SAAS,EAAE,EAAE;QACjE,IAAI,CAAC,iBAAiB,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE;YAC1C,iBAAiB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;YACjC,gBAAgB,CAAC,IAAI,CAAC,gBAAgB,GAAG,kBAAkB,CAAC,CAAA;YAC5D,kBAAkB,IAAI,CAAC,CAAA;SACxB;KACF;IAED,MAAM,YAAY,GAAG,gBAAgB,CAAC,GAAG,CAAC,0BAAe,CAAC,CAAA;IAC1D,OAAO,IAAI,GAAG,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC,CAAE,CAAC,CAAC,CAAC,CAAA;AACxE,CAAC;AAED,MAAM,GAAG,GACP,8EAA8E,CAAA;AAEhF,MAAM,WAAW,GAAG,sBAAsB,EAAE,CAAA;AAC5C,MAAM,WAAW,GAAwB,IAAI,GAAG,CAC9C,CAAC,GAAG,WAAW,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CACzC,CAAA;AAED,SAAgB,GAAG,CACjB,KAAa,EACb,QAA6B,IAAI,GAAG,EAAE;IAEtC,IAAI,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE;QACpB,OAAO,KAAK,CAAC,GAAG,CAAC,KAAK,CAAE,CAAA;KACzB;IAED,IAAI,IAAI,GAAG,IAAA,qBAAU,EAAC,KAAK,CAAC,CAAA;IAC5B,IAAI,KAAK,GAAG,IAAA,mBAAQ,EAAC,IAAI,CAAC,CAAA;IAE1B,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE;QACtB,OAAO,KAAK,CAAA;KACb;IAED,iDAAiD;IACjD,OAAO,IAAI,EAAE;QACX,MAAM,QAAQ,GAAkC,IAAI,GAAG,EAAE,CAAA;QACzD,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;YACrB,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAA;YACrC,QAAQ,CAAC,GAAG;YACV,4CAA4C;YAC5C,OAAO,IAAI,KAAK,WAAW,IAAI,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAChE,IAAI,CACL,CAAA;QACH,CAAC,CAAC,CAAA;QACF,MAAM,IAAI,GAAG,CAAC,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAA;QAEjC,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC,CAAA;QAE9C,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,QAAQ,CAAC,EAAE;YAC9C,MAAK;SACN;QAED,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,GAAG,MAAM,CAAA;QAC9B,IAAI,OAAO,GAAa,EAAE,CAAA;QAC1B,IAAI,CAAC,GAAG,CAAC,CAAA;QAET,OAAO,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE;YACtB,MAAM,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,CAAA;YAChC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE;gBACZ,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAA;gBACvC,MAAK;aACN;YACD,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;YAC1C,CAAC,GAAG,CAAC,CAAA;YAEL,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,MAAM,EAAE;gBACtE,OAAO,CAAC,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC,CAAA;gBAC5B,CAAC,IAAI,CAAC,CAAA;aACP;iBAAM;gBACL,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAE,CAAC,CAAA;gBACtB,CAAC,IAAI,CAAC,CAAA;aACP;SACF;QAED,IAAI,GAAG,OAAO,CAAA;QACd,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE;YACrB,MAAK;SACN;aAAM;YACL,KAAK,GAAG,IAAA,mBAAQ,EAAC,IAAI,CAAC,CAAA;SACvB;KACF;IAED,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;IAC7B,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,CAAA;IAExB,OAAO,MAAM,CAAA;AACf,CAAC;AApED,kBAoEC;AAED,QAAe,CAAC,CAAC,eAAe,CAC9B,IAAY,EACZ,QAA6B,IAAI,GAAG,EAAE;IAEtC,KAAK,IAAI,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE;QACtC,KAAK,GAAG,CAAC,GAAG,WAAW,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;aACnC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;aACpC,IAAI,CAAC,EAAE,CAAC,CAAA;QAEX,MAAM,SAAS,GAAG,GAAG,CAAC,KAAK,EAAE,KAAK,CAAC;aAChC,KAAK,CAAC,GAAG,CAAC;aACV,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,CAAE,CAAC,CAAA;QAE1B,MAAM,SAAS,CAAA;KAChB;AACH,CAAC;AAfD,0CAeC;AAED;;GAEG;AACH,SAAgB,kBAAkB,CAChC,IAAY,EACZ,UAAkB,EAClB,QAA6B,IAAI,GAAG,EAAE;IAEtC,MAAM,cAAc,GAAG,eAAe,CAAC,IAAI,EAAE,KAAK,CAAC,CAAA;IACnD,IAAI,KAAK,GAAG,CAAC,CAAA;IACb,KAAK,MAAM,MAAM,IAAI,cAAc,EAAE;QACnC,KAAK,IAAI,MAAM,CAAC,MAAM,CAAA;QACtB,IAAI,KAAK,GAAG,UAAU,EAAE;YACtB,OAAO,KAAK,CAAA;SACb;KACF;IACD,OAAO,KAAK,CAAA;AACd,CAAC;AAdD,gDAcC;AAED,SAAgB,MAAM,CACpB,IAAY,EACZ,QAA6B,IAAI,GAAG,EAAE;IAEtC,OAAO,CAAC,GAAG,eAAe,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;AAClD,CAAC;AALD,wBAKC;AAED,SAAgB,WAAW,CAAC,KAAa;IACvC,MAAM,YAAY,GAAG,OAAO,CAAC,KAAK,CAAC,CAAA;IACnC,IAAI,OAAO,YAAY,KAAK,WAAW,EAAE;QACvC,OAAO,EAAE,CAAA;KACV;IACD,MAAM,YAAY,GAAG,IAAA,qBAAU,EAAC,YAAY,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC,CAAA;IAC7E,OAAO,WAAW,CAAC,MAAM,CAAC,IAAI,UAAU,CAAC,YAAY,CAAC,EAAE;QACtD,MAAM,EAAE,IAAI;KACb,CAAC,CAAA;AACJ,CAAC;AATD,kCASC;AAED,MAAM,oBAAoB,GAAG,KAAM,CAAA;AACnC,MAAM,kBAAkB,GAAG,KAAM,CAAA;AACjC,SAAgB,kCAAkC,CAAC,MAAc;IAC/D,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,KAAK,CAAA;IACrC,kDAAkD;IAClD,MAAM,YAAY,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;IACzD,OAAO,CACL,YAAY,IAAI,oBAAoB,IAAI,YAAY,IAAI,kBAAkB,CAC3E,CAAA;AACH,CAAC;AAPD,gFAOC;AAED,QAAe,CAAC,CAAC,eAAe,CAC9B,MAAwB;IAExB,IAAI,MAAM,GAAG,EAAE,CAAA;IAEf,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE;QAC1B,MAAM,IAAI,WAAW,CAAC,KAAK,CAAC,CAAA;QAE5B,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,kCAAkC,CAAC,MAAM,CAAC,EAAE;YACrE,yEAAyE;YACzE,uCAAuC;YACvC,SAAQ;SACT;aAAM;YACL,MAAM,MAAM,CAAA;YACZ,eAAe;YACf,MAAM,GAAG,EAAE,CAAA;SACZ;KACF;IAED,+CAA+C;IAC/C,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE;QACrB,MAAM,MAAM,CAAA;KACb;AACH,CAAC;AAvBD,0CAuBC;AAED;;;;;GAKG;AACI,KAAK,SAAS,CAAC,CAAC,oBAAoB,CACzC,MAA6B;IAE7B,IAAI,MAAM,GAAG,EAAE,CAAA;IAEf,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,MAAM,EAAE;QAChC,MAAM,IAAI,WAAW,CAAC,KAAK,CAAC,CAAA;QAE5B,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,kCAAkC,CAAC,MAAM,CAAC,EAAE;YACrE,yEAAyE;YACzE,uCAAuC;YACvC,SAAQ;SACT;aAAM;YACL,MAAM,MAAM,CAAA;YACZ,eAAe;YACf,MAAM,GAAG,EAAE,CAAA;SACZ;KACF;IAED,+CAA+C;IAC/C,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE;QACrB,MAAM,MAAM,CAAA;KACb;AACH,CAAC;AAvBD,oDAuBC;AAED,SAAgB,MAAM,CAAC,MAAgB;IACrC,OAAO,CAAC,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;AAC9C,CAAC;AAFD,wBAEC;AAED,MAAM,UAAU,GAAG;IACjB,MAAM;IACN,MAAM;IACN,eAAe;IACf,eAAe;IACf,oBAAoB;IACpB,kBAAkB;IAClB,GAAG;CACJ,CAAA;AAED,yBAAyB;AACzB,oDAAoD;AACpD,kBAAe,UAAU,CAAA"}

package/cjs/main.test.js

@@ -0,0 +1,86 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+/* eslint-disable compat/compat */
+const main_1 = require("./main");
+describe('basic functionality', () => {
+    it('empty string', () => {
+        const str = '';
+        expect((0, main_1.encode)(str)).toEqual([]);
+        expect((0, main_1.decode)((0, main_1.encode)(str))).toEqual(str);
+        expect((0, main_1.isWithinTokenLimit)(str, 0)).toBe(0);
+        expect((0, main_1.isWithinTokenLimit)(str, 3)).toBe(0);
+    });
+    it('space', () => {
+        const str = ' ';
+        expect((0, main_1.encode)(str)).toEqual([220]);
+        expect((0, main_1.decode)((0, main_1.encode)(str))).toEqual(str);
+        expect((0, main_1.isWithinTokenLimit)(str, 3)).toBe(1);
+        expect((0, main_1.isWithinTokenLimit)(str, 0)).toBe(false);
+    });
+    it('tab', () => {
+        const str = '\t';
+        expect((0, main_1.encode)(str)).toEqual([197]);
+        expect((0, main_1.decode)((0, main_1.encode)(str))).toEqual(str);
+    });
+    it('simple text', () => {
+        const str = 'This is some text';
+        expect((0, main_1.encode)(str)).toEqual([1212, 318, 617, 2420]);
+        expect((0, main_1.decode)((0, main_1.encode)(str))).toEqual(str);
+        expect((0, main_1.isWithinTokenLimit)(str, 3)).toBe(false);
+        expect((0, main_1.isWithinTokenLimit)(str, 5)).toBe(4);
+    });
+    it('multi-token word', () => {
+        const str = 'indivisible';
+        expect((0, main_1.encode)(str)).toEqual([521, 452, 12843]);
+        expect((0, main_1.decode)((0, main_1.encode)(str))).toEqual(str);
+        expect((0, main_1.isWithinTokenLimit)(str, 3)).toBe(3);
+    });
+    const helloWorldTokens = [31373, 50169, 233, 995, 12520, 234, 235];
+    it('emojis', () => {
+        const str = 'hello 👋 world 🌍';
+        expect((0, main_1.encode)(str)).toEqual(helloWorldTokens);
+        expect((0, main_1.decode)((0, main_1.encode)(str))).toEqual(str);
+        expect((0, main_1.isWithinTokenLimit)(str, 4)).toBe(false);
+        expect((0, main_1.isWithinTokenLimit)(str, 400)).toBe(7);
+    });
+    it('decode token-by-token via generator', () => {
+        const generator = (0, main_1.decodeGenerator)(helloWorldTokens);
+        expect(generator.next().value).toBe('hello');
+        expect(generator.next().value).toBe(' ');
+        expect(generator.next().value).toBe('👋');
+        expect(generator.next().value).toBe(' world');
+        expect(generator.next().value).toBe(' ');
+        expect(generator.next().value).toBe('🌍');
+    });
+    async function* getHelloWorldTokensAsync() {
+        for (const token of helloWorldTokens) {
+            // eslint-disable-next-line no-await-in-loop
+            yield await Promise.resolve(token);
+        }
+    }
+    it('decode token-by-token via async generator', async () => {
+        const generator = (0, main_1.decodeAsyncGenerator)(getHelloWorldTokensAsync());
+        const decoded = ['hello', ' ', '👋', ' world', ' ', '🌍'];
+        for await (const value of generator) {
+            expect(value).toEqual(decoded.shift());
+        }
+    });
+    it('properties of Object', () => {
+        const str = 'toString constructor hasOwnProperty valueOf';
+        expect((0, main_1.encode)(str)).toEqual([
+            1462, 10100, 23772, 468, 23858, 21746, 1988, 5189,
+        ]);
+        expect((0, main_1.decode)((0, main_1.encode)(str))).toEqual(str);
+    });
+    it('text with commas', () => {
+        const str = 'hello, I am a text, and I have commas. a,b,c';
+        expect((0, main_1.decode)((0, main_1.encode)(str))).toEqual(str);
+        expect((0, main_1.encode)(str)).toStrictEqual([
+            31373, 11, 314, 716, 257, 2420, 11, 290, 314, 423, 725, 292, 13, 257,
+            11, 65, 11, 66,
+        ]);
+        expect((0, main_1.isWithinTokenLimit)(str, 15)).toBe(false);
+        expect((0, main_1.isWithinTokenLimit)(str, 300)).toBe(18);
+    });
+});
+//# sourceMappingURL=main.test.js.map

package/cjs/main.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"main.test.js","sourceRoot":"","sources":["../src/main.test.ts"],"names":[],"mappings":";;AAAA,kCAAkC;AAClC,iCAMe;AAEf,QAAQ,CAAC,qBAAqB,EAAE,GAAG,EAAE;IACnC,EAAE,CAAC,cAAc,EAAE,GAAG,EAAE;QACtB,MAAM,GAAG,GAAG,EAAE,CAAA;QACd,MAAM,CAAC,IAAA,aAAM,EAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;QAC/B,MAAM,CAAC,IAAA,aAAM,EAAC,IAAA,aAAM,EAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;QACxC,MAAM,CAAC,IAAA,yBAAkB,EAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QAC1C,MAAM,CAAC,IAAA,yBAAkB,EAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IAC5C,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;QACf,MAAM,GAAG,GAAG,GAAG,CAAA;QACf,MAAM,CAAC,IAAA,aAAM,EAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;QAClC,MAAM,CAAC,IAAA,aAAM,EAAC,IAAA,aAAM,EAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;QACxC,MAAM,CAAC,IAAA,yBAAkB,EAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QAC1C,MAAM,CAAC,IAAA,yBAAkB,EAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;IAChD,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;QACb,MAAM,GAAG,GAAG,IAAI,CAAA;QAChB,MAAM,CAAC,IAAA,aAAM,EAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;QAClC,MAAM,CAAC,IAAA,aAAM,EAAC,IAAA,aAAM,EAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;IAC1C,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,aAAa,EAAE,GAAG,EAAE;QACrB,MAAM,GAAG,GAAG,mBAAmB,CAAA;QAC/B,MAAM,CAAC,IAAA,aAAM,EAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAK,EAAE,GAAG,EAAE,GAAG,EAAE,IAAK,CAAC,CAAC,CAAA;QACrD,MAAM,CAAC,IAAA,aAAM,EAAC,IAAA,aAAM,EAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;QACxC,MAAM,CAAC,IAAA,yBAAkB,EAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QAC9C,MAAM,CAAC,IAAA,yBAAkB,EAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IAC5C,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,kBAAkB,EAAE,GAAG,EAAE;QAC1B,MAAM,GAAG,GAAG,aAAa,CAAA;QACzB,MAAM,CAAC,IAAA,aAAM,EAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,KAAM,CAAC,CAAC,CAAA;QAC/C,MAAM,CAAC,IAAA,aAAM,EAAC,IAAA,aAAM,EAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;QACxC,MAAM,CAAC,IAAA,yBAAkB,EAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IAC5C,CAAC,CAAC,CAAA;IAEF,MAAM,gBAAgB,GAAG,CAAC,KAAM,EAAE,KAAM,EAAE,GAAG,EAAE,GAAG,EAAE,KAAM,EAAE,GAAG,EAAE,GAAG,CAAC,CAAA;IAErE,EAAE,CAAC,QAAQ,EAAE,GAAG,EAAE;QAChB,MAAM,GAAG,GAAG,mBAAmB,CAAA;QAC/B,MAAM,CAAC,IAAA,aAAM,EAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAA;QAC7C,MAAM,CAAC,IAAA,aAAM,EAAC,IAAA,aAAM,EAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;QACxC,MAAM,CAAC,IAAA,yBAAkB,EAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QAC9C,MAAM,CAAC,IAAA,yBAAkB,EAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IAC9C,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,qCAAqC,EAAE,GAAG,EAAE;QAC7C,MAAM,SAAS,GAAG,IAAA,sBAAe,EAAC,gBAAgB,CAAC,CAAA;QACnD,MAAM,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;QAC5C,MAAM,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QACxC,MAAM,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACzC,MAAM,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;QAC7C,MAAM,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QACxC,MAAM,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IAC3C,CAAC,CAAC,CAAA;IAEF,KAAK,SAAS,CAAC,CAAC,wBAAwB;QACtC,KAAK,MAAM,KAAK,IAAI,gBAAgB,EAAE;YACpC,4CAA4C;YAC5C,MAAM,MAAM,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;SACnC;IACH,CAAC;IAED,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;QACzD,MAAM,SAAS,GAAG,IAAA,2BAAoB,EAAC,wBAAwB,EAAE,CAAC,CAAA;QAClE,MAAM,OAAO,GAAG,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,EAAE,IAAI,CAAC,CAAA;QACzD,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,SAAS,EAAE;YACnC,MAAM,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,CAAA;SACvC;IACH,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,sBAAsB,EAAE,GAAG,EAAE;QAC9B,MAAM,GAAG,GAAG,6CAA6C,CAAA;QAEzD,MAAM,CAAC,IAAA,aAAM,EAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC;YAC1B,IAAK,EAAE,KAAM,EAAE,KAAM,EAAE,GAAG,EAAE,KAAM,EAAE,KAAM,EAAE,IAAK,EAAE,IAAK;SACzD,CAAC,CAAA;QACF,MAAM,CAAC,IAAA,aAAM,EAAC,IAAA,aAAM,EAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;IAC1C,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,kBAAkB,EAAE,GAAG,EAAE;QAC1B,MAAM,GAAG,GAAG,8CAA8C,CAAA;QAC1D,MAAM,CAAC,IAAA,aAAM,EAAC,IAAA,aAAM,EAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;QACxC,MAAM,CAAC,IAAA,aAAM,EAAC,GAAG,CAAC,CAAC,CAAC,aAAa,CAAC;YAChC,KAAM,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,IAAK,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG;YACtE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE;SACf,CAAC,CAAA;QACF,MAAM,CAAC,IAAA,yBAAkB,EAAC,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QAC/C,MAAM,CAAC,IAAA,yBAAkB,EAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IAC/C,CAAC,CAAC,CAAA;AACJ,CAAC,CAAC,CAAA"}

package/cjs/utils.d.ts

@@ -1,6 +1,22 @@
-export declare function dictZip<T extends PropertyKey, U>(x: T[], y: U[]): Record<T, U>;
-export declare function range(x: number, y: number): number[];
-export declare const getCharCode: (x: string) => number;
-export declare const getCharFromCode: (x: number) => string;
-export declare function getPairs(word: string[]): [string, string][];
-export declare const splitToken: (token: string) => string[];
+/**
+ * @template T
+ * @template U
+ * @param {T[]} x
+ * @param {U[]} y
+ * @returns {Record<T, U>}
+ */
+export function dictZip<T, U>(x: T[], y: U[]): Record<T, U>;
+/**
+ * @param {number} x
+ * @param {number} y
+ * @returns {number[]}
+ */
+export function range(x: number, y: number): number[];
+/**
+ * @param {string[]} word
+ * @returns {[string, string][]}
+ */
+export function getPairs(word: string[]): [string, string][];
+export function getCharCode(x: string): number;
+export function getCharFromCode(x: number): string;
+export function splitToken(token: string): string[];

package/cjs/utils.js

@@ -3,8 +3,15 @@
 /* eslint-disable unicorn/prefer-code-point */
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.splitToken = exports.getPairs = exports.getCharFromCode = exports.getCharCode = exports.range = exports.dictZip = void 0;
+/**
+ * @template T
+ * @template U
+ * @param {T[]} x
+ * @param {U[]} y
+ * @returns {Record<T, U>}
+ */
 function dictZip(x, y) {
-    // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
+    /** @type {Record<T, U>} */
     const result = {};
     x.forEach((k, i) => {
         result[k] = y[i];
@@ -12,14 +19,31 @@ function dictZip(x, y) {
     return result;
 }
 exports.dictZip = dictZip;
+/**
+ * @param {number} x
+ * @param {number} y
+ * @returns {number[]}
+ */
 function range(x, y) {
     return Array.from({ length: y - x }, (_, i) => x + i);
 }
 exports.range = range;
+/**
+ * @param {string} x
+ * @returns {number}
+ */
 const getCharCode = (x) => x.charCodeAt(0);
 exports.getCharCode = getCharCode;
+/**
+ * @param {number} x
+ * @returns {string}
+ */
 const getCharFromCode = (x) => String.fromCharCode(x);
 exports.getCharFromCode = getCharFromCode;
+/**
+ * @param {string[]} word
+ * @returns {[string, string][]}
+ */
 function getPairs(word) {
     /** @type {[string, string][]} */
     const pairs = [];
@@ -32,6 +56,10 @@ function getPairs(word) {
     return pairs;
 }
 exports.getPairs = getPairs;
+/**
+ * @param {string} token
+ * @returns {string[]}
+ */
 const splitToken = (token) => token.split('');
 exports.splitToken = splitToken;
 //# sourceMappingURL=utils.js.map

package/cjs/utils.js.map

@@ -1 +1 @@
-{"version":3,"file":"utils.js","sourceRoot":"","sources":["../src/utils.ts"],"names":[],"mappings":";AAAA,0CAA0C;AAC1C,8CAA8C;;;AAE9C,SAAgB,OAAO,CACrB,CAAM,EACN,CAAM;IAEN,yEAAyE;IACzE,MAAM,MAAM,GAAG,EAAkB,CAAA;IACjC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACjB,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAE,CAAA;IACnB,CAAC,CAAC,CAAA;IACF,OAAO,MAAM,CAAA;AACf,CAAC;AAVD,0BAUC;AAED,SAAgB,KAAK,CAAC,CAAS,EAAE,CAAS;IACxC,OAAO,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;AACvD,CAAC;AAFD,sBAEC;AAEM,MAAM,WAAW,GAAG,CAAC,CAAS,EAAU,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAA;AAApD,QAAA,WAAW,eAAyC;AAE1D,MAAM,eAAe,GAAG,CAAC,CAAS,EAAU,EAAE,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAA;AAA/D,QAAA,eAAe,mBAAgD;AAE5E,SAAgB,QAAQ,CAAC,IAAc;IACrC,iCAAiC;IACjC,MAAM,KAAK,GAAuB,EAAE,CAAA;IACpC,IAAI,QAAQ,GAAG,IAAI,CAAC,CAAC,CAAC,CAAA;IACtB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;QACpC,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,CAAA;QACpB,KAAK,CAAC,IAAI,CAAC,CAAC,QAAS,EAAE,IAAK,CAAC,CAAC,CAAA;QAC9B,QAAQ,GAAG,IAAI,CAAA;KAChB;IACD,OAAO,KAAK,CAAA;AACd,CAAC;AAVD,4BAUC;AAEM,MAAM,UAAU,GAAG,CAAC,KAAa,EAAY,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;AAAzD,QAAA,UAAU,cAA+C"}
+{"version":3,"file":"utils.js","sourceRoot":"","sources":["../src/utils.js"],"names":[],"mappings":";AAAA,0CAA0C;AAC1C,8CAA8C;;;AAE9C;;;;;;GAMG;AACH,SAAgB,OAAO,CAAC,CAAC,EAAE,CAAC;IAC1B,2BAA2B;IAC3B,MAAM,MAAM,GAAG,EAAE,CAAA;IACjB,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACjB,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAA;IAClB,CAAC,CAAC,CAAA;IACF,OAAO,MAAM,CAAA;AACf,CAAC;AAPD,0BAOC;AAED;;;;GAIG;AACH,SAAgB,KAAK,CAAC,CAAC,EAAE,CAAC;IACxB,OAAO,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;AACvD,CAAC;AAFD,sBAEC;AAED;;;GAGG;AACI,MAAM,WAAW,GAAG,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAA;AAApC,QAAA,WAAW,eAAyB;AAEjD;;;GAGG;AACI,MAAM,eAAe,GAAG,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAA;AAA/C,QAAA,eAAe,mBAAgC;AAE5D;;;GAGG;AACH,SAAgB,QAAQ,CAAC,IAAI;IAC3B,iCAAiC;IACjC,MAAM,KAAK,GAAG,EAAE,CAAA;IAChB,IAAI,QAAQ,GAAG,IAAI,CAAC,CAAC,CAAC,CAAA;IACtB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;QACpC,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,CAAA;QACpB,KAAK,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,CAAA;QAC5B,QAAQ,GAAG,IAAI,CAAA;KAChB;IACD,OAAO,KAAK,CAAA;AACd,CAAC;AAVD,4BAUC;AAED;;;GAGG;AACI,MAAM,UAAU,GAAG,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;AAAvC,QAAA,UAAU,cAA6B"}