gpt-tokenizer 1.0.2 → 1.0.4

package/cjs/encoder.test.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"encoder.test.js","sourceRoot":"","sources":["../src/encoder.test.ts"],"names":[],"mappings":";;AAAA,kCAAkC;AAClC,uCAMkB;AAElB,QAAQ,CAAC,qBAAqB,EAAE,GAAG,EAAE;IACnC,EAAE,CAAC,cAAc,EAAE,GAAG,EAAE;QACtB,MAAM,GAAG,GAAG,EAAE,CAAA;QACd,MAAM,CAAC,IAAA,gBAAM,EAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;QAC/B,MAAM,CAAC,IAAA,gBAAM,EAAC,IAAA,gBAAM,EAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;QACxC,MAAM,CAAC,IAAA,4BAAkB,EAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QAC1C,MAAM,CAAC,IAAA,4BAAkB,EAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IAC5C,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;QACf,MAAM,GAAG,GAAG,GAAG,CAAA;QACf,MAAM,CAAC,IAAA,gBAAM,EAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;QAClC,MAAM,CAAC,IAAA,gBAAM,EAAC,IAAA,gBAAM,EAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;QACxC,MAAM,CAAC,IAAA,4BAAkB,EAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QAC1C,MAAM,CAAC,IAAA,4BAAkB,EAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;IAChD,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;QACb,MAAM,GAAG,GAAG,IAAI,CAAA;QAChB,MAAM,CAAC,IAAA,gBAAM,EAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;QAClC,MAAM,CAAC,IAAA,gBAAM,EAAC,IAAA,gBAAM,EAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;IAC1C,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,aAAa,EAAE,GAAG,EAAE;QACrB,MAAM,GAAG,GAAG,mBAAmB,CAAA;QAC/B,MAAM,CAAC,IAAA,gBAAM,EAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAK,EAAE,GAAG,EAAE,GAAG,EAAE,IAAK,CAAC,CAAC,CAAA;QACrD,MAAM,CAAC,IAAA,gBAAM,EAAC,IAAA,gBAAM,EAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;QACxC,MAAM,CAAC,IAAA,4BAAkB,EAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QAC9C,MAAM,CAAC,IAAA,4BAAkB,EAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IAC5C,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,kBAAkB,EAAE,GAAG,EAAE;QAC1B,MAAM,GAAG,GAAG,aAAa,CAAA;QACzB,MAAM,CAAC,IAAA,gBAAM,EAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,KAAM,CAAC,CAAC,CAAA;QAC/C,MAAM,CAAC,IAAA,gBAAM,EAAC,IAAA,gBAAM,EAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;QACxC,MAAM,CAAC,IAAA,4BAAkB,EAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IAC5C,CAAC,CAAC,CAAA;IAEF,MAAM,gBAAgB,GAAG,CAAC,KAAM,EAAE,KAAM,EAAE,GAAG,EAAE,GAAG,EAAE,KAAM,EAAE,GAAG,EAAE,GAAG,CAAC,CAAA;IAErE,EAAE,CAAC,QAAQ,EAAE,GAAG,EAAE;QAChB,MAAM,GAAG,GAAG,mBAAmB,CAAA;QAC/B,MAAM,CAAC,IAAA,gBAAM,EAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAA;QAC7C,MAAM,CAAC,IAAA,gBAAM,EAAC,IAAA,gBAAM,EAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;QACxC,MAAM,CAAC,IAAA,4BAAkB,EAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QAC9C,MAAM,CAAC,IAAA,4BAAkB,EAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IAC9C,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,qCAAqC,EAAE,GAAG,EAAE;QAC7C,MAAM,SAAS,GAAG,IAAA,yBAAe,EAAC,gBAAgB,CAAC,CAAA;QACnD,MAAM,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;QAC5C,MAAM,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QACxC,MAAM,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACzC,MAAM,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;QAC7C,MAAM,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QACxC,MAAM,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IAC3C,CAAC,CAAC,CAAA;IAEF,KAAK,SAAS,CAAC,CAAC,wBAAwB;QACtC,KAAK,MAAM,KAAK,IAAI,gBAAgB,EAAE;YACpC,4CAA4C;YAC5C,MAAM,MAAM,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;SACnC;IACH,CAAC;IAED,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;QACzD,MAAM,SAAS,GAAG,IAAA,8BAAoB,EAAC,wBAAwB,EAAE,CAAC,CAAA;QAClE,MAAM,OAAO,GAAG,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,EAAE,IAAI,CAAC,CAAA;QACzD,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,SAAS,EAAE;YACnC,MAAM,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,CAAA;SACvC;IACH,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,sBAAsB,EAAE,GAAG,EAAE;QAC9B,MAAM,GAAG,GAAG,6CAA6C,CAAA;QAEzD,MAAM,CAAC,IAAA,gBAAM,EAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC;YAC1B,IAAK,EAAE,KAAM,EAAE,KAAM,EAAE,GAAG,EAAE,KAAM,EAAE,KAAM,EAAE,IAAK,EAAE,IAAK;SACzD,CAAC,CAAA;QACF,MAAM,CAAC,IAAA,gBAAM,EAAC,IAAA,gBAAM,EAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;IAC1C,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,kBAAkB,EAAE,GAAG,EAAE;QAC1B,MAAM,GAAG,GAAG,8CAA8C,CAAA;QAC1D,MAAM,CAAC,IAAA,gBAAM,EAAC,IAAA,gBAAM,EAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;QACxC,MAAM,CAAC,IAAA,gBAAM,EAAC,GAAG,CAAC,CAAC,CAAC,aAAa,CAAC;YAChC,KAAM,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,IAAK,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG;YACtE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE;SACf,CAAC,CAAA;QACF,MAAM,CAAC,IAAA,4BAAkB,EAAC,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QAC/C,MAAM,CAAC,IAAA,4BAAkB,EAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IAC/C,CAAC,CAAC,CAAA;AACJ,CAAC,CAAC,CAAA"}

package/cjs/getBpe.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"getBpe.js","sourceRoot":"","sources":["../src/getBpe.ts"],"names":[],"mappings":";;;;;AAAA,4CAAmB;AACnB,gDAAuB;AACvB,mCAAwC;AAExC,MAAM,OAAO,GAAG,YAAE,CAAC,YAAY,CAC7B,cAAI,CAAC,IAAI,CAAC,SAAS,EAAE,mBAAmB,CAAC,EACzC,MAAM,CACP,CAAA;AACD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;AAEjC,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAC7C,CAAC;KACE,KAAK,CAAC,OAAO,CAAC;KACd,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC;KAClC,IAAI,CAAC,GAAG,CAAC,CACb,CAAA;AAED,MAAM,QAAQ,GAAG,IAAA,eAAO,EAAC,SAAS,EAAE,IAAA,aAAK,EAAC,CAAC,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC,CAAA;AAE/D,YAAE,CAAC,aAAa,CACd,cAAI,CAAC,IAAI,CAAC,SAAS,EAAE,uBAAuB,CAAC,EAC7C,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CACzB,CAAA"}

package/esm/encoder.d.ts

@@ -1,16 +0,0 @@
-/**
- * @returns {false | number} false if token limit is exceeded, otherwise the number of tokens
- */
-export declare function isWithinTokenLimit(text: string, tokenLimit: number, cache?: Map<string, string>): false | number;
-export declare function encode(text: string, cache?: Map<string, string>): number[];
-export declare function decodeToken(token: number): string;
-export declare function endsWithIncompleteUtfPairSurrogate(string: string): boolean;
-export declare function decodeGenerator(tokens: Iterable<number>): Generator<string, void, undefined>;
-/**
- * Decode tokens asynchronously and yield the decoded strings, one by one.
- * Will not yield for tokens that include a high surrogate, but wait for the next token.
- * @param {AsyncIterable<number>} tokens
- * @returns {AsyncGenerator<string, void, undefined>}
- */
-export declare function decodeAsyncGenerator(tokens: AsyncIterable<number>): AsyncGenerator<string, void, undefined>;
-export declare function decode(tokens: number[]): string;

package/esm/encoder.js

@@ -1,188 +0,0 @@
-/* eslint-disable unicorn/prefer-spread */
-/* eslint-disable unicorn/prefer-code-point */
-// eslint-disable-next-line @typescript-eslint/ban-ts-comment, @typescript-eslint/prefer-ts-expect-error
-// @ts-ignore
-import bpeRanksData from '../data/bpeRanks.json';
-// eslint-disable-next-line @typescript-eslint/ban-ts-comment, @typescript-eslint/prefer-ts-expect-error
-// @ts-ignore
-import encoderData from '../data/encoder.json';
-import { dictZip, getCharCode, getCharFromCode, getPairs, range, splitToken, } from './utils';
-const encoder = encoderData;
-const bpeRanks = bpeRanksData;
-const decoder = dictZip(Object.values(encoder), Object.keys(encoder));
-const textEncoder = new TextEncoder();
-const textDecoder = new TextDecoder();
-// eslint-disable-next-line no-magic-numbers
-const UPPER_BYTE_VALUE = 2 ** 8;
-function createByteToUnicodeMap() {
-    const asciiRange = range(getCharCode('!'), getCharCode('~') + 1);
-    const latin1Range1 = range(getCharCode('¡'), getCharCode('¬') + 1);
-    const latin1Range2 = range(getCharCode('®'), getCharCode('ÿ') + 1);
-    const initialCodePoints = [...asciiRange, ...latin1Range1, ...latin1Range2];
-    const mappedCodePoints = [...initialCodePoints];
-    let newCodePointOffset = 0;
-    for (let byteValue = 0; byteValue < UPPER_BYTE_VALUE; byteValue++) {
-        if (!initialCodePoints.includes(byteValue)) {
-            initialCodePoints.push(byteValue);
-            mappedCodePoints.push(UPPER_BYTE_VALUE + newCodePointOffset);
-            newCodePointOffset += 1;
-        }
-    }
-    const unicodeChars = mappedCodePoints.map(getCharFromCode);
-    return new Map(initialCodePoints.map((x, i) => [x, unicodeChars[i]]));
-}
-const pat = /'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+/gu;
-const byteEncoder = createByteToUnicodeMap();
-const byteDecoder = new Map([...byteEncoder].map(([k, v]) => [v, k]));
-function bpe(token, cache = new Map()) {
-    if (cache.has(token)) {
-        return cache.get(token);
-    }
-    let word = splitToken(token);
-    let pairs = getPairs(word);
-    if (pairs.length === 0) {
-        return token;
-    }
-    // eslint-disable-next-line no-constant-condition
-    while (true) {
-        const minPairs = new Map();
-        pairs.forEach((pair) => {
-            const rank = bpeRanks[pair.join(',')];
-            minPairs.set(
-            // eslint-disable-next-line no-magic-numbers
-            typeof rank === 'undefined' || Number.isNaN(rank) ? 10e10 : rank, pair);
-        });
-        const keys = [...minPairs.keys()];
-        const bigram = minPairs.get(Math.min(...keys));
-        if (!bigram || !(bigram.join(',') in bpeRanks)) {
-            break;
-        }
-        const [first, second] = bigram;
-        let newWord = [];
-        let i = 0;
-        while (i < word.length) {
-            const j = word.indexOf(first, i);
-            if (j === -1) {
-                newWord = newWord.concat(word.slice(i));
-                break;
-            }
-            newWord = newWord.concat(word.slice(i, j));
-            i = j;
-            if (word[i] === first && i < word.length - 1 && word[i + 1] === second) {
-                newWord.push(first + second);
-                i += 2;
-            }
-            else {
-                newWord.push(word[i]);
-                i += 1;
-            }
-        }
-        word = newWord;
-        if (word.length === 1) {
-            break;
-        }
-        else {
-            pairs = getPairs(word);
-        }
-    }
-    const result = word.join(' ');
-    cache.set(token, result);
-    return result;
-}
-function* encodeGenerator(text, cache = new Map()) {
-    for (let [token] of text.matchAll(pat)) {
-        token = [...textEncoder.encode(token)]
-            .map((x) => byteEncoder.get(x) ?? '')
-            .join('');
-        const newTokens = bpe(token, cache)
-            .split(' ')
-            .map((x) => encoder[x]);
-        yield newTokens;
-    }
-}
-/**
- * @returns {false | number} false if token limit is exceeded, otherwise the number of tokens
- */
-export function isWithinTokenLimit(text, tokenLimit, cache = new Map()) {
-    const tokenGenerator = encodeGenerator(text, cache);
-    let count = 0;
-    for (const tokens of tokenGenerator) {
-        count += tokens.length;
-        if (count > tokenLimit) {
-            return false;
-        }
-    }
-    return count;
-}
-export function encode(text, cache = new Map()) {
-    return [...encodeGenerator(text, cache)].flat(1);
-}
-export function decodeToken(token) {
-    const decodedToken = decoder[token];
-    if (typeof decodedToken === 'undefined') {
-        return '';
-    }
-    const decodedBytes = splitToken(decodedToken).map((x) => byteDecoder.get(x));
-    return textDecoder.decode(new Uint8Array(decodedBytes), {
-        stream: true,
-    });
-}
-const HIGH_SURROGATE_START = 55296;
-const HIGH_SURROGATE_END = 56319;
-export function endsWithIncompleteUtfPairSurrogate(string) {
-    if (string.length === 0)
-        return false;
-    // Check if the last character is a high surrogate
-    const lastCharCode = string.charCodeAt(string.length - 1);
-    return (lastCharCode >= HIGH_SURROGATE_START && lastCharCode <= HIGH_SURROGATE_END);
-}
-export function* decodeGenerator(tokens) {
-    let buffer = '';
-    for (const token of tokens) {
-        buffer += decodeToken(token);
-        if (buffer.length === 0 || endsWithIncompleteUtfPairSurrogate(buffer)) {
-            // Keep the high surrogate in the buffer and continue with the next token
-            // eslint-disable-next-line no-continue
-            continue;
-        }
-        else {
-            yield buffer;
-            // reset buffer
-            buffer = '';
-        }
-    }
-    // Yield any remaining characters in the buffer
-    if (buffer.length > 0) {
-        yield buffer;
-    }
-}
-/**
- * Decode tokens asynchronously and yield the decoded strings, one by one.
- * Will not yield for tokens that include a high surrogate, but wait for the next token.
- * @param {AsyncIterable<number>} tokens
- * @returns {AsyncGenerator<string, void, undefined>}
- */
-export async function* decodeAsyncGenerator(tokens) {
-    let buffer = '';
-    for await (const token of tokens) {
-        buffer += decodeToken(token);
-        if (buffer.length === 0 || endsWithIncompleteUtfPairSurrogate(buffer)) {
-            // Keep the high surrogate in the buffer and continue with the next token
-            // eslint-disable-next-line no-continue
-            continue;
-        }
-        else {
-            yield buffer;
-            // reset buffer
-            buffer = '';
-        }
-    }
-    // Yield any remaining characters in the buffer
-    if (buffer.length > 0) {
-        yield buffer;
-    }
-}
-export function decode(tokens) {
-    return [...decodeGenerator(tokens)].join('');
-}
-//# sourceMappingURL=encoder.js.map

package/esm/encoder.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"encoder.js","sourceRoot":"","sources":["../src/encoder.ts"],"names":[],"mappings":"AAAA,0CAA0C;AAC1C,8CAA8C;AAE9C,wGAAwG;AACxG,aAAa;AACb,OAAO,YAAY,MAAM,uBAAuB,CAAA;AAChD,wGAAwG;AACxG,aAAa;AACb,OAAO,WAAW,MAAM,sBAAsB,CAAA;AAC9C,OAAO,EACL,OAAO,EACP,WAAW,EACX,eAAe,EACf,QAAQ,EACR,KAAK,EACL,UAAU,GACX,MAAM,SAAS,CAAA;AAEhB,MAAM,OAAO,GAA2B,WAAW,CAAA;AACnD,MAAM,QAAQ,GAA2B,YAAY,CAAA;AAErD,MAAM,OAAO,GAA2B,OAAO,CAC7C,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,EACtB,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CACrB,CAAA;AAED,MAAM,WAAW,GAAG,IAAI,WAAW,EAAE,CAAA;AACrC,MAAM,WAAW,GAAG,IAAI,WAAW,EAAE,CAAA;AAErC,4CAA4C;AAC5C,MAAM,gBAAgB,GAAG,CAAC,IAAI,CAAC,CAAA;AAE/B,SAAS,sBAAsB;IAC7B,MAAM,UAAU,GAAG,KAAK,CAAC,WAAW,CAAC,GAAG,CAAC,EAAE,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAA;IAChE,MAAM,YAAY,GAAG,KAAK,CAAC,WAAW,CAAC,GAAG,CAAC,EAAE,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAA;IAClE,MAAM,YAAY,GAAG,KAAK,CAAC,WAAW,CAAC,GAAG,CAAC,EAAE,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAA;IAElE,MAAM,iBAAiB,GAAG,CAAC,GAAG,UAAU,EAAE,GAAG,YAAY,EAAE,GAAG,YAAY,CAAC,CAAA;IAC3E,MAAM,gBAAgB,GAAG,CAAC,GAAG,iBAAiB,CAAC,CAAA;IAE/C,IAAI,kBAAkB,GAAG,CAAC,CAAA;IAC1B,KAAK,IAAI,SAAS,GAAG,CAAC,EAAE,SAAS,GAAG,gBAAgB,EAAE,SAAS,EAAE,EAAE;QACjE,IAAI,CAAC,iBAAiB,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE;YAC1C,iBAAiB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;YACjC,gBAAgB,CAAC,IAAI,CAAC,gBAAgB,GAAG,kBAAkB,CAAC,CAAA;YAC5D,kBAAkB,IAAI,CAAC,CAAA;SACxB;KACF;IAED,MAAM,YAAY,GAAG,gBAAgB,CAAC,GAAG,CAAC,eAAe,CAAC,CAAA;IAC1D,OAAO,IAAI,GAAG,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC,CAAE,CAAC,CAAC,CAAC,CAAA;AACxE,CAAC;AAED,MAAM,GAAG,GACP,8EAA8E,CAAA;AAEhF,MAAM,WAAW,GAAG,sBAAsB,EAAE,CAAA;AAC5C,MAAM,WAAW,GAAwB,IAAI,GAAG,CAC9C,CAAC,GAAG,WAAW,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CACzC,CAAA;AAED,SAAS,GAAG,CAAC,KAAa,EAAE,QAA6B,IAAI,GAAG,EAAE;IAChE,IAAI,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE;QACpB,OAAO,KAAK,CAAC,GAAG,CAAC,KAAK,CAAE,CAAA;KACzB;IAED,IAAI,IAAI,GAAG,UAAU,CAAC,KAAK,CAAC,CAAA;IAC5B,IAAI,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAA;IAE1B,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE;QACtB,OAAO,KAAK,CAAA;KACb;IAED,iDAAiD;IACjD,OAAO,IAAI,EAAE;QACX,MAAM,QAAQ,GAAkC,IAAI,GAAG,EAAE,CAAA;QACzD,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;YACrB,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAA;YACrC,QAAQ,CAAC,GAAG;YACV,4CAA4C;YAC5C,OAAO,IAAI,KAAK,WAAW,IAAI,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAChE,IAAI,CACL,CAAA;QACH,CAAC,CAAC,CAAA;QACF,MAAM,IAAI,GAAG,CAAC,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAA;QAEjC,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC,CAAA;QAE9C,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,QAAQ,CAAC,EAAE;YAC9C,MAAK;SACN;QAED,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,GAAG,MAAM,CAAA;QAC9B,IAAI,OAAO,GAAa,EAAE,CAAA;QAC1B,IAAI,CAAC,GAAG,CAAC,CAAA;QAET,OAAO,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE;YACtB,MAAM,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,CAAA;YAChC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE;gBACZ,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAA;gBACvC,MAAK;aACN;YACD,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;YAC1C,CAAC,GAAG,CAAC,CAAA;YAEL,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,MAAM,EAAE;gBACtE,OAAO,CAAC,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC,CAAA;gBAC5B,CAAC,IAAI,CAAC,CAAA;aACP;iBAAM;gBACL,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAE,CAAC,CAAA;gBACtB,CAAC,IAAI,CAAC,CAAA;aACP;SACF;QAED,IAAI,GAAG,OAAO,CAAA;QACd,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE;YACrB,MAAK;SACN;aAAM;YACL,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAA;SACvB;KACF;IAED,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;IAC7B,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,CAAA;IAExB,OAAO,MAAM,CAAA;AACf,CAAC;AAED,QAAQ,CAAC,CAAC,eAAe,CACvB,IAAY,EACZ,QAA6B,IAAI,GAAG,EAAE;IAEtC,KAAK,IAAI,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE;QACtC,KAAK,GAAG,CAAC,GAAG,WAAW,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;aACnC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;aACpC,IAAI,CAAC,EAAE,CAAC,CAAA;QAEX,MAAM,SAAS,GAAG,GAAG,CAAC,KAAK,EAAE,KAAK,CAAC;aAChC,KAAK,CAAC,GAAG,CAAC;aACV,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,CAAE,CAAC,CAAA;QAE1B,MAAM,SAAS,CAAA;KAChB;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,kBAAkB,CAChC,IAAY,EACZ,UAAkB,EAClB,QAA6B,IAAI,GAAG,EAAE;IAEtC,MAAM,cAAc,GAAG,eAAe,CAAC,IAAI,EAAE,KAAK,CAAC,CAAA;IACnD,IAAI,KAAK,GAAG,CAAC,CAAA;IACb,KAAK,MAAM,MAAM,IAAI,cAAc,EAAE;QACnC,KAAK,IAAI,MAAM,CAAC,MAAM,CAAA;QACtB,IAAI,KAAK,GAAG,UAAU,EAAE;YACtB,OAAO,KAAK,CAAA;SACb;KACF;IACD,OAAO,KAAK,CAAA;AACd,CAAC;AAED,MAAM,UAAU,MAAM,CACpB,IAAY,EACZ,QAA6B,IAAI,GAAG,EAAE;IAEtC,OAAO,CAAC,GAAG,eAAe,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;AAClD,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,KAAa;IACvC,MAAM,YAAY,GAAG,OAAO,CAAC,KAAK,CAAC,CAAA;IACnC,IAAI,OAAO,YAAY,KAAK,WAAW,EAAE;QACvC,OAAO,EAAE,CAAA;KACV;IACD,MAAM,YAAY,GAAG,UAAU,CAAC,YAAY,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC,CAAA;IAC7E,OAAO,WAAW,CAAC,MAAM,CAAC,IAAI,UAAU,CAAC,YAAY,CAAC,EAAE;QACtD,MAAM,EAAE,IAAI;KACb,CAAC,CAAA;AACJ,CAAC;AAED,MAAM,oBAAoB,GAAG,KAAM,CAAA;AACnC,MAAM,kBAAkB,GAAG,KAAM,CAAA;AACjC,MAAM,UAAU,kCAAkC,CAAC,MAAc;IAC/D,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,KAAK,CAAA;IACrC,kDAAkD;IAClD,MAAM,YAAY,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;IACzD,OAAO,CACL,YAAY,IAAI,oBAAoB,IAAI,YAAY,IAAI,kBAAkB,CAC3E,CAAA;AACH,CAAC;AAED,MAAM,SAAS,CAAC,CAAC,eAAe,CAC9B,MAAwB;IAExB,IAAI,MAAM,GAAG,EAAE,CAAA;IAEf,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE;QAC1B,MAAM,IAAI,WAAW,CAAC,KAAK,CAAC,CAAA;QAE5B,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,kCAAkC,CAAC,MAAM,CAAC,EAAE;YACrE,yEAAyE;YACzE,uCAAuC;YACvC,SAAQ;SACT;aAAM;YACL,MAAM,MAAM,CAAA;YACZ,eAAe;YACf,MAAM,GAAG,EAAE,CAAA;SACZ;KACF;IAED,+CAA+C;IAC/C,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE;QACrB,MAAM,MAAM,CAAA;KACb;AACH,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,oBAAoB,CACzC,MAA6B;IAE7B,IAAI,MAAM,GAAG,EAAE,CAAA;IAEf,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,MAAM,EAAE;QAChC,MAAM,IAAI,WAAW,CAAC,KAAK,CAAC,CAAA;QAE5B,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,kCAAkC,CAAC,MAAM,CAAC,EAAE;YACrE,yEAAyE;YACzE,uCAAuC;YACvC,SAAQ;SACT;aAAM;YACL,MAAM,MAAM,CAAA;YACZ,eAAe;YACf,MAAM,GAAG,EAAE,CAAA;SACZ;KACF;IAED,+CAA+C;IAC/C,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE;QACrB,MAAM,MAAM,CAAA;KACb;AACH,CAAC;AAED,MAAM,UAAU,MAAM,CAAC,MAAgB;IACrC,OAAO,CAAC,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;AAC9C,CAAC"}

package/esm/encoder.test.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"encoder.test.js","sourceRoot":"","sources":["../src/encoder.test.ts"],"names":[],"mappings":"AAAA,kCAAkC;AAClC,OAAO,EACL,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,kBAAkB,GACnB,MAAM,WAAW,CAAA;AAElB,QAAQ,CAAC,qBAAqB,EAAE,GAAG,EAAE;IACnC,EAAE,CAAC,cAAc,EAAE,GAAG,EAAE;QACtB,MAAM,GAAG,GAAG,EAAE,CAAA;QACd,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;QAC/B,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;QACxC,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QAC1C,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IAC5C,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;QACf,MAAM,GAAG,GAAG,GAAG,CAAA;QACf,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;QAClC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;QACxC,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QAC1C,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;IAChD,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;QACb,MAAM,GAAG,GAAG,IAAI,CAAA;QAChB,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;QAClC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;IAC1C,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,aAAa,EAAE,GAAG,EAAE;QACrB,MAAM,GAAG,GAAG,mBAAmB,CAAA;QAC/B,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAK,EAAE,GAAG,EAAE,GAAG,EAAE,IAAK,CAAC,CAAC,CAAA;QACrD,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;QACxC,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QAC9C,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IAC5C,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,kBAAkB,EAAE,GAAG,EAAE;QAC1B,MAAM,GAAG,GAAG,aAAa,CAAA;QACzB,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,KAAM,CAAC,CAAC,CAAA;QAC/C,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;QACxC,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IAC5C,CAAC,CAAC,CAAA;IAEF,MAAM,gBAAgB,GAAG,CAAC,KAAM,EAAE,KAAM,EAAE,GAAG,EAAE,GAAG,EAAE,KAAM,EAAE,GAAG,EAAE,GAAG,CAAC,CAAA;IAErE,EAAE,CAAC,QAAQ,EAAE,GAAG,EAAE;QAChB,MAAM,GAAG,GAAG,mBAAmB,CAAA;QAC/B,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAA;QAC7C,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;QACxC,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QAC9C,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IAC9C,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,qCAAqC,EAAE,GAAG,EAAE;QAC7C,MAAM,SAAS,GAAG,eAAe,CAAC,gBAAgB,CAAC,CAAA;QACnD,MAAM,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;QAC5C,MAAM,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QACxC,MAAM,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACzC,MAAM,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;QAC7C,MAAM,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QACxC,MAAM,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IAC3C,CAAC,CAAC,CAAA;IAEF,KAAK,SAAS,CAAC,CAAC,wBAAwB;QACtC,KAAK,MAAM,KAAK,IAAI,gBAAgB,EAAE;YACpC,4CAA4C;YAC5C,MAAM,MAAM,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;SACnC;IACH,CAAC;IAED,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;QACzD,MAAM,SAAS,GAAG,oBAAoB,CAAC,wBAAwB,EAAE,CAAC,CAAA;QAClE,MAAM,OAAO,GAAG,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,EAAE,IAAI,CAAC,CAAA;QACzD,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,SAAS,EAAE;YACnC,MAAM,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,CAAA;SACvC;IACH,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,sBAAsB,EAAE,GAAG,EAAE;QAC9B,MAAM,GAAG,GAAG,6CAA6C,CAAA;QAEzD,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC;YAC1B,IAAK,EAAE,KAAM,EAAE,KAAM,EAAE,GAAG,EAAE,KAAM,EAAE,KAAM,EAAE,IAAK,EAAE,IAAK;SACzD,CAAC,CAAA;QACF,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;IAC1C,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,kBAAkB,EAAE,GAAG,EAAE;QAC1B,MAAM,GAAG,GAAG,8CAA8C,CAAA;QAC1D,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;QACxC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,aAAa,CAAC;YAChC,KAAM,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,IAAK,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG;YACtE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE;SACf,CAAC,CAAA;QACF,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QAC/C,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IAC/C,CAAC,CAAC,CAAA;AACJ,CAAC,CAAC,CAAA"}

package/esm/getBpe.js

@@ -1,12 +0,0 @@
-import fs from 'fs';
-import path from 'path';
-import { dictZip, range } from './utils';
-const bpeFile = fs.readFileSync(path.join(__dirname, '../data/vocab.bpe'), 'utf8');
-const lines = bpeFile.split('\n');
-const bpeMerges = lines.slice(1, -1).map((x) => x
-    .split(/(\s+)/)
-    .filter((e) => e.trim().length > 0)
-    .join(','));
-const bpeRanks = dictZip(bpeMerges, range(0, bpeMerges.length));
-fs.writeFileSync(path.join(__dirname, '../data/bpeRanks.json'), JSON.stringify(bpeRanks));
-//# sourceMappingURL=getBpe.js.map

package/esm/getBpe.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"getBpe.js","sourceRoot":"","sources":["../src/getBpe.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,IAAI,CAAA;AACnB,OAAO,IAAI,MAAM,MAAM,CAAA;AACvB,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,SAAS,CAAA;AAExC,MAAM,OAAO,GAAG,EAAE,CAAC,YAAY,CAC7B,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,mBAAmB,CAAC,EACzC,MAAM,CACP,CAAA;AACD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;AAEjC,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAC7C,CAAC;KACE,KAAK,CAAC,OAAO,CAAC;KACd,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC;KAClC,IAAI,CAAC,GAAG,CAAC,CACb,CAAA;AAED,MAAM,QAAQ,GAAG,OAAO,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC,CAAA;AAE/D,EAAE,CAAC,aAAa,CACd,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,uBAAuB,CAAC,EAC7C,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CACzB,CAAA"}

package/src/encoder.ts

@@ -1,252 +0,0 @@
-/* eslint-disable unicorn/prefer-spread */
-/* eslint-disable unicorn/prefer-code-point */
-
-// eslint-disable-next-line @typescript-eslint/ban-ts-comment, @typescript-eslint/prefer-ts-expect-error
-// @ts-ignore
-import bpeRanksData from '../data/bpeRanks.json'
-// eslint-disable-next-line @typescript-eslint/ban-ts-comment, @typescript-eslint/prefer-ts-expect-error
-// @ts-ignore
-import encoderData from '../data/encoder.json'
-import {
-  dictZip,
-  getCharCode,
-  getCharFromCode,
-  getPairs,
-  range,
-  splitToken,
-} from './utils'
-
-const encoder: Record<string, number> = encoderData
-const bpeRanks: Record<string, number> = bpeRanksData
-
-const decoder: Record<number, string> = dictZip(
-  Object.values(encoder),
-  Object.keys(encoder),
-)
-
-const textEncoder = new TextEncoder()
-const textDecoder = new TextDecoder()
-
-// eslint-disable-next-line no-magic-numbers
-const UPPER_BYTE_VALUE = 2 ** 8
-
-function createByteToUnicodeMap(): Map<number, string> {
-  const asciiRange = range(getCharCode('!'), getCharCode('~') + 1)
-  const latin1Range1 = range(getCharCode('¡'), getCharCode('¬') + 1)
-  const latin1Range2 = range(getCharCode('®'), getCharCode('ÿ') + 1)
-
-  const initialCodePoints = [...asciiRange, ...latin1Range1, ...latin1Range2]
-  const mappedCodePoints = [...initialCodePoints]
-
-  let newCodePointOffset = 0
-  for (let byteValue = 0; byteValue < UPPER_BYTE_VALUE; byteValue++) {
-    if (!initialCodePoints.includes(byteValue)) {
-      initialCodePoints.push(byteValue)
-      mappedCodePoints.push(UPPER_BYTE_VALUE + newCodePointOffset)
-      newCodePointOffset += 1
-    }
-  }
-
-  const unicodeChars = mappedCodePoints.map(getCharFromCode)
-  return new Map(initialCodePoints.map((x, i) => [x, unicodeChars[i]!]))
-}
-
-const pat =
-  /'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+/gu
-
-const byteEncoder = createByteToUnicodeMap()
-const byteDecoder: Map<string, number> = new Map(
-  [...byteEncoder].map(([k, v]) => [v, k]),
-)
-
-function bpe(token: string, cache: Map<string, string> = new Map()): string {
-  if (cache.has(token)) {
-    return cache.get(token)!
-  }
-
-  let word = splitToken(token)
-  let pairs = getPairs(word)
-
-  if (pairs.length === 0) {
-    return token
-  }
-
-  // eslint-disable-next-line no-constant-condition
-  while (true) {
-    const minPairs: Map<number, [string, string]> = new Map()
-    pairs.forEach((pair) => {
-      const rank = bpeRanks[pair.join(',')]
-      minPairs.set(
-        // eslint-disable-next-line no-magic-numbers
-        typeof rank === 'undefined' || Number.isNaN(rank) ? 10e10 : rank,
-        pair,
-      )
-    })
-    const keys = [...minPairs.keys()]
-
-    const bigram = minPairs.get(Math.min(...keys))
-
-    if (!bigram || !(bigram.join(',') in bpeRanks)) {
-      break
-    }
-
-    const [first, second] = bigram
-    let newWord: string[] = []
-    let i = 0
-
-    while (i < word.length) {
-      const j = word.indexOf(first, i)
-      if (j === -1) {
-        newWord = newWord.concat(word.slice(i))
-        break
-      }
-      newWord = newWord.concat(word.slice(i, j))
-      i = j
-
-      if (word[i] === first && i < word.length - 1 && word[i + 1] === second) {
-        newWord.push(first + second)
-        i += 2
-      } else {
-        newWord.push(word[i]!)
-        i += 1
-      }
-    }
-
-    word = newWord
-    if (word.length === 1) {
-      break
-    } else {
-      pairs = getPairs(word)
-    }
-  }
-
-  const result = word.join(' ')
-  cache.set(token, result)
-
-  return result
-}
-
-function* encodeGenerator(
-  text: string,
-  cache: Map<string, string> = new Map(),
-): Generator<number[], void, undefined> {
-  for (let [token] of text.matchAll(pat)) {
-    token = [...textEncoder.encode(token)]
-      .map((x) => byteEncoder.get(x) ?? '')
-      .join('')
-
-    const newTokens = bpe(token, cache)
-      .split(' ')
-      .map((x) => encoder[x]!)
-
-    yield newTokens
-  }
-}
-
-/**
- * @returns {false | number} false if token limit is exceeded, otherwise the number of tokens
- */
-export function isWithinTokenLimit(
-  text: string,
-  tokenLimit: number,
-  cache: Map<string, string> = new Map(),
-): false | number {
-  const tokenGenerator = encodeGenerator(text, cache)
-  let count = 0
-  for (const tokens of tokenGenerator) {
-    count += tokens.length
-    if (count > tokenLimit) {
-      return false
-    }
-  }
-  return count
-}
-
-export function encode(
-  text: string,
-  cache: Map<string, string> = new Map(),
-): number[] {
-  return [...encodeGenerator(text, cache)].flat(1)
-}
-
-export function decodeToken(token: number): string {
-  const decodedToken = decoder[token]
-  if (typeof decodedToken === 'undefined') {
-    return ''
-  }
-  const decodedBytes = splitToken(decodedToken).map((x) => byteDecoder.get(x)!)
-  return textDecoder.decode(new Uint8Array(decodedBytes), {
-    stream: true,
-  })
-}
-
-const HIGH_SURROGATE_START = 55_296
-const HIGH_SURROGATE_END = 56_319
-export function endsWithIncompleteUtfPairSurrogate(string: string): boolean {
-  if (string.length === 0) return false
-  // Check if the last character is a high surrogate
-  const lastCharCode = string.charCodeAt(string.length - 1)
-  return (
-    lastCharCode >= HIGH_SURROGATE_START && lastCharCode <= HIGH_SURROGATE_END
-  )
-}
-
-export function* decodeGenerator(
-  tokens: Iterable<number>,
-): Generator<string, void, undefined> {
-  let buffer = ''
-
-  for (const token of tokens) {
-    buffer += decodeToken(token)
-
-    if (buffer.length === 0 || endsWithIncompleteUtfPairSurrogate(buffer)) {
-      // Keep the high surrogate in the buffer and continue with the next token
-      // eslint-disable-next-line no-continue
-      continue
-    } else {
-      yield buffer
-      // reset buffer
-      buffer = ''
-    }
-  }
-
-  // Yield any remaining characters in the buffer
-  if (buffer.length > 0) {
-    yield buffer
-  }
-}
-
-/**
- * Decode tokens asynchronously and yield the decoded strings, one by one.
- * Will not yield for tokens that include a high surrogate, but wait for the next token.
- * @param {AsyncIterable<number>} tokens
- * @returns {AsyncGenerator<string, void, undefined>}
- */
-export async function* decodeAsyncGenerator(
-  tokens: AsyncIterable<number>,
-): AsyncGenerator<string, void, undefined> {
-  let buffer = ''
-
-  for await (const token of tokens) {
-    buffer += decodeToken(token)
-
-    if (buffer.length === 0 || endsWithIncompleteUtfPairSurrogate(buffer)) {
-      // Keep the high surrogate in the buffer and continue with the next token
-      // eslint-disable-next-line no-continue
-      continue
-    } else {
-      yield buffer
-      // reset buffer
-      buffer = ''
-    }
-  }
-
-  // Yield any remaining characters in the buffer
-  if (buffer.length > 0) {
-    yield buffer
-  }
-}
-
-export function decode(tokens: number[]): string {
-  return [...decodeGenerator(tokens)].join('')
-}

package/src/getBpe.ts

@@ -1,23 +0,0 @@
-import fs from 'fs'
-import path from 'path'
-import { dictZip, range } from './utils'
-
-const bpeFile = fs.readFileSync(
-  path.join(__dirname, '../data/vocab.bpe'),
-  'utf8',
-)
-const lines = bpeFile.split('\n')
-
-const bpeMerges = lines.slice(1, -1).map((x) =>
-  x
-    .split(/(\s+)/)
-    .filter((e) => e.trim().length > 0)
-    .join(','),
-)
-
-const bpeRanks = dictZip(bpeMerges, range(0, bpeMerges.length))
-
-fs.writeFileSync(
-  path.join(__dirname, '../data/bpeRanks.json'),
-  JSON.stringify(bpeRanks),
-)

package/src/utils.ts

@@ -1,36 +0,0 @@
-/* eslint-disable unicorn/prefer-spread */
-/* eslint-disable unicorn/prefer-code-point */
-
-export function dictZip<T extends PropertyKey, U>(
-  x: T[],
-  y: U[],
-): Record<T, U> {
-  // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
-  const result = {} as Record<T, U>
-  x.forEach((k, i) => {
-    result[k] = y[i]!
-  })
-  return result
-}
-
-export function range(x: number, y: number): number[] {
-  return Array.from({ length: y - x }, (_, i) => x + i)
-}
-
-export const getCharCode = (x: string): number => x.charCodeAt(0)
-
-export const getCharFromCode = (x: number): string => String.fromCharCode(x)
-
-export function getPairs(word: string[]): [string, string][] {
-  /** @type {[string, string][]} */
-  const pairs: [string, string][] = []
-  let prevChar = word[0]
-  for (let i = 1; i < word.length; i++) {
-    const char = word[i]
-    pairs.push([prevChar!, char!])
-    prevChar = char
-  }
-  return pairs
-}
-
-export const splitToken = (token: string): string[] => token.split('')