gpt-tokenizer 1.0.0

package/LICENSE

@@ -0,0 +1,23 @@
+MIT License
+
+Copyright (c) 2020 AIDungeon
+Copyright (c) 2023 syonfox
+Copyright (c) 2023 Bazyli Brzoska
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,194 @@
+# `gpt-tokenizer`
+
+`gpt-tokenizer` is a highly optimized Token Byte Pair Encoder/Decoder for GPT-2, GPT-3, GPT-3.5 and GPT-4 designed for JavaScript applications. OpenAI's GPT models utilize byte pair encoding to transform text into a sequence of integers before feeding them into the model. This package is a JavaScript implementation of OpenAI's original Python encoder/decoder, which can be found [here](https://github.com/openai/gpt-2).
+
+This package is a fork of [latitudegames/GPT-3-Encoder](https://github.com/latitudegames/GPT-3-Encoder), improving on various aspects, such as:
+
+- Adding generator versions of both decoder and encoder
+- Providing the ability to decode an asynchronous stream of data (using `decodeAsyncGenerator` and `decodeGenerator` with any iterable input)
+- Removing the global cache to prevent memory leaks
+- Adding a highly performant `isWithinTokenLimit` function to assess token limit without encoding the entire text
+- Improving overall performance by eliminating transitive arrays
+- Including precomputed `bpeRanks`
+- Adding type-checking
+- Fixing minor bugs (thanks to TypeScript)
+- Works in the browser out-of-the-box
+
+## Installation
+
+As NPM package:
+
+```bash
+npm install gpt-tokenizer
+```
+
+As an UMD module:
+
+```html
+<script src="https://unpkg.com/gpt-tokenizer" />
+```
+
+## Usage
+
+```typescript
+import {
+  encode,
+  decode,
+  isWithinTokenLimit,
+  encodeGenerator,
+  decodeGenerator,
+  decodeAsyncGenerator,
+} from 'gpt-tokenizer'
+
+const text = 'Hello, world!'
+const tokenLimit = 10
+
+// Encode text into tokens
+const tokens = encode(text)
+
+// Decode tokens back into text
+const decodedText = decode(tokens)
+
+// Check if text is within the token limit
+// returns false if the limit is exceeded, otherwise returns the actual number of tokens (truthy value)
+const withinTokenLimit = isWithinTokenLimit(text, tokenLimit)
+
+// Encode text using generator
+for (const tokenChunk of encodeGenerator(text)) {
+  console.log(tokenChunk)
+}
+
+// Decode tokens using generator
+for (const textChunk of decodeGenerator(tokens)) {
+  console.log(textChunk)
+}
+
+// Decode tokens using async generator
+// (assuming `asyncTokens` is an AsyncIterableIterator<number>)
+for await (const textChunk of decodeAsyncGenerator(asyncTokens)) {
+  console.log(textChunk)
+}
+```
+
+## Caching Between Runs of Encode-related Functions
+
+You may want to encode multiple pieces of text with similar content or structure. In such cases, using a single cache between runs of encode-related functions can help improve performance. By sharing the cache, you can reuse the results of previously calculated byte pair encodings, thereby reducing redundant computations.
+
+However, it's important to be aware of potential memory consumption issues when using a shared cache when encoding lots of higher range unicode characters (non-latin/complex alphabets, emojis), potentially leading to performance degradation or even crashes due to excessive memory usage.
+
+In such a case, it is recommended to use a custom `Map` implementation that uses a LRU cache to limit the size of the cache.
+
+Here's an example of how to use a shared cache between runs of the `encode` function:
+
+```typescript
+import { encode } from 'gpt-tokenizer'
+
+const cache = new Map()
+
+const text1 = 'Hello, world!'
+const text2 = 'Hello, everyone!'
+
+const tokens1 = encode(text1, cache)
+const tokens2 = encode(text2, cache)
+```
+
+## API
+
+### `encode(text: string, cache?: Map<string, string>): number[]`
+
+Encodes the given text into a sequence of tokens. Use this method when you need to transform a piece of text into the token format that GPT-2 or GPT-3 models can process. You can provide an optional cache to store and reuse byte pair encoding results between multiple calls.
+
+Example:
+
+```typescript
+import { encode } from 'gpt-tokenizer'
+
+const text = 'Hello, world!'
+const tokens = encode(text)
+```
+
+### `decode(tokens: number[]): string`
+
+Decodes a sequence of tokens back into text. Use this method when you want to convert the output tokens from GPT-2 or GPT-3 models back into human-readable text.
+
+Example:
+
+```typescript
+import { decode } from 'gpt-tokenizer'
+
+const tokens = [18435, 198, 23132, 328]
+const text = decode(tokens)
+```
+
+### `isWithinTokenLimit(text: string, tokenLimit: number, cache?: Map<string, string>): false | number`
+
+Checks if the text is within the token limit. Returns `false` if the limit is exceeded, otherwise returns the number of tokens. Use this method to quickly check if a given text is within the token limit imposed by GPT-2 or GPT-3 models, without encoding the entire text.
+
+Example:
+
+```typescript
+import { isWithinTokenLimit } from 'gpt-tokenizer'
+
+const text = 'Hello, world!'
+const tokenLimit = 10
+const withinTokenLimit = isWithinTokenLimit(text, tokenLimit)
+```
+
+### `encodeGenerator(text: string, cache?: Map<string, string>): Generator<number[], void, undefined>`
+
+Encodes the given text using a generator, yielding chunks of tokens.
+Use this method when you want to encode text in chunks, which can be useful for processing large texts or streaming data.
+
+Example:
+
+```typescript
+import { encodeGenerator } from 'gpt-tokenizer'
+
+const text = 'Hello, world!'
+const tokens = []
+for (const tokenChunk of encodeGenerator(text)) {
+  tokens.push(...tokenChunk)
+}
+```
+
+### `decodeGenerator(tokens: Iterable<number>): Generator<string, void, undefined>`
+
+Decodes a sequence of tokens using a generator, yielding chunks of decoded text.
+Use this method when you want to decode tokens in chunks, which can be useful for processing large outputs or streaming data.
+
+Example:
+
+```typescript
+import { decodeGenerator } from 'gpt-tokenizer'
+
+const tokens = [18435, 198, 23132, 328]
+let decodedText = ''
+for (const textChunk of decodeGenerator(tokens)) {
+  decodedText += textChunk
+}
+```
+
+### `decodeAsyncGenerator(tokens: AsyncIterable<number>): AsyncGenerator<string, void, undefined>`
+
+Decodes a sequence of tokens asynchronously using a generator, yielding chunks of decoded text. Use this method when you want to decode tokens in chunks asynchronously, which can be useful for processing large outputs or streaming data in an asynchronous context.
+
+Example:
+
+```javascript
+import { decodeAsyncGenerator } from 'gpt-tokenizer'
+
+async function processTokens(asyncTokensIterator) {
+  let decodedText = ''
+  for await (const textChunk of decodeAsyncGenerator(asyncTokensIterator)) {
+    decodedText += textChunk
+  }
+}
+```
+
+## License
+
+MIT
+
+## Contributing
+
+Contributions are welcome! Please open a pull request or an issue to discuss your ideas, bug reports, or any other inquiries.

package/cjs/encoder.d.ts

@@ -0,0 +1,16 @@
+/**
+ * @returns {false | number} false if token limit is exceeded, otherwise the number of tokens
+ */
+export declare function isWithinTokenLimit(text: string, tokenLimit: number, cache?: Map<string, string>): false | number;
+export declare function encode(text: string, cache?: Map<string, string>): number[];
+export declare function decodeToken(token: number): string;
+export declare function endsWithIncompleteUtfPairSurrogate(string: string): boolean;
+export declare function decodeGenerator(tokens: Iterable<number>): Generator<string, void, undefined>;
+/**
+ * Decode tokens asynchronously and yield the decoded strings, one by one.
+ * Will not yield for tokens that include a high surrogate, but wait for the next token.
+ * @param {AsyncIterable<number>} tokens
+ * @returns {AsyncGenerator<string, void, undefined>}
+ */
+export declare function decodeAsyncGenerator(tokens: AsyncIterable<number>): AsyncGenerator<string, void, undefined>;
+export declare function decode(tokens: number[]): string;

package/cjs/encoder.js

@@ -0,0 +1,201 @@
+"use strict";
+/* eslint-disable unicorn/prefer-spread */
+/* eslint-disable unicorn/prefer-code-point */
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.decode = exports.decodeAsyncGenerator = exports.decodeGenerator = exports.endsWithIncompleteUtfPairSurrogate = exports.decodeToken = exports.encode = exports.isWithinTokenLimit = void 0;
+// eslint-disable-next-line @typescript-eslint/ban-ts-comment, @typescript-eslint/prefer-ts-expect-error
+// @ts-ignore
+const bpeRanks_json_1 = __importDefault(require("../data/bpeRanks.json"));
+// eslint-disable-next-line @typescript-eslint/ban-ts-comment, @typescript-eslint/prefer-ts-expect-error
+// @ts-ignore
+const encoder_json_1 = __importDefault(require("../data/encoder.json"));
+const utils_1 = require("./utils");
+const encoder = encoder_json_1.default;
+const bpeRanks = bpeRanks_json_1.default;
+const decoder = (0, utils_1.dictZip)(Object.values(encoder), Object.keys(encoder));
+const textEncoder = new TextEncoder();
+const textDecoder = new TextDecoder();
+// eslint-disable-next-line no-magic-numbers
+const UPPER_BYTE_VALUE = 2 ** 8;
+function createByteToUnicodeMap() {
+    const asciiRange = (0, utils_1.range)((0, utils_1.getCharCode)('!'), (0, utils_1.getCharCode)('~') + 1);
+    const latin1Range1 = (0, utils_1.range)((0, utils_1.getCharCode)('¡'), (0, utils_1.getCharCode)('¬') + 1);
+    const latin1Range2 = (0, utils_1.range)((0, utils_1.getCharCode)('®'), (0, utils_1.getCharCode)('ÿ') + 1);
+    const initialCodePoints = [...asciiRange, ...latin1Range1, ...latin1Range2];
+    const mappedCodePoints = [...initialCodePoints];
+    let newCodePointOffset = 0;
+    for (let byteValue = 0; byteValue < UPPER_BYTE_VALUE; byteValue++) {
+        if (!initialCodePoints.includes(byteValue)) {
+            initialCodePoints.push(byteValue);
+            mappedCodePoints.push(UPPER_BYTE_VALUE + newCodePointOffset);
+            newCodePointOffset += 1;
+        }
+    }
+    const unicodeChars = mappedCodePoints.map(utils_1.getCharFromCode);
+    return new Map(initialCodePoints.map((x, i) => [x, unicodeChars[i]]));
+}
+const pat = /'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+/gu;
+const byteEncoder = createByteToUnicodeMap();
+const byteDecoder = new Map([...byteEncoder].map(([k, v]) => [v, k]));
+function bpe(token, cache = new Map()) {
+    if (cache.has(token)) {
+        return cache.get(token);
+    }
+    let word = (0, utils_1.splitToken)(token);
+    let pairs = (0, utils_1.getPairs)(word);
+    if (pairs.length === 0) {
+        return token;
+    }
+    // eslint-disable-next-line no-constant-condition
+    while (true) {
+        const minPairs = new Map();
+        pairs.forEach((pair) => {
+            const rank = bpeRanks[pair.join(',')];
+            minPairs.set(
+            // eslint-disable-next-line no-magic-numbers
+            typeof rank === 'undefined' || Number.isNaN(rank) ? 10e10 : rank, pair);
+        });
+        const keys = [...minPairs.keys()];
+        const bigram = minPairs.get(Math.min(...keys));
+        if (!bigram || !(bigram.join(',') in bpeRanks)) {
+            break;
+        }
+        const [first, second] = bigram;
+        let newWord = [];
+        let i = 0;
+        while (i < word.length) {
+            const j = word.indexOf(first, i);
+            if (j === -1) {
+                newWord = newWord.concat(word.slice(i));
+                break;
+            }
+            newWord = newWord.concat(word.slice(i, j));
+            i = j;
+            if (word[i] === first && i < word.length - 1 && word[i + 1] === second) {
+                newWord.push(first + second);
+                i += 2;
+            }
+            else {
+                newWord.push(word[i]);
+                i += 1;
+            }
+        }
+        word = newWord;
+        if (word.length === 1) {
+            break;
+        }
+        else {
+            pairs = (0, utils_1.getPairs)(word);
+        }
+    }
+    const result = word.join(' ');
+    cache.set(token, result);
+    return result;
+}
+function* encodeGenerator(text, cache = new Map()) {
+    for (let [token] of text.matchAll(pat)) {
+        token = [...textEncoder.encode(token)]
+            .map((x) => byteEncoder.get(x) ?? '')
+            .join('');
+        const newTokens = bpe(token, cache)
+            .split(' ')
+            .map((x) => encoder[x]);
+        yield newTokens;
+    }
+}
+/**
+ * @returns {false | number} false if token limit is exceeded, otherwise the number of tokens
+ */
+function isWithinTokenLimit(text, tokenLimit, cache = new Map()) {
+    const tokenGenerator = encodeGenerator(text, cache);
+    let count = 0;
+    for (const tokens of tokenGenerator) {
+        count += tokens.length;
+        if (count > tokenLimit) {
+            return false;
+        }
+    }
+    return count;
+}
+exports.isWithinTokenLimit = isWithinTokenLimit;
+function encode(text, cache = new Map()) {
+    return [...encodeGenerator(text, cache)].flat(1);
+}
+exports.encode = encode;
+function decodeToken(token) {
+    const decodedToken = decoder[token];
+    if (typeof decodedToken === 'undefined') {
+        return '';
+    }
+    const decodedBytes = (0, utils_1.splitToken)(decodedToken).map((x) => byteDecoder.get(x));
+    return textDecoder.decode(new Uint8Array(decodedBytes), {
+        stream: true,
+    });
+}
+exports.decodeToken = decodeToken;
+const HIGH_SURROGATE_START = 55296;
+const HIGH_SURROGATE_END = 56319;
+function endsWithIncompleteUtfPairSurrogate(string) {
+    if (string.length === 0)
+        return false;
+    // Check if the last character is a high surrogate
+    const lastCharCode = string.charCodeAt(string.length - 1);
+    return (lastCharCode >= HIGH_SURROGATE_START && lastCharCode <= HIGH_SURROGATE_END);
+}
+exports.endsWithIncompleteUtfPairSurrogate = endsWithIncompleteUtfPairSurrogate;
+function* decodeGenerator(tokens) {
+    let buffer = '';
+    for (const token of tokens) {
+        buffer += decodeToken(token);
+        if (buffer.length === 0 || endsWithIncompleteUtfPairSurrogate(buffer)) {
+            // Keep the high surrogate in the buffer and continue with the next token
+            // eslint-disable-next-line no-continue
+            continue;
+        }
+        else {
+            yield buffer;
+            // reset buffer
+            buffer = '';
+        }
+    }
+    // Yield any remaining characters in the buffer
+    if (buffer.length > 0) {
+        yield buffer;
+    }
+}
+exports.decodeGenerator = decodeGenerator;
+/**
+ * Decode tokens asynchronously and yield the decoded strings, one by one.
+ * Will not yield for tokens that include a high surrogate, but wait for the next token.
+ * @param {AsyncIterable<number>} tokens
+ * @returns {AsyncGenerator<string, void, undefined>}
+ */
+async function* decodeAsyncGenerator(tokens) {
+    let buffer = '';
+    for await (const token of tokens) {
+        buffer += decodeToken(token);
+        if (buffer.length === 0 || endsWithIncompleteUtfPairSurrogate(buffer)) {
+            // Keep the high surrogate in the buffer and continue with the next token
+            // eslint-disable-next-line no-continue
+            continue;
+        }
+        else {
+            yield buffer;
+            // reset buffer
+            buffer = '';
+        }
+    }
+    // Yield any remaining characters in the buffer
+    if (buffer.length > 0) {
+        yield buffer;
+    }
+}
+exports.decodeAsyncGenerator = decodeAsyncGenerator;
+function decode(tokens) {
+    return [...decodeGenerator(tokens)].join('');
+}
+exports.decode = decode;
+//# sourceMappingURL=encoder.js.map

package/cjs/encoder.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"encoder.js","sourceRoot":"","sources":["../src/encoder.ts"],"names":[],"mappings":";AAAA,0CAA0C;AAC1C,8CAA8C;;;;;;AAE9C,wGAAwG;AACxG,aAAa;AACb,0EAAgD;AAChD,wGAAwG;AACxG,aAAa;AACb,wEAA8C;AAC9C,mCAOgB;AAEhB,MAAM,OAAO,GAA2B,sBAAW,CAAA;AACnD,MAAM,QAAQ,GAA2B,uBAAY,CAAA;AAErD,MAAM,OAAO,GAA2B,IAAA,eAAO,EAC7C,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,EACtB,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CACrB,CAAA;AAED,MAAM,WAAW,GAAG,IAAI,WAAW,EAAE,CAAA;AACrC,MAAM,WAAW,GAAG,IAAI,WAAW,EAAE,CAAA;AAErC,4CAA4C;AAC5C,MAAM,gBAAgB,GAAG,CAAC,IAAI,CAAC,CAAA;AAE/B,SAAS,sBAAsB;IAC7B,MAAM,UAAU,GAAG,IAAA,aAAK,EAAC,IAAA,mBAAW,EAAC,GAAG,CAAC,EAAE,IAAA,mBAAW,EAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAA;IAChE,MAAM,YAAY,GAAG,IAAA,aAAK,EAAC,IAAA,mBAAW,EAAC,GAAG,CAAC,EAAE,IAAA,mBAAW,EAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAA;IAClE,MAAM,YAAY,GAAG,IAAA,aAAK,EAAC,IAAA,mBAAW,EAAC,GAAG,CAAC,EAAE,IAAA,mBAAW,EAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAA;IAElE,MAAM,iBAAiB,GAAG,CAAC,GAAG,UAAU,EAAE,GAAG,YAAY,EAAE,GAAG,YAAY,CAAC,CAAA;IAC3E,MAAM,gBAAgB,GAAG,CAAC,GAAG,iBAAiB,CAAC,CAAA;IAE/C,IAAI,kBAAkB,GAAG,CAAC,CAAA;IAC1B,KAAK,IAAI,SAAS,GAAG,CAAC,EAAE,SAAS,GAAG,gBAAgB,EAAE,SAAS,EAAE,EAAE;QACjE,IAAI,CAAC,iBAAiB,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE;YAC1C,iBAAiB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;YACjC,gBAAgB,CAAC,IAAI,CAAC,gBAAgB,GAAG,kBAAkB,CAAC,CAAA;YAC5D,kBAAkB,IAAI,CAAC,CAAA;SACxB;KACF;IAED,MAAM,YAAY,GAAG,gBAAgB,CAAC,GAAG,CAAC,uBAAe,CAAC,CAAA;IAC1D,OAAO,IAAI,GAAG,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC,CAAE,CAAC,CAAC,CAAC,CAAA;AACxE,CAAC;AAED,MAAM,GAAG,GACP,8EAA8E,CAAA;AAEhF,MAAM,WAAW,GAAG,sBAAsB,EAAE,CAAA;AAC5C,MAAM,WAAW,GAAwB,IAAI,GAAG,CAC9C,CAAC,GAAG,WAAW,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CACzC,CAAA;AAED,SAAS,GAAG,CAAC,KAAa,EAAE,QAA6B,IAAI,GAAG,EAAE;IAChE,IAAI,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE;QACpB,OAAO,KAAK,CAAC,GAAG,CAAC,KAAK,CAAE,CAAA;KACzB;IAED,IAAI,IAAI,GAAG,IAAA,kBAAU,EAAC,KAAK,CAAC,CAAA;IAC5B,IAAI,KAAK,GAAG,IAAA,gBAAQ,EAAC,IAAI,CAAC,CAAA;IAE1B,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE;QACtB,OAAO,KAAK,CAAA;KACb;IAED,iDAAiD;IACjD,OAAO,IAAI,EAAE;QACX,MAAM,QAAQ,GAAkC,IAAI,GAAG,EAAE,CAAA;QACzD,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;YACrB,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAA;YACrC,QAAQ,CAAC,GAAG;YACV,4CAA4C;YAC5C,OAAO,IAAI,KAAK,WAAW,IAAI,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAChE,IAAI,CACL,CAAA;QACH,CAAC,CAAC,CAAA;QACF,MAAM,IAAI,GAAG,CAAC,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAA;QAEjC,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC,CAAA;QAE9C,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,QAAQ,CAAC,EAAE;YAC9C,MAAK;SACN;QAED,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,GAAG,MAAM,CAAA;QAC9B,IAAI,OAAO,GAAa,EAAE,CAAA;QAC1B,IAAI,CAAC,GAAG,CAAC,CAAA;QAET,OAAO,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE;YACtB,MAAM,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,CAAA;YAChC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE;gBACZ,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAA;gBACvC,MAAK;aACN;YACD,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;YAC1C,CAAC,GAAG,CAAC,CAAA;YAEL,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,MAAM,EAAE;gBACtE,OAAO,CAAC,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC,CAAA;gBAC5B,CAAC,IAAI,CAAC,CAAA;aACP;iBAAM;gBACL,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAE,CAAC,CAAA;gBACtB,CAAC,IAAI,CAAC,CAAA;aACP;SACF;QAED,IAAI,GAAG,OAAO,CAAA;QACd,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE;YACrB,MAAK;SACN;aAAM;YACL,KAAK,GAAG,IAAA,gBAAQ,EAAC,IAAI,CAAC,CAAA;SACvB;KACF;IAED,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;IAC7B,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,CAAA;IAExB,OAAO,MAAM,CAAA;AACf,CAAC;AAED,QAAQ,CAAC,CAAC,eAAe,CACvB,IAAY,EACZ,QAA6B,IAAI,GAAG,EAAE;IAEtC,KAAK,IAAI,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE;QACtC,KAAK,GAAG,CAAC,GAAG,WAAW,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;aACnC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;aACpC,IAAI,CAAC,EAAE,CAAC,CAAA;QAEX,MAAM,SAAS,GAAG,GAAG,CAAC,KAAK,EAAE,KAAK,CAAC;aAChC,KAAK,CAAC,GAAG,CAAC;aACV,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,CAAE,CAAC,CAAA;QAE1B,MAAM,SAAS,CAAA;KAChB;AACH,CAAC;AAED;;GAEG;AACH,SAAgB,kBAAkB,CAChC,IAAY,EACZ,UAAkB,EAClB,QAA6B,IAAI,GAAG,EAAE;IAEtC,MAAM,cAAc,GAAG,eAAe,CAAC,IAAI,EAAE,KAAK,CAAC,CAAA;IACnD,IAAI,KAAK,GAAG,CAAC,CAAA;IACb,KAAK,MAAM,MAAM,IAAI,cAAc,EAAE;QACnC,KAAK,IAAI,MAAM,CAAC,MAAM,CAAA;QACtB,IAAI,KAAK,GAAG,UAAU,EAAE;YACtB,OAAO,KAAK,CAAA;SACb;KACF;IACD,OAAO,KAAK,CAAA;AACd,CAAC;AAdD,gDAcC;AAED,SAAgB,MAAM,CACpB,IAAY,EACZ,QAA6B,IAAI,GAAG,EAAE;IAEtC,OAAO,CAAC,GAAG,eAAe,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;AAClD,CAAC;AALD,wBAKC;AAED,SAAgB,WAAW,CAAC,KAAa;IACvC,MAAM,YAAY,GAAG,OAAO,CAAC,KAAK,CAAC,CAAA;IACnC,IAAI,OAAO,YAAY,KAAK,WAAW,EAAE;QACvC,OAAO,EAAE,CAAA;KACV;IACD,MAAM,YAAY,GAAG,IAAA,kBAAU,EAAC,YAAY,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC,CAAA;IAC7E,OAAO,WAAW,CAAC,MAAM,CAAC,IAAI,UAAU,CAAC,YAAY,CAAC,EAAE;QACtD,MAAM,EAAE,IAAI;KACb,CAAC,CAAA;AACJ,CAAC;AATD,kCASC;AAED,MAAM,oBAAoB,GAAG,KAAM,CAAA;AACnC,MAAM,kBAAkB,GAAG,KAAM,CAAA;AACjC,SAAgB,kCAAkC,CAAC,MAAc;IAC/D,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,KAAK,CAAA;IACrC,kDAAkD;IAClD,MAAM,YAAY,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;IACzD,OAAO,CACL,YAAY,IAAI,oBAAoB,IAAI,YAAY,IAAI,kBAAkB,CAC3E,CAAA;AACH,CAAC;AAPD,gFAOC;AAED,QAAe,CAAC,CAAC,eAAe,CAC9B,MAAwB;IAExB,IAAI,MAAM,GAAG,EAAE,CAAA;IAEf,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE;QAC1B,MAAM,IAAI,WAAW,CAAC,KAAK,CAAC,CAAA;QAE5B,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,kCAAkC,CAAC,MAAM,CAAC,EAAE;YACrE,yEAAyE;YACzE,uCAAuC;YACvC,SAAQ;SACT;aAAM;YACL,MAAM,MAAM,CAAA;YACZ,eAAe;YACf,MAAM,GAAG,EAAE,CAAA;SACZ;KACF;IAED,+CAA+C;IAC/C,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE;QACrB,MAAM,MAAM,CAAA;KACb;AACH,CAAC;AAvBD,0CAuBC;AAED;;;;;GAKG;AACI,KAAK,SAAS,CAAC,CAAC,oBAAoB,CACzC,MAA6B;IAE7B,IAAI,MAAM,GAAG,EAAE,CAAA;IAEf,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,MAAM,EAAE;QAChC,MAAM,IAAI,WAAW,CAAC,KAAK,CAAC,CAAA;QAE5B,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,kCAAkC,CAAC,MAAM,CAAC,EAAE;YACrE,yEAAyE;YACzE,uCAAuC;YACvC,SAAQ;SACT;aAAM;YACL,MAAM,MAAM,CAAA;YACZ,eAAe;YACf,MAAM,GAAG,EAAE,CAAA;SACZ;KACF;IAED,+CAA+C;IAC/C,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE;QACrB,MAAM,MAAM,CAAA;KACb;AACH,CAAC;AAvBD,oDAuBC;AAED,SAAgB,MAAM,CAAC,MAAgB;IACrC,OAAO,CAAC,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;AAC9C,CAAC;AAFD,wBAEC"}

package/cjs/encoder.test.d.ts

@@ -0,0 +1 @@
+export {};

package/cjs/encoder.test.js

@@ -0,0 +1,86 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+/* eslint-disable compat/compat */
+const encoder_1 = require("./encoder");
+describe('basic functionality', () => {
+    it('empty string', () => {
+        const str = '';
+        expect((0, encoder_1.encode)(str)).toEqual([]);
+        expect((0, encoder_1.decode)((0, encoder_1.encode)(str))).toEqual(str);
+        expect((0, encoder_1.isWithinTokenLimit)(str, 0)).toBe(0);
+        expect((0, encoder_1.isWithinTokenLimit)(str, 3)).toBe(0);
+    });
+    it('space', () => {
+        const str = ' ';
+        expect((0, encoder_1.encode)(str)).toEqual([220]);
+        expect((0, encoder_1.decode)((0, encoder_1.encode)(str))).toEqual(str);
+        expect((0, encoder_1.isWithinTokenLimit)(str, 3)).toBe(1);
+        expect((0, encoder_1.isWithinTokenLimit)(str, 0)).toBe(false);
+    });
+    it('tab', () => {
+        const str = '\t';
+        expect((0, encoder_1.encode)(str)).toEqual([197]);
+        expect((0, encoder_1.decode)((0, encoder_1.encode)(str))).toEqual(str);
+    });
+    it('simple text', () => {
+        const str = 'This is some text';
+        expect((0, encoder_1.encode)(str)).toEqual([1212, 318, 617, 2420]);
+        expect((0, encoder_1.decode)((0, encoder_1.encode)(str))).toEqual(str);
+        expect((0, encoder_1.isWithinTokenLimit)(str, 3)).toBe(false);
+        expect((0, encoder_1.isWithinTokenLimit)(str, 5)).toBe(4);
+    });
+    it('multi-token word', () => {
+        const str = 'indivisible';
+        expect((0, encoder_1.encode)(str)).toEqual([521, 452, 12843]);
+        expect((0, encoder_1.decode)((0, encoder_1.encode)(str))).toEqual(str);
+        expect((0, encoder_1.isWithinTokenLimit)(str, 3)).toBe(3);
+    });
+    const helloWorldTokens = [31373, 50169, 233, 995, 12520, 234, 235];
+    it('emojis', () => {
+        const str = 'hello 👋 world 🌍';
+        expect((0, encoder_1.encode)(str)).toEqual(helloWorldTokens);
+        expect((0, encoder_1.decode)((0, encoder_1.encode)(str))).toEqual(str);
+        expect((0, encoder_1.isWithinTokenLimit)(str, 4)).toBe(false);
+        expect((0, encoder_1.isWithinTokenLimit)(str, 400)).toBe(7);
+    });
+    it('decode token-by-token via generator', () => {
+        const generator = (0, encoder_1.decodeGenerator)(helloWorldTokens);
+        expect(generator.next().value).toBe('hello');
+        expect(generator.next().value).toBe(' ');
+        expect(generator.next().value).toBe('👋');
+        expect(generator.next().value).toBe(' world');
+        expect(generator.next().value).toBe(' ');
+        expect(generator.next().value).toBe('🌍');
+    });
+    async function* getHelloWorldTokensAsync() {
+        for (const token of helloWorldTokens) {
+            // eslint-disable-next-line no-await-in-loop
+            yield await Promise.resolve(token);
+        }
+    }
+    it('decode token-by-token via async generator', async () => {
+        const generator = (0, encoder_1.decodeAsyncGenerator)(getHelloWorldTokensAsync());
+        const decoded = ['hello', ' ', '👋', ' world', ' ', '🌍'];
+        for await (const value of generator) {
+            expect(value).toEqual(decoded.shift());
+        }
+    });
+    it('properties of Object', () => {
+        const str = 'toString constructor hasOwnProperty valueOf';
+        expect((0, encoder_1.encode)(str)).toEqual([
+            1462, 10100, 23772, 468, 23858, 21746, 1988, 5189,
+        ]);
+        expect((0, encoder_1.decode)((0, encoder_1.encode)(str))).toEqual(str);
+    });
+    it('text with commas', () => {
+        const str = 'hello, I am a text, and I have commas. a,b,c';
+        expect((0, encoder_1.decode)((0, encoder_1.encode)(str))).toEqual(str);
+        expect((0, encoder_1.encode)(str)).toStrictEqual([
+            31373, 11, 314, 716, 257, 2420, 11, 290, 314, 423, 725, 292, 13, 257,
+            11, 65, 11, 66,
+        ]);
+        expect((0, encoder_1.isWithinTokenLimit)(str, 15)).toBe(false);
+        expect((0, encoder_1.isWithinTokenLimit)(str, 300)).toBe(18);
+    });
+});
+//# sourceMappingURL=encoder.test.js.map

package/cjs/encoder.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"encoder.test.js","sourceRoot":"","sources":["../src/encoder.test.ts"],"names":[],"mappings":";;AAAA,kCAAkC;AAClC,uCAMkB;AAElB,QAAQ,CAAC,qBAAqB,EAAE,GAAG,EAAE;IACnC,EAAE,CAAC,cAAc,EAAE,GAAG,EAAE;QACtB,MAAM,GAAG,GAAG,EAAE,CAAA;QACd,MAAM,CAAC,IAAA,gBAAM,EAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;QAC/B,MAAM,CAAC,IAAA,gBAAM,EAAC,IAAA,gBAAM,EAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;QACxC,MAAM,CAAC,IAAA,4BAAkB,EAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QAC1C,MAAM,CAAC,IAAA,4BAAkB,EAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IAC5C,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;QACf,MAAM,GAAG,GAAG,GAAG,CAAA;QACf,MAAM,CAAC,IAAA,gBAAM,EAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;QAClC,MAAM,CAAC,IAAA,gBAAM,EAAC,IAAA,gBAAM,EAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;QACxC,MAAM,CAAC,IAAA,4BAAkB,EAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QAC1C,MAAM,CAAC,IAAA,4BAAkB,EAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;IAChD,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;QACb,MAAM,GAAG,GAAG,IAAI,CAAA;QAChB,MAAM,CAAC,IAAA,gBAAM,EAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;QAClC,MAAM,CAAC,IAAA,gBAAM,EAAC,IAAA,gBAAM,EAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;IAC1C,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,aAAa,EAAE,GAAG,EAAE;QACrB,MAAM,GAAG,GAAG,mBAAmB,CAAA;QAC/B,MAAM,CAAC,IAAA,gBAAM,EAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAK,EAAE,GAAG,EAAE,GAAG,EAAE,IAAK,CAAC,CAAC,CAAA;QACrD,MAAM,CAAC,IAAA,gBAAM,EAAC,IAAA,gBAAM,EAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;QACxC,MAAM,CAAC,IAAA,4BAAkB,EAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QAC9C,MAAM,CAAC,IAAA,4BAAkB,EAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IAC5C,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,kBAAkB,EAAE,GAAG,EAAE;QAC1B,MAAM,GAAG,GAAG,aAAa,CAAA;QACzB,MAAM,CAAC,IAAA,gBAAM,EAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,KAAM,CAAC,CAAC,CAAA;QAC/C,MAAM,CAAC,IAAA,gBAAM,EAAC,IAAA,gBAAM,EAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;QACxC,MAAM,CAAC,IAAA,4BAAkB,EAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IAC5C,CAAC,CAAC,CAAA;IAEF,MAAM,gBAAgB,GAAG,CAAC,KAAM,EAAE,KAAM,EAAE,GAAG,EAAE,GAAG,EAAE,KAAM,EAAE,GAAG,EAAE,GAAG,CAAC,CAAA;IAErE,EAAE,CAAC,QAAQ,EAAE,GAAG,EAAE;QAChB,MAAM,GAAG,GAAG,mBAAmB,CAAA;QAC/B,MAAM,CAAC,IAAA,gBAAM,EAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAA;QAC7C,MAAM,CAAC,IAAA,gBAAM,EAAC,IAAA,gBAAM,EAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;QACxC,MAAM,CAAC,IAAA,4BAAkB,EAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QAC9C,MAAM,CAAC,IAAA,4BAAkB,EAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IAC9C,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,qCAAqC,EAAE,GAAG,EAAE;QAC7C,MAAM,SAAS,GAAG,IAAA,yBAAe,EAAC,gBAAgB,CAAC,CAAA;QACnD,MAAM,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;QAC5C,MAAM,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QACxC,MAAM,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACzC,MAAM,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;QAC7C,MAAM,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QACxC,MAAM,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IAC3C,CAAC,CAAC,CAAA;IAEF,KAAK,SAAS,CAAC,CAAC,wBAAwB;QACtC,KAAK,MAAM,KAAK,IAAI,gBAAgB,EAAE;YACpC,4CAA4C;YAC5C,MAAM,MAAM,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;SACnC;IACH,CAAC;IAED,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;QACzD,MAAM,SAAS,GAAG,IAAA,8BAAoB,EAAC,wBAAwB,EAAE,CAAC,CAAA;QAClE,MAAM,OAAO,GAAG,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,EAAE,IAAI,CAAC,CAAA;QACzD,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,SAAS,EAAE;YACnC,MAAM,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,CAAA;SACvC;IACH,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,sBAAsB,EAAE,GAAG,EAAE;QAC9B,MAAM,GAAG,GAAG,6CAA6C,CAAA;QAEzD,MAAM,CAAC,IAAA,gBAAM,EAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC;YAC1B,IAAK,EAAE,KAAM,EAAE,KAAM,EAAE,GAAG,EAAE,KAAM,EAAE,KAAM,EAAE,IAAK,EAAE,IAAK;SACzD,CAAC,CAAA;QACF,MAAM,CAAC,IAAA,gBAAM,EAAC,IAAA,gBAAM,EAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;IAC1C,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,kBAAkB,EAAE,GAAG,EAAE;QAC1B,MAAM,GAAG,GAAG,8CAA8C,CAAA;QAC1D,MAAM,CAAC,IAAA,gBAAM,EAAC,IAAA,gBAAM,EAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;QACxC,MAAM,CAAC,IAAA,gBAAM,EAAC,GAAG,CAAC,CAAC,CAAC,aAAa,CAAC;YAChC,KAAM,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,IAAK,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG;YACtE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE;SACf,CAAC,CAAA;QACF,MAAM,CAAC,IAAA,4BAAkB,EAAC,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QAC/C,MAAM,CAAC,IAAA,4BAAkB,EAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IAC/C,CAAC,CAAC,CAAA;AACJ,CAAC,CAAC,CAAA"}

package/cjs/getBpe.d.ts

@@ -0,0 +1 @@
+export {};

package/cjs/getBpe.js

@@ -0,0 +1,17 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const fs_1 = __importDefault(require("fs"));
+const path_1 = __importDefault(require("path"));
+const utils_1 = require("./utils");
+const bpeFile = fs_1.default.readFileSync(path_1.default.join(__dirname, '../data/vocab.bpe'), 'utf8');
+const lines = bpeFile.split('\n');
+const bpeMerges = lines.slice(1, -1).map((x) => x
+    .split(/(\s+)/)
+    .filter((e) => e.trim().length > 0)
+    .join(','));
+const bpeRanks = (0, utils_1.dictZip)(bpeMerges, (0, utils_1.range)(0, bpeMerges.length));
+fs_1.default.writeFileSync(path_1.default.join(__dirname, '../data/bpeRanks.json'), JSON.stringify(bpeRanks));
+//# sourceMappingURL=getBpe.js.map

package/cjs/getBpe.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"getBpe.js","sourceRoot":"","sources":["../src/getBpe.ts"],"names":[],"mappings":";;;;;AAAA,4CAAmB;AACnB,gDAAuB;AACvB,mCAAwC;AAExC,MAAM,OAAO,GAAG,YAAE,CAAC,YAAY,CAC7B,cAAI,CAAC,IAAI,CAAC,SAAS,EAAE,mBAAmB,CAAC,EACzC,MAAM,CACP,CAAA;AACD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;AAEjC,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAC7C,CAAC;KACE,KAAK,CAAC,OAAO,CAAC;KACd,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC;KAClC,IAAI,CAAC,GAAG,CAAC,CACb,CAAA;AAED,MAAM,QAAQ,GAAG,IAAA,eAAO,EAAC,SAAS,EAAE,IAAA,aAAK,EAAC,CAAC,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC,CAAA;AAE/D,YAAE,CAAC,aAAa,CACd,cAAI,CAAC,IAAI,CAAC,SAAS,EAAE,uBAAuB,CAAC,EAC7C,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CACzB,CAAA"}

package/cjs/main.d.ts

@@ -0,0 +1 @@
+export * from './encoder';

package/cjs/main.js

@@ -0,0 +1,18 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+__exportStar(require("./encoder"), exports);
+//# sourceMappingURL=main.js.map

package/cjs/main.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"main.js","sourceRoot":"","sources":["../src/main.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,4CAAyB"}

package/cjs/utils.d.ts

@@ -0,0 +1,6 @@
+export declare function dictZip<T extends PropertyKey, U>(x: T[], y: U[]): Record<T, U>;
+export declare function range(x: number, y: number): number[];
+export declare const getCharCode: (x: string) => number;
+export declare const getCharFromCode: (x: number) => string;
+export declare function getPairs(word: string[]): [string, string][];
+export declare const splitToken: (token: string) => string[];

package/cjs/utils.js

@@ -0,0 +1,37 @@
+"use strict";
+/* eslint-disable unicorn/prefer-spread */
+/* eslint-disable unicorn/prefer-code-point */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.splitToken = exports.getPairs = exports.getCharFromCode = exports.getCharCode = exports.range = exports.dictZip = void 0;
+function dictZip(x, y) {
+    // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
+    const result = {};
+    x.forEach((k, i) => {
+        result[k] = y[i];
+    });
+    return result;
+}
+exports.dictZip = dictZip;
+function range(x, y) {
+    return Array.from({ length: y - x }, (_, i) => x + i);
+}
+exports.range = range;
+const getCharCode = (x) => x.charCodeAt(0);
+exports.getCharCode = getCharCode;
+const getCharFromCode = (x) => String.fromCharCode(x);
+exports.getCharFromCode = getCharFromCode;
+function getPairs(word) {
+    /** @type {[string, string][]} */
+    const pairs = [];
+    let prevChar = word[0];
+    for (let i = 1; i < word.length; i++) {
+        const char = word[i];
+        pairs.push([prevChar, char]);
+        prevChar = char;
+    }
+    return pairs;
+}
+exports.getPairs = getPairs;
+const splitToken = (token) => token.split('');
+exports.splitToken = splitToken;
+//# sourceMappingURL=utils.js.map

package/cjs/utils.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"utils.js","sourceRoot":"","sources":["../src/utils.ts"],"names":[],"mappings":";AAAA,0CAA0C;AAC1C,8CAA8C;;;AAE9C,SAAgB,OAAO,CACrB,CAAM,EACN,CAAM;IAEN,yEAAyE;IACzE,MAAM,MAAM,GAAG,EAAkB,CAAA;IACjC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACjB,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAE,CAAA;IACnB,CAAC,CAAC,CAAA;IACF,OAAO,MAAM,CAAA;AACf,CAAC;AAVD,0BAUC;AAED,SAAgB,KAAK,CAAC,CAAS,EAAE,CAAS;IACxC,OAAO,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;AACvD,CAAC;AAFD,sBAEC;AAEM,MAAM,WAAW,GAAG,CAAC,CAAS,EAAU,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAA;AAApD,QAAA,WAAW,eAAyC;AAE1D,MAAM,eAAe,GAAG,CAAC,CAAS,EAAU,EAAE,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAA;AAA/D,QAAA,eAAe,mBAAgD;AAE5E,SAAgB,QAAQ,CAAC,IAAc;IACrC,iCAAiC;IACjC,MAAM,KAAK,GAAuB,EAAE,CAAA;IACpC,IAAI,QAAQ,GAAG,IAAI,CAAC,CAAC,CAAC,CAAA;IACtB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;QACpC,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,CAAA;QACpB,KAAK,CAAC,IAAI,CAAC,CAAC,QAAS,EAAE,IAAK,CAAC,CAAC,CAAA;QAC9B,QAAQ,GAAG,IAAI,CAAA;KAChB;IACD,OAAO,KAAK,CAAA;AACd,CAAC;AAVD,4BAUC;AAEM,MAAM,UAAU,GAAG,CAAC,KAAa,EAAY,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;AAAzD,QAAA,UAAU,cAA+C"}