gpt-tokenizer 1.0.2 → 1.0.4

package/src/getData.ts

@@ -0,0 +1,34 @@
+import fs from 'fs'
+import path from 'path'
+import { dictZip, range } from './utils'
+
+const bpeFile = fs.readFileSync(
+  path.join(__dirname, '../data/vocab.bpe'),
+  'utf8',
+)
+const lines = bpeFile.split('\n')
+
+const bpeMerges = lines.slice(1, -1).map((x) =>
+  x
+    .split(/(\s+)/)
+    .filter((e) => e.trim().length > 0)
+    .join(','),
+)
+
+const bpeRanks = dictZip(bpeMerges, range(0, bpeMerges.length))
+// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
+const encoder: object = require('../data/encoder.json')
+
+fs.mkdirSync(path.join(__dirname, 'data'), { recursive: true })
+fs.writeFileSync(
+  path.join(__dirname, 'data/bpeRanks.js'),
+  `/* eslint-disable */\n// @ts-nocheck\n// prettier-ignore\nexport default ${JSON.stringify(
+    bpeRanks,
+  )}`,
+)
+fs.writeFileSync(
+  path.join(__dirname, 'data/encoder.js'),
+  `/* eslint-disable */\n// @ts-nocheck\n// prettier-ignore\nexport default ${JSON.stringify(
+    encoder,
+  )}`,
+)

package/src/{encoder.test.ts → main.test.ts}

@@ -5,7 +5,7 @@ import {
   decodeGenerator,
   encode,
   isWithinTokenLimit,
-} from './encoder'
+} from './main'
 
 describe('basic functionality', () => {
   it('empty string', () => {

package/src/main.ts

@@ -1 +1,266 @@
-export * from './encoder'
+/* eslint-disable import/extensions */
+/* eslint-disable unicorn/prefer-spread */
+/* eslint-disable unicorn/prefer-code-point */
+
+import bpeRanksData from './data/bpeRanks.js'
+import encoderData from './data/encoder.js'
+import {
+  dictZip,
+  getCharCode,
+  getCharFromCode,
+  getPairs,
+  range,
+  splitToken,
+} from './utils.js'
+
+const encoder: Record<string, number> = encoderData
+const bpeRanks: Record<string, number> = bpeRanksData
+
+const decoder: Record<number, string> = dictZip(
+  Object.values(encoder),
+  Object.keys(encoder),
+)
+
+const textEncoder = new TextEncoder()
+const textDecoder = new TextDecoder()
+
+// eslint-disable-next-line no-magic-numbers
+const UPPER_BYTE_VALUE = 2 ** 8
+
+function createByteToUnicodeMap(): Map<number, string> {
+  const asciiRange = range(getCharCode('!'), getCharCode('~') + 1)
+  const latin1Range1 = range(getCharCode('¡'), getCharCode('¬') + 1)
+  const latin1Range2 = range(getCharCode('®'), getCharCode('ÿ') + 1)
+
+  const initialCodePoints = [...asciiRange, ...latin1Range1, ...latin1Range2]
+  const mappedCodePoints = [...initialCodePoints]
+
+  let newCodePointOffset = 0
+  for (let byteValue = 0; byteValue < UPPER_BYTE_VALUE; byteValue++) {
+    if (!initialCodePoints.includes(byteValue)) {
+      initialCodePoints.push(byteValue)
+      mappedCodePoints.push(UPPER_BYTE_VALUE + newCodePointOffset)
+      newCodePointOffset += 1
+    }
+  }
+
+  const unicodeChars = mappedCodePoints.map(getCharFromCode)
+  return new Map(initialCodePoints.map((x, i) => [x, unicodeChars[i]!]))
+}
+
+const pat =
+  /'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+/gu
+
+const byteEncoder = createByteToUnicodeMap()
+const byteDecoder: Map<string, number> = new Map(
+  [...byteEncoder].map(([k, v]) => [v, k]),
+)
+
+export function bpe(
+  token: string,
+  cache: Map<string, string> = new Map(),
+): string {
+  if (cache.has(token)) {
+    return cache.get(token)!
+  }
+
+  let word = splitToken(token)
+  let pairs = getPairs(word)
+
+  if (pairs.length === 0) {
+    return token
+  }
+
+  // eslint-disable-next-line no-constant-condition
+  while (true) {
+    const minPairs: Map<number, [string, string]> = new Map()
+    pairs.forEach((pair) => {
+      const rank = bpeRanks[pair.join(',')]
+      minPairs.set(
+        // eslint-disable-next-line no-magic-numbers
+        typeof rank === 'undefined' || Number.isNaN(rank) ? 10e10 : rank,
+        pair,
+      )
+    })
+    const keys = [...minPairs.keys()]
+
+    const bigram = minPairs.get(Math.min(...keys))
+
+    if (!bigram || !(bigram.join(',') in bpeRanks)) {
+      break
+    }
+
+    const [first, second] = bigram
+    let newWord: string[] = []
+    let i = 0
+
+    while (i < word.length) {
+      const j = word.indexOf(first, i)
+      if (j === -1) {
+        newWord = newWord.concat(word.slice(i))
+        break
+      }
+      newWord = newWord.concat(word.slice(i, j))
+      i = j
+
+      if (word[i] === first && i < word.length - 1 && word[i + 1] === second) {
+        newWord.push(first + second)
+        i += 2
+      } else {
+        newWord.push(word[i]!)
+        i += 1
+      }
+    }
+
+    word = newWord
+    if (word.length === 1) {
+      break
+    } else {
+      pairs = getPairs(word)
+    }
+  }
+
+  const result = word.join(' ')
+  cache.set(token, result)
+
+  return result
+}
+
+export function* encodeGenerator(
+  text: string,
+  cache: Map<string, string> = new Map(),
+): Generator<number[], void, undefined> {
+  for (let [token] of text.matchAll(pat)) {
+    token = [...textEncoder.encode(token)]
+      .map((x) => byteEncoder.get(x) ?? '')
+      .join('')
+
+    const newTokens = bpe(token, cache)
+      .split(' ')
+      .map((x) => encoder[x]!)
+
+    yield newTokens
+  }
+}
+
+/**
+ * @returns {false | number} false if token limit is exceeded, otherwise the number of tokens
+ */
+export function isWithinTokenLimit(
+  text: string,
+  tokenLimit: number,
+  cache: Map<string, string> = new Map(),
+): false | number {
+  const tokenGenerator = encodeGenerator(text, cache)
+  let count = 0
+  for (const tokens of tokenGenerator) {
+    count += tokens.length
+    if (count > tokenLimit) {
+      return false
+    }
+  }
+  return count
+}
+
+export function encode(
+  text: string,
+  cache: Map<string, string> = new Map(),
+): number[] {
+  return [...encodeGenerator(text, cache)].flat(1)
+}
+
+export function decodeToken(token: number): string {
+  const decodedToken = decoder[token]
+  if (typeof decodedToken === 'undefined') {
+    return ''
+  }
+  const decodedBytes = splitToken(decodedToken).map((x) => byteDecoder.get(x)!)
+  return textDecoder.decode(new Uint8Array(decodedBytes), {
+    stream: true,
+  })
+}
+
+const HIGH_SURROGATE_START = 55_296
+const HIGH_SURROGATE_END = 56_319
+export function endsWithIncompleteUtfPairSurrogate(string: string): boolean {
+  if (string.length === 0) return false
+  // Check if the last character is a high surrogate
+  const lastCharCode = string.charCodeAt(string.length - 1)
+  return (
+    lastCharCode >= HIGH_SURROGATE_START && lastCharCode <= HIGH_SURROGATE_END
+  )
+}
+
+export function* decodeGenerator(
+  tokens: Iterable<number>,
+): Generator<string, void, undefined> {
+  let buffer = ''
+
+  for (const token of tokens) {
+    buffer += decodeToken(token)
+
+    if (buffer.length === 0 || endsWithIncompleteUtfPairSurrogate(buffer)) {
+      // Keep the high surrogate in the buffer and continue with the next token
+      // eslint-disable-next-line no-continue
+      continue
+    } else {
+      yield buffer
+      // reset buffer
+      buffer = ''
+    }
+  }
+
+  // Yield any remaining characters in the buffer
+  if (buffer.length > 0) {
+    yield buffer
+  }
+}
+
+/**
+ * Decode tokens asynchronously and yield the decoded strings, one by one.
+ * Will not yield for tokens that include a high surrogate, but wait for the next token.
+ * @param {AsyncIterable<number>} tokens
+ * @returns {AsyncGenerator<string, void, undefined>}
+ */
+export async function* decodeAsyncGenerator(
+  tokens: AsyncIterable<number>,
+): AsyncGenerator<string, void, undefined> {
+  let buffer = ''
+
+  for await (const token of tokens) {
+    buffer += decodeToken(token)
+
+    if (buffer.length === 0 || endsWithIncompleteUtfPairSurrogate(buffer)) {
+      // Keep the high surrogate in the buffer and continue with the next token
+      // eslint-disable-next-line no-continue
+      continue
+    } else {
+      yield buffer
+      // reset buffer
+      buffer = ''
+    }
+  }
+
+  // Yield any remaining characters in the buffer
+  if (buffer.length > 0) {
+    yield buffer
+  }
+}
+
+export function decode(tokens: number[]): string {
+  return [...decodeGenerator(tokens)].join('')
+}
+
+const gptEncoder = {
+  encode,
+  decode,
+  encodeGenerator,
+  decodeGenerator,
+  decodeAsyncGenerator,
+  isWithinTokenLimit,
+  bpe,
+}
+
+// used by the UMD export
+// eslint-disable-next-line import/no-default-export
+export default gptEncoder

package/src/utils.js

@@ -0,0 +1,61 @@
+/* eslint-disable unicorn/prefer-spread */
+/* eslint-disable unicorn/prefer-code-point */
+
+/**
+ * @template T
+ * @template U
+ * @param {T[]} x
+ * @param {U[]} y
+ * @returns {Record<T, U>}
+ */
+export function dictZip(x, y) {
+  /** @type {Record<T, U>} */
+  const result = {}
+  x.forEach((k, i) => {
+    result[k] = y[i]
+  })
+  return result
+}
+
+/**
+ * @param {number} x
+ * @param {number} y
+ * @returns {number[]}
+ */
+export function range(x, y) {
+  return Array.from({ length: y - x }, (_, i) => x + i)
+}
+
+/**
+ * @param {string} x
+ * @returns {number}
+ */
+export const getCharCode = (x) => x.charCodeAt(0)
+
+/**
+ * @param {number} x
+ * @returns {string}
+ */
+export const getCharFromCode = (x) => String.fromCharCode(x)
+
+/**
+ * @param {string[]} word
+ * @returns {[string, string][]}
+ */
+export function getPairs(word) {
+  /** @type {[string, string][]} */
+  const pairs = []
+  let prevChar = word[0]
+  for (let i = 1; i < word.length; i++) {
+    const char = word[i]
+    pairs.push([prevChar, char])
+    prevChar = char
+  }
+  return pairs
+}
+
+/**
+ * @param {string} token
+ * @returns {string[]}
+ */
+export const splitToken = (token) => token.split('')

package/cjs/encoder.d.ts

@@ -1,16 +0,0 @@
-/**
- * @returns {false | number} false if token limit is exceeded, otherwise the number of tokens
- */
-export declare function isWithinTokenLimit(text: string, tokenLimit: number, cache?: Map<string, string>): false | number;
-export declare function encode(text: string, cache?: Map<string, string>): number[];
-export declare function decodeToken(token: number): string;
-export declare function endsWithIncompleteUtfPairSurrogate(string: string): boolean;
-export declare function decodeGenerator(tokens: Iterable<number>): Generator<string, void, undefined>;
-/**
- * Decode tokens asynchronously and yield the decoded strings, one by one.
- * Will not yield for tokens that include a high surrogate, but wait for the next token.
- * @param {AsyncIterable<number>} tokens
- * @returns {AsyncGenerator<string, void, undefined>}
- */
-export declare function decodeAsyncGenerator(tokens: AsyncIterable<number>): AsyncGenerator<string, void, undefined>;
-export declare function decode(tokens: number[]): string;

package/cjs/encoder.js

@@ -1,201 +0,0 @@
-"use strict";
-/* eslint-disable unicorn/prefer-spread */
-/* eslint-disable unicorn/prefer-code-point */
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.decode = exports.decodeAsyncGenerator = exports.decodeGenerator = exports.endsWithIncompleteUtfPairSurrogate = exports.decodeToken = exports.encode = exports.isWithinTokenLimit = void 0;
-// eslint-disable-next-line @typescript-eslint/ban-ts-comment, @typescript-eslint/prefer-ts-expect-error
-// @ts-ignore
-const bpeRanks_json_1 = __importDefault(require("../data/bpeRanks.json"));
-// eslint-disable-next-line @typescript-eslint/ban-ts-comment, @typescript-eslint/prefer-ts-expect-error
-// @ts-ignore
-const encoder_json_1 = __importDefault(require("../data/encoder.json"));
-const utils_1 = require("./utils");
-const encoder = encoder_json_1.default;
-const bpeRanks = bpeRanks_json_1.default;
-const decoder = (0, utils_1.dictZip)(Object.values(encoder), Object.keys(encoder));
-const textEncoder = new TextEncoder();
-const textDecoder = new TextDecoder();
-// eslint-disable-next-line no-magic-numbers
-const UPPER_BYTE_VALUE = 2 ** 8;
-function createByteToUnicodeMap() {
-    const asciiRange = (0, utils_1.range)((0, utils_1.getCharCode)('!'), (0, utils_1.getCharCode)('~') + 1);
-    const latin1Range1 = (0, utils_1.range)((0, utils_1.getCharCode)('¡'), (0, utils_1.getCharCode)('¬') + 1);
-    const latin1Range2 = (0, utils_1.range)((0, utils_1.getCharCode)('®'), (0, utils_1.getCharCode)('ÿ') + 1);
-    const initialCodePoints = [...asciiRange, ...latin1Range1, ...latin1Range2];
-    const mappedCodePoints = [...initialCodePoints];
-    let newCodePointOffset = 0;
-    for (let byteValue = 0; byteValue < UPPER_BYTE_VALUE; byteValue++) {
-        if (!initialCodePoints.includes(byteValue)) {
-            initialCodePoints.push(byteValue);
-            mappedCodePoints.push(UPPER_BYTE_VALUE + newCodePointOffset);
-            newCodePointOffset += 1;
-        }
-    }
-    const unicodeChars = mappedCodePoints.map(utils_1.getCharFromCode);
-    return new Map(initialCodePoints.map((x, i) => [x, unicodeChars[i]]));
-}
-const pat = /'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+/gu;
-const byteEncoder = createByteToUnicodeMap();
-const byteDecoder = new Map([...byteEncoder].map(([k, v]) => [v, k]));
-function bpe(token, cache = new Map()) {
-    if (cache.has(token)) {
-        return cache.get(token);
-    }
-    let word = (0, utils_1.splitToken)(token);
-    let pairs = (0, utils_1.getPairs)(word);
-    if (pairs.length === 0) {
-        return token;
-    }
-    // eslint-disable-next-line no-constant-condition
-    while (true) {
-        const minPairs = new Map();
-        pairs.forEach((pair) => {
-            const rank = bpeRanks[pair.join(',')];
-            minPairs.set(
-            // eslint-disable-next-line no-magic-numbers
-            typeof rank === 'undefined' || Number.isNaN(rank) ? 10e10 : rank, pair);
-        });
-        const keys = [...minPairs.keys()];
-        const bigram = minPairs.get(Math.min(...keys));
-        if (!bigram || !(bigram.join(',') in bpeRanks)) {
-            break;
-        }
-        const [first, second] = bigram;
-        let newWord = [];
-        let i = 0;
-        while (i < word.length) {
-            const j = word.indexOf(first, i);
-            if (j === -1) {
-                newWord = newWord.concat(word.slice(i));
-                break;
-            }
-            newWord = newWord.concat(word.slice(i, j));
-            i = j;
-            if (word[i] === first && i < word.length - 1 && word[i + 1] === second) {
-                newWord.push(first + second);
-                i += 2;
-            }
-            else {
-                newWord.push(word[i]);
-                i += 1;
-            }
-        }
-        word = newWord;
-        if (word.length === 1) {
-            break;
-        }
-        else {
-            pairs = (0, utils_1.getPairs)(word);
-        }
-    }
-    const result = word.join(' ');
-    cache.set(token, result);
-    return result;
-}
-function* encodeGenerator(text, cache = new Map()) {
-    for (let [token] of text.matchAll(pat)) {
-        token = [...textEncoder.encode(token)]
-            .map((x) => byteEncoder.get(x) ?? '')
-            .join('');
-        const newTokens = bpe(token, cache)
-            .split(' ')
-            .map((x) => encoder[x]);
-        yield newTokens;
-    }
-}
-/**
- * @returns {false | number} false if token limit is exceeded, otherwise the number of tokens
- */
-function isWithinTokenLimit(text, tokenLimit, cache = new Map()) {
-    const tokenGenerator = encodeGenerator(text, cache);
-    let count = 0;
-    for (const tokens of tokenGenerator) {
-        count += tokens.length;
-        if (count > tokenLimit) {
-            return false;
-        }
-    }
-    return count;
-}
-exports.isWithinTokenLimit = isWithinTokenLimit;
-function encode(text, cache = new Map()) {
-    return [...encodeGenerator(text, cache)].flat(1);
-}
-exports.encode = encode;
-function decodeToken(token) {
-    const decodedToken = decoder[token];
-    if (typeof decodedToken === 'undefined') {
-        return '';
-    }
-    const decodedBytes = (0, utils_1.splitToken)(decodedToken).map((x) => byteDecoder.get(x));
-    return textDecoder.decode(new Uint8Array(decodedBytes), {
-        stream: true,
-    });
-}
-exports.decodeToken = decodeToken;
-const HIGH_SURROGATE_START = 55296;
-const HIGH_SURROGATE_END = 56319;
-function endsWithIncompleteUtfPairSurrogate(string) {
-    if (string.length === 0)
-        return false;
-    // Check if the last character is a high surrogate
-    const lastCharCode = string.charCodeAt(string.length - 1);
-    return (lastCharCode >= HIGH_SURROGATE_START && lastCharCode <= HIGH_SURROGATE_END);
-}
-exports.endsWithIncompleteUtfPairSurrogate = endsWithIncompleteUtfPairSurrogate;
-function* decodeGenerator(tokens) {
-    let buffer = '';
-    for (const token of tokens) {
-        buffer += decodeToken(token);
-        if (buffer.length === 0 || endsWithIncompleteUtfPairSurrogate(buffer)) {
-            // Keep the high surrogate in the buffer and continue with the next token
-            // eslint-disable-next-line no-continue
-            continue;
-        }
-        else {
-            yield buffer;
-            // reset buffer
-            buffer = '';
-        }
-    }
-    // Yield any remaining characters in the buffer
-    if (buffer.length > 0) {
-        yield buffer;
-    }
-}
-exports.decodeGenerator = decodeGenerator;
-/**
- * Decode tokens asynchronously and yield the decoded strings, one by one.
- * Will not yield for tokens that include a high surrogate, but wait for the next token.
- * @param {AsyncIterable<number>} tokens
- * @returns {AsyncGenerator<string, void, undefined>}
- */
-async function* decodeAsyncGenerator(tokens) {
-    let buffer = '';
-    for await (const token of tokens) {
-        buffer += decodeToken(token);
-        if (buffer.length === 0 || endsWithIncompleteUtfPairSurrogate(buffer)) {
-            // Keep the high surrogate in the buffer and continue with the next token
-            // eslint-disable-next-line no-continue
-            continue;
-        }
-        else {
-            yield buffer;
-            // reset buffer
-            buffer = '';
-        }
-    }
-    // Yield any remaining characters in the buffer
-    if (buffer.length > 0) {
-        yield buffer;
-    }
-}
-exports.decodeAsyncGenerator = decodeAsyncGenerator;
-function decode(tokens) {
-    return [...decodeGenerator(tokens)].join('');
-}
-exports.decode = decode;
-//# sourceMappingURL=encoder.js.map

package/cjs/encoder.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"encoder.js","sourceRoot":"","sources":["../src/encoder.ts"],"names":[],"mappings":";AAAA,0CAA0C;AAC1C,8CAA8C;;;;;;AAE9C,wGAAwG;AACxG,aAAa;AACb,0EAAgD;AAChD,wGAAwG;AACxG,aAAa;AACb,wEAA8C;AAC9C,mCAOgB;AAEhB,MAAM,OAAO,GAA2B,sBAAW,CAAA;AACnD,MAAM,QAAQ,GAA2B,uBAAY,CAAA;AAErD,MAAM,OAAO,GAA2B,IAAA,eAAO,EAC7C,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,EACtB,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CACrB,CAAA;AAED,MAAM,WAAW,GAAG,IAAI,WAAW,EAAE,CAAA;AACrC,MAAM,WAAW,GAAG,IAAI,WAAW,EAAE,CAAA;AAErC,4CAA4C;AAC5C,MAAM,gBAAgB,GAAG,CAAC,IAAI,CAAC,CAAA;AAE/B,SAAS,sBAAsB;IAC7B,MAAM,UAAU,GAAG,IAAA,aAAK,EAAC,IAAA,mBAAW,EAAC,GAAG,CAAC,EAAE,IAAA,mBAAW,EAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAA;IAChE,MAAM,YAAY,GAAG,IAAA,aAAK,EAAC,IAAA,mBAAW,EAAC,GAAG,CAAC,EAAE,IAAA,mBAAW,EAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAA;IAClE,MAAM,YAAY,GAAG,IAAA,aAAK,EAAC,IAAA,mBAAW,EAAC,GAAG,CAAC,EAAE,IAAA,mBAAW,EAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAA;IAElE,MAAM,iBAAiB,GAAG,CAAC,GAAG,UAAU,EAAE,GAAG,YAAY,EAAE,GAAG,YAAY,CAAC,CAAA;IAC3E,MAAM,gBAAgB,GAAG,CAAC,GAAG,iBAAiB,CAAC,CAAA;IAE/C,IAAI,kBAAkB,GAAG,CAAC,CAAA;IAC1B,KAAK,IAAI,SAAS,GAAG,CAAC,EAAE,SAAS,GAAG,gBAAgB,EAAE,SAAS,EAAE,EAAE;QACjE,IAAI,CAAC,iBAAiB,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE;YAC1C,iBAAiB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;YACjC,gBAAgB,CAAC,IAAI,CAAC,gBAAgB,GAAG,kBAAkB,CAAC,CAAA;YAC5D,kBAAkB,IAAI,CAAC,CAAA;SACxB;KACF;IAED,MAAM,YAAY,GAAG,gBAAgB,CAAC,GAAG,CAAC,uBAAe,CAAC,CAAA;IAC1D,OAAO,IAAI,GAAG,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC,CAAE,CAAC,CAAC,CAAC,CAAA;AACxE,CAAC;AAED,MAAM,GAAG,GACP,8EAA8E,CAAA;AAEhF,MAAM,WAAW,GAAG,sBAAsB,EAAE,CAAA;AAC5C,MAAM,WAAW,GAAwB,IAAI,GAAG,CAC9C,CAAC,GAAG,WAAW,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CACzC,CAAA;AAED,SAAS,GAAG,CAAC,KAAa,EAAE,QAA6B,IAAI,GAAG,EAAE;IAChE,IAAI,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE;QACpB,OAAO,KAAK,CAAC,GAAG,CAAC,KAAK,CAAE,CAAA;KACzB;IAED,IAAI,IAAI,GAAG,IAAA,kBAAU,EAAC,KAAK,CAAC,CAAA;IAC5B,IAAI,KAAK,GAAG,IAAA,gBAAQ,EAAC,IAAI,CAAC,CAAA;IAE1B,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE;QACtB,OAAO,KAAK,CAAA;KACb;IAED,iDAAiD;IACjD,OAAO,IAAI,EAAE;QACX,MAAM,QAAQ,GAAkC,IAAI,GAAG,EAAE,CAAA;QACzD,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;YACrB,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAA;YACrC,QAAQ,CAAC,GAAG;YACV,4CAA4C;YAC5C,OAAO,IAAI,KAAK,WAAW,IAAI,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAChE,IAAI,CACL,CAAA;QACH,CAAC,CAAC,CAAA;QACF,MAAM,IAAI,GAAG,CAAC,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAA;QAEjC,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC,CAAA;QAE9C,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,QAAQ,CAAC,EAAE;YAC9C,MAAK;SACN;QAED,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,GAAG,MAAM,CAAA;QAC9B,IAAI,OAAO,GAAa,EAAE,CAAA;QAC1B,IAAI,CAAC,GAAG,CAAC,CAAA;QAET,OAAO,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE;YACtB,MAAM,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,CAAA;YAChC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE;gBACZ,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAA;gBACvC,MAAK;aACN;YACD,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;YAC1C,CAAC,GAAG,CAAC,CAAA;YAEL,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,MAAM,EAAE;gBACtE,OAAO,CAAC,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC,CAAA;gBAC5B,CAAC,IAAI,CAAC,CAAA;aACP;iBAAM;gBACL,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAE,CAAC,CAAA;gBACtB,CAAC,IAAI,CAAC,CAAA;aACP;SACF;QAED,IAAI,GAAG,OAAO,CAAA;QACd,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE;YACrB,MAAK;SACN;aAAM;YACL,KAAK,GAAG,IAAA,gBAAQ,EAAC,IAAI,CAAC,CAAA;SACvB;KACF;IAED,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;IAC7B,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,CAAA;IAExB,OAAO,MAAM,CAAA;AACf,CAAC;AAED,QAAQ,CAAC,CAAC,eAAe,CACvB,IAAY,EACZ,QAA6B,IAAI,GAAG,EAAE;IAEtC,KAAK,IAAI,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE;QACtC,KAAK,GAAG,CAAC,GAAG,WAAW,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;aACnC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;aACpC,IAAI,CAAC,EAAE,CAAC,CAAA;QAEX,MAAM,SAAS,GAAG,GAAG,CAAC,KAAK,EAAE,KAAK,CAAC;aAChC,KAAK,CAAC,GAAG,CAAC;aACV,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,CAAE,CAAC,CAAA;QAE1B,MAAM,SAAS,CAAA;KAChB;AACH,CAAC;AAED;;GAEG;AACH,SAAgB,kBAAkB,CAChC,IAAY,EACZ,UAAkB,EAClB,QAA6B,IAAI,GAAG,EAAE;IAEtC,MAAM,cAAc,GAAG,eAAe,CAAC,IAAI,EAAE,KAAK,CAAC,CAAA;IACnD,IAAI,KAAK,GAAG,CAAC,CAAA;IACb,KAAK,MAAM,MAAM,IAAI,cAAc,EAAE;QACnC,KAAK,IAAI,MAAM,CAAC,MAAM,CAAA;QACtB,IAAI,KAAK,GAAG,UAAU,EAAE;YACtB,OAAO,KAAK,CAAA;SACb;KACF;IACD,OAAO,KAAK,CAAA;AACd,CAAC;AAdD,gDAcC;AAED,SAAgB,MAAM,CACpB,IAAY,EACZ,QAA6B,IAAI,GAAG,EAAE;IAEtC,OAAO,CAAC,GAAG,eAAe,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;AAClD,CAAC;AALD,wBAKC;AAED,SAAgB,WAAW,CAAC,KAAa;IACvC,MAAM,YAAY,GAAG,OAAO,CAAC,KAAK,CAAC,CAAA;IACnC,IAAI,OAAO,YAAY,KAAK,WAAW,EAAE;QACvC,OAAO,EAAE,CAAA;KACV;IACD,MAAM,YAAY,GAAG,IAAA,kBAAU,EAAC,YAAY,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC,CAAA;IAC7E,OAAO,WAAW,CAAC,MAAM,CAAC,IAAI,UAAU,CAAC,YAAY,CAAC,EAAE;QACtD,MAAM,EAAE,IAAI;KACb,CAAC,CAAA;AACJ,CAAC;AATD,kCASC;AAED,MAAM,oBAAoB,GAAG,KAAM,CAAA;AACnC,MAAM,kBAAkB,GAAG,KAAM,CAAA;AACjC,SAAgB,kCAAkC,CAAC,MAAc;IAC/D,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,KAAK,CAAA;IACrC,kDAAkD;IAClD,MAAM,YAAY,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;IACzD,OAAO,CACL,YAAY,IAAI,oBAAoB,IAAI,YAAY,IAAI,kBAAkB,CAC3E,CAAA;AACH,CAAC;AAPD,gFAOC;AAED,QAAe,CAAC,CAAC,eAAe,CAC9B,MAAwB;IAExB,IAAI,MAAM,GAAG,EAAE,CAAA;IAEf,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE;QAC1B,MAAM,IAAI,WAAW,CAAC,KAAK,CAAC,CAAA;QAE5B,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,kCAAkC,CAAC,MAAM,CAAC,EAAE;YACrE,yEAAyE;YACzE,uCAAuC;YACvC,SAAQ;SACT;aAAM;YACL,MAAM,MAAM,CAAA;YACZ,eAAe;YACf,MAAM,GAAG,EAAE,CAAA;SACZ;KACF;IAED,+CAA+C;IAC/C,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE;QACrB,MAAM,MAAM,CAAA;KACb;AACH,CAAC;AAvBD,0CAuBC;AAED;;;;;GAKG;AACI,KAAK,SAAS,CAAC,CAAC,oBAAoB,CACzC,MAA6B;IAE7B,IAAI,MAAM,GAAG,EAAE,CAAA;IAEf,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,MAAM,EAAE;QAChC,MAAM,IAAI,WAAW,CAAC,KAAK,CAAC,CAAA;QAE5B,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,kCAAkC,CAAC,MAAM,CAAC,EAAE;YACrE,yEAAyE;YACzE,uCAAuC;YACvC,SAAQ;SACT;aAAM;YACL,MAAM,MAAM,CAAA;YACZ,eAAe;YACf,MAAM,GAAG,EAAE,CAAA;SACZ;KACF;IAED,+CAA+C;IAC/C,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE;QACrB,MAAM,MAAM,CAAA;KACb;AACH,CAAC;AAvBD,oDAuBC;AAED,SAAgB,MAAM,CAAC,MAAgB;IACrC,OAAO,CAAC,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;AAC9C,CAAC;AAFD,wBAEC"}

package/cjs/encoder.test.js

@@ -1,86 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-/* eslint-disable compat/compat */
-const encoder_1 = require("./encoder");
-describe('basic functionality', () => {
-    it('empty string', () => {
-        const str = '';
-        expect((0, encoder_1.encode)(str)).toEqual([]);
-        expect((0, encoder_1.decode)((0, encoder_1.encode)(str))).toEqual(str);
-        expect((0, encoder_1.isWithinTokenLimit)(str, 0)).toBe(0);
-        expect((0, encoder_1.isWithinTokenLimit)(str, 3)).toBe(0);
-    });
-    it('space', () => {
-        const str = ' ';
-        expect((0, encoder_1.encode)(str)).toEqual([220]);
-        expect((0, encoder_1.decode)((0, encoder_1.encode)(str))).toEqual(str);
-        expect((0, encoder_1.isWithinTokenLimit)(str, 3)).toBe(1);
-        expect((0, encoder_1.isWithinTokenLimit)(str, 0)).toBe(false);
-    });
-    it('tab', () => {
-        const str = '\t';
-        expect((0, encoder_1.encode)(str)).toEqual([197]);
-        expect((0, encoder_1.decode)((0, encoder_1.encode)(str))).toEqual(str);
-    });
-    it('simple text', () => {
-        const str = 'This is some text';
-        expect((0, encoder_1.encode)(str)).toEqual([1212, 318, 617, 2420]);
-        expect((0, encoder_1.decode)((0, encoder_1.encode)(str))).toEqual(str);
-        expect((0, encoder_1.isWithinTokenLimit)(str, 3)).toBe(false);
-        expect((0, encoder_1.isWithinTokenLimit)(str, 5)).toBe(4);
-    });
-    it('multi-token word', () => {
-        const str = 'indivisible';
-        expect((0, encoder_1.encode)(str)).toEqual([521, 452, 12843]);
-        expect((0, encoder_1.decode)((0, encoder_1.encode)(str))).toEqual(str);
-        expect((0, encoder_1.isWithinTokenLimit)(str, 3)).toBe(3);
-    });
-    const helloWorldTokens = [31373, 50169, 233, 995, 12520, 234, 235];
-    it('emojis', () => {
-        const str = 'hello 👋 world 🌍';
-        expect((0, encoder_1.encode)(str)).toEqual(helloWorldTokens);
-        expect((0, encoder_1.decode)((0, encoder_1.encode)(str))).toEqual(str);
-        expect((0, encoder_1.isWithinTokenLimit)(str, 4)).toBe(false);
-        expect((0, encoder_1.isWithinTokenLimit)(str, 400)).toBe(7);
-    });
-    it('decode token-by-token via generator', () => {
-        const generator = (0, encoder_1.decodeGenerator)(helloWorldTokens);
-        expect(generator.next().value).toBe('hello');
-        expect(generator.next().value).toBe(' ');
-        expect(generator.next().value).toBe('👋');
-        expect(generator.next().value).toBe(' world');
-        expect(generator.next().value).toBe(' ');
-        expect(generator.next().value).toBe('🌍');
-    });
-    async function* getHelloWorldTokensAsync() {
-        for (const token of helloWorldTokens) {
-            // eslint-disable-next-line no-await-in-loop
-            yield await Promise.resolve(token);
-        }
-    }
-    it('decode token-by-token via async generator', async () => {
-        const generator = (0, encoder_1.decodeAsyncGenerator)(getHelloWorldTokensAsync());
-        const decoded = ['hello', ' ', '👋', ' world', ' ', '🌍'];
-        for await (const value of generator) {
-            expect(value).toEqual(decoded.shift());
-        }
-    });
-    it('properties of Object', () => {
-        const str = 'toString constructor hasOwnProperty valueOf';
-        expect((0, encoder_1.encode)(str)).toEqual([
-            1462, 10100, 23772, 468, 23858, 21746, 1988, 5189,
-        ]);
-        expect((0, encoder_1.decode)((0, encoder_1.encode)(str))).toEqual(str);
-    });
-    it('text with commas', () => {
-        const str = 'hello, I am a text, and I have commas. a,b,c';
-        expect((0, encoder_1.decode)((0, encoder_1.encode)(str))).toEqual(str);
-        expect((0, encoder_1.encode)(str)).toStrictEqual([
-            31373, 11, 314, 716, 257, 2420, 11, 290, 314, 423, 725, 292, 13, 257,
-            11, 65, 11, 66,
-        ]);
-        expect((0, encoder_1.isWithinTokenLimit)(str, 15)).toBe(false);
-        expect((0, encoder_1.isWithinTokenLimit)(str, 300)).toBe(18);
-    });
-});
-//# sourceMappingURL=encoder.test.js.map