gpt-tokenizer 2.0.0 → 2.1.1

package/README.md

@@ -4,21 +4,22 @@
 
 `gpt-tokenizer` is a highly optimized Token Byte Pair Encoder/Decoder for all OpenAI's models (including those used by GPT-2, GPT-3, GPT-3.5 and GPT-4). It's written in TypeScript, and is fully compatible with all modern JavaScript environments.
 
+This package is a port of OpenAI's [tiktoken](https://github.com/openai/tiktoken), with some additional features sprinkled on top.
+
 OpenAI's GPT models utilize byte pair encoding to transform text into a sequence of integers before feeding them into the model.
 
 As of 2023, it is the most feature-complete, open-source GPT tokenizer on NPM. It implements some unique features, such as:
 
+- Support for easily tokenizing chats thanks to the `encodeChat` function
 - Support for all current OpenAI models (available encodings: `r50k_base`, `p50k_base`, `p50k_edit` and `cl100k_base`)
-- Generator function versions of both the decoder and encoder
+- Generator function versions of both the decoder and encoder functions
 - Provides the ability to decode an asynchronous stream of data (using `decodeAsyncGenerator` and `decodeGenerator` with any iterable input)
 - No global cache (no accidental memory leaks, as with the original GPT-3-Encoder implementation)
-- Includes a highly performant `isWithinTokenLimit` function to assess token limit without encoding the entire text
+- Includes a highly performant `isWithinTokenLimit` function to assess token limit without encoding the entire text/chat
 - Improves overall performance by eliminating transitive arrays
 - Type-safe (written in TypeScript)
 - Works in the browser out-of-the-box
 
-This package is a port of OpenAI's [tiktoken](https://github.com/openai/tiktoken), with some additional features sprinkled on top.
-
 Thanks to @dmitry-brazhenko's [SharpToken](https://github.com/dmitry-brazhenko/SharpToken), whose code was served as a reference for the port.
 
 Historical note: This package started off as a fork of [latitudegames/GPT-3-Encoder](https://github.com/latitudegames/GPT-3-Encoder), but version 2.0 was rewritten from scratch.
@@ -38,21 +39,25 @@ npm install gpt-tokenizer
 
 <script>
   // the package is now available as a global:
-  const { encode, decode } = GPTTokenizer
+  const { encode, decode } = GPTTokenizer_cl100k_base
 </script>
 ```
 
-If you wish to use a custom encoding, fetch the relevant script:
+If you wish to use a custom encoding, fetch the relevant script.
 
 - https://unpkg.com/gpt-tokenizer/dist/cl100k_base.js
 - https://unpkg.com/gpt-tokenizer/dist/p50k_base.js
 - https://unpkg.com/gpt-tokenizer/dist/p50k_edit.js
 - https://unpkg.com/gpt-tokenizer/dist/r50k_base.js
 
+The global name is a concatenation: `GPTTokenizer_${encoding}`.
+
 Refer to [supported models and their encodings](#Supported-models-and-their-encodings) section for more information.
 
 ## Playground
 
+The playground is published under a memorable URL: https://gpt-tokenizer.dev/
+
 You can play with the package in the browser using the [Playground](https://codesandbox.io/s/gpt-tokenizer-tjcjoz?fontsize=14&hidenavigation=1&theme=dark).
 
 [![GPT Tokenizer Playground](./docs/gpt-tokenizer.png)](https://codesandbox.io/s/gpt-tokenizer-tjcjoz?fontsize=14&hidenavigation=1&theme=dark)
@@ -64,6 +69,7 @@ The playground mimics the official [OpenAI Tokenizer](https://platform.openai.co
 ```typescript
 import {
   encode,
+  encodeChat,
   decode,
   isWithinTokenLimit,
   encodeGenerator,
@@ -84,6 +90,18 @@ const decodedText = decode(tokens)
 // returns false if the limit is exceeded, otherwise returns the actual number of tokens (truthy value)
 const withinTokenLimit = isWithinTokenLimit(text, tokenLimit)
 
+// Example chat:
+const chat = [
+  { role: 'system', content: 'You are a helpful assistant.' },
+  { role: 'assistant', content: 'gpt-tokenizer is awesome.' },
+]
+
+// Encode chat into tokens
+const chatTokens = encodeChat(chat)
+
+// Check if chat is within the token limit
+const chatWithinTokenLimit = isWithinTokenLimit(chat, tokenLimit)
+
 // Encode text using generator
 for (const tokenChunk of encodeGenerator(text)) {
   console.log(tokenChunk)
@@ -127,10 +145,13 @@ import {
 
 chat:
 
-- `gpt-4` (`cl100k_base`)
+- `gpt-4-32k` (`cl100k_base`)
+- `gpt-4-0314` (`cl100k_base`)
+- `gpt-4-32k-0314` (`cl100k_base`)
 - `gpt-3.5-turbo` (`cl100k_base`)
+- `gpt-3.5-turbo-0301` (`cl100k_base`)
 
-text:
+text-only:
 
 - `text-davinci-003` (`p50k_base`)
 - `text-davinci-002` (`p50k_base`)
@@ -216,6 +237,24 @@ const tokenLimit = 10
 const withinTokenLimit = isWithinTokenLimit(text, tokenLimit)
 ```
 
+### `encodeChat(chat: ChatMessage[], model?: ModelName): number[]`
+
+Encodes the given chat into a sequence of tokens.
+
+If you didn't import the model version directly, or if `model` wasn't provided during initialization, it must be provided here to correctly tokenize the chat for a given model. Use this method when you need to transform a chat into the token format that the GPT models can process.
+
+Example:
+
+```typescript
+import { encodeChat } from 'gpt-tokenizer'
+
+const chat = [
+  { role: 'system', content: 'You are a helpful assistant.' },
+  { role: 'assistant', content: 'gpt-tokenizer is awesome.' },
+]
+const tokens = encodeChat(chat)
+```
+
 ### `encodeGenerator(text: string): Generator<number[], void, undefined>`
 
 Encodes the given text using a generator, yielding chunks of tokens.
@@ -233,6 +272,10 @@ for (const tokenChunk of encodeGenerator(text)) {
 }
 ```
 
+### `encodeChatGenerator(chat: Iterator<ChatMessage>, model?: ModelName): Generator<number[], void, undefined>`
+
+Same as `encodeChat`, but uses a generator as output, and may use any iterator as the input `chat`.
+
 ### `decodeGenerator(tokens: Iterable<number>): Generator<string, void, undefined>`
 
 Decodes a sequence of tokens using a generator, yielding chunks of decoded text.
@@ -284,6 +327,9 @@ import {
   FimMiddle,
   FimPrefix,
   FimSuffix,
+  ImStart,
+  ImEnd,
+  ImSep,
   encode,
 } from 'gpt-tokenizer'
 

package/cjs/GptEncoding.d.ts

@@ -1,6 +1,18 @@
 import { type EncodingName, type ModelName } from './mapping.js';
 import { type GetMergeableRanksAsyncFn, type GetMergeableRanksFn } from './modelParams.js';
-export declare const DISALLOW_ALL_SPECIAL_TOKENS = "all";
+export declare const ALL_SPECIAL_TOKENS = "all";
+export interface EncodeOptions {
+    allowedSpecial?: Set<string>;
+    disallowedSpecial?: Set<string>;
+}
+export interface ChatMessage {
+    role?: 'system' | 'user' | 'assistant';
+    name?: string;
+    content: string;
+}
+export interface EncodeChatOptions {
+    primeWithAssistantResponse?: string;
+}
 export declare class GptEncoding {
     static EndOfPrompt: string;
     static EndOfText: string;
@@ -8,6 +20,7 @@ export declare class GptEncoding {
     static FimPrefix: string;
     static FimSuffix: string;
     decoder: TextDecoder;
+    modelName?: ModelName;
     private bytePairEncodingCoreProcessor;
     private specialTokenMapping;
     private constructor();
@@ -15,14 +28,28 @@ export declare class GptEncoding {
     static getEncodingApiForModel(modelName: ModelName, getMergeableRanks: GetMergeableRanksFn): GptEncoding;
     static getEncodingApiAsync(encodingName: EncodingName, getMergeableRanks: GetMergeableRanksAsyncFn): Promise<GptEncoding>;
     static getEncodingApiForModelAsync(modelName: ModelName, getMergeableRanks: GetMergeableRanksAsyncFn): Promise<GptEncoding>;
-    private static getMaxValueFromMap;
-    private static specialTokenRegex;
-    encodeGenerator(lineToEncode: string, allowedSpecial?: Set<string>, disallowedSpecial?: Set<string>): Generator<number[], number>;
-    encode(lineToEncode: string, allowedSpecial?: Set<string>, disallowedSpecial?: Set<string>): number[];
+    encodeGenerator(lineToEncode: string, { allowedSpecial, disallowedSpecial, }?: EncodeOptions): Generator<number[], number, undefined>;
+    encode(lineToEncode: string, encodeOptions?: EncodeOptions): number[];
+    /**
+     * Progressively tokenizes an OpenAI chat.
+     * Warning: gpt-3.5-turbo and gpt-4 chat format may change over time.
+     * Returns tokens assuming the 'gpt-3.5-turbo-0301' / 'gpt-4-0314' format.
+     * Based on OpenAI's guidelines: https://github.com/openai/openai-python/blob/main/chatml.md
+     * Also mentioned in section 6 of this document: https://github.com/openai/openai-cookbook/blob/main/examples/How_to_count_tokens_with_tiktoken.ipynb
+     */
+    encodeChatGenerator(chat: Iterable<ChatMessage>, model?: "gpt-4" | "gpt-4-32k" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo" | "gpt-3.5-turbo-0301" | "text-davinci-003" | "text-davinci-002" | "text-davinci-001" | "text-curie-001" | "text-babbage-001" | "text-ada-001" | "davinci" | "curie" | "babbage" | "ada" | "code-davinci-002" | "code-davinci-001" | "code-cushman-002" | "code-cushman-001" | "davinci-codex" | "cushman-codex" | "text-davinci-edit-001" | "code-davinci-edit-001" | "text-embedding-ada-002" | "text-similarity-davinci-001" | "text-similarity-curie-001" | "text-similarity-babbage-001" | "text-similarity-ada-001" | "text-search-davinci-doc-001" | "text-search-curie-doc-001" | "text-search-babbage-doc-001" | "text-search-ada-doc-001" | "code-search-babbage-code-001" | "code-search-ada-code-001" | undefined): Generator<number[], void, undefined>;
+    /**
+     * Encodes a chat into a single array of tokens.
+     * Warning: gpt-3.5-turbo and gpt-4 chat format may change over time.
+     * Returns tokens assuming the 'gpt-3.5-turbo-0301' / 'gpt-4-0314' format.
+     * Based on OpenAI's guidelines: https://github.com/openai/openai-python/blob/main/chatml.md
+     * Also mentioned in section 6 of this document: https://github.com/openai/openai-cookbook/blob/main/examples/How_to_count_tokens_with_tiktoken.ipynb
+     */
+    encodeChat(chat: readonly ChatMessage[], model?: "gpt-4" | "gpt-4-32k" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo" | "gpt-3.5-turbo-0301" | "text-davinci-003" | "text-davinci-002" | "text-davinci-001" | "text-curie-001" | "text-babbage-001" | "text-ada-001" | "davinci" | "curie" | "babbage" | "ada" | "code-davinci-002" | "code-davinci-001" | "code-cushman-002" | "code-cushman-001" | "davinci-codex" | "cushman-codex" | "text-davinci-edit-001" | "code-davinci-edit-001" | "text-embedding-ada-002" | "text-similarity-davinci-001" | "text-similarity-curie-001" | "text-similarity-babbage-001" | "text-similarity-ada-001" | "text-search-davinci-doc-001" | "text-search-curie-doc-001" | "text-search-babbage-doc-001" | "text-search-ada-doc-001" | "code-search-babbage-code-001" | "code-search-ada-code-001" | undefined): number[];
     /**
      * @returns {false | number} false if token limit is exceeded, otherwise the number of tokens
      */
-    isWithinTokenLimit(text: string, tokenLimit: number): false | number;
+    isWithinTokenLimit(input: string | Iterable<ChatMessage>, tokenLimit: number): false | number;
     decodeGenerator(inputTokensToDecode: Iterable<number>): Generator<string, void>;
     decodeAsyncGenerator(inputTokensToDecode: AsyncIterable<number>): AsyncGenerator<string, void>;
     decode(inputTokensToDecode: Iterable<number>): string;

package/cjs/GptEncoding.js

@@ -1,14 +1,14 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.GptEncoding = exports.DISALLOW_ALL_SPECIAL_TOKENS = void 0;
+exports.GptEncoding = exports.ALL_SPECIAL_TOKENS = void 0;
 /* eslint-disable no-param-reassign */
 const BytePairEncodingCore_js_1 = require("./BytePairEncodingCore.js");
-const escapeRegExp_js_1 = require("./escapeRegExp.js");
 const mapping_js_1 = require("./mapping.js");
 const modelParams_js_1 = require("./modelParams.js");
 const specialTokens_js_1 = require("./specialTokens.js");
 const utfUtil_js_1 = require("./utfUtil.js");
-exports.DISALLOW_ALL_SPECIAL_TOKENS = 'all';
+const util_js_1 = require("./util.js");
+exports.ALL_SPECIAL_TOKENS = 'all';
 class GptEncoding {
     static EndOfPrompt = specialTokens_js_1.EndOfPrompt;
     static EndOfText = specialTokens_js_1.EndOfText;
@@ -16,10 +16,11 @@ class GptEncoding {
     static FimPrefix = specialTokens_js_1.FimPrefix;
     static FimSuffix = specialTokens_js_1.FimSuffix;
     decoder = new TextDecoder('utf8');
+    modelName;
     bytePairEncodingCoreProcessor;
     specialTokenMapping;
-    constructor({ tokenSplitRegex, mergeableBytePairRanks, specialTokenMapping, expectedVocabularySize, }) {
-        const maxTokenValue = Math.max(GptEncoding.getMaxValueFromMap(mergeableBytePairRanks), GptEncoding.getMaxValueFromMap(specialTokenMapping));
+    constructor({ tokenSplitRegex, mergeableBytePairRanks, specialTokenMapping, expectedVocabularySize, modelName, }) {
+        const maxTokenValue = Math.max((0, util_js_1.getMaxValueFromMap)(mergeableBytePairRanks), (0, util_js_1.getMaxValueFromMap)(specialTokenMapping));
         this.specialTokenMapping = specialTokenMapping;
         if (expectedVocabularySize !== undefined) {
             if (mergeableBytePairRanks.size + specialTokenMapping.size !==
@@ -41,14 +42,16 @@ class GptEncoding {
         this.decodeGenerator = this.decodeGenerator.bind(this);
         this.decodeAsyncGenerator = this.decodeAsyncGenerator.bind(this);
         this.isWithinTokenLimit = this.isWithinTokenLimit.bind(this);
+        this.modelName = modelName;
     }
     static getEncodingApi(encodingName, getMergeableRanks) {
-        const modelParams = (0, modelParams_js_1.getModelParams)(encodingName, getMergeableRanks);
+        const modelParams = (0, modelParams_js_1.getEncodingParams)(encodingName, getMergeableRanks);
         return new GptEncoding(modelParams);
     }
     static getEncodingApiForModel(modelName, getMergeableRanks) {
         const encodingName = mapping_js_1.modelToEncodingMap[modelName];
-        return GptEncoding.getEncodingApi(encodingName, getMergeableRanks);
+        const modelParams = (0, modelParams_js_1.getEncodingParams)(encodingName, getMergeableRanks);
+        return new GptEncoding({ ...modelParams, modelName });
     }
     static async getEncodingApiAsync(encodingName, getMergeableRanks) {
         const modelParams = await (0, modelParams_js_1.getModelParamsAsync)(encodingName, getMergeableRanks);
@@ -56,33 +59,21 @@ class GptEncoding {
     }
     static async getEncodingApiForModelAsync(modelName, getMergeableRanks) {
         const encodingName = mapping_js_1.modelToEncodingMap[modelName];
-        return GptEncoding.getEncodingApiAsync(encodingName, getMergeableRanks);
-    }
-    static getMaxValueFromMap(map) {
-        let max = 0;
-        map.forEach((val) => {
-            max = Math.max(max, val);
-        });
-        return max;
-    }
-    static specialTokenRegex(tokens) {
-        const escapedTokens = [...tokens].map(escapeRegExp_js_1.escapeRegExp);
-        const inner = escapedTokens.join('|');
-        return new RegExp(`(${inner})`);
+        const modelParams = await (0, modelParams_js_1.getModelParamsAsync)(encodingName, getMergeableRanks);
+        return new GptEncoding({ ...modelParams, modelName });
     }
-    encodeGenerator(lineToEncode, allowedSpecial = new Set(), disallowedSpecial = new Set([
-        exports.DISALLOW_ALL_SPECIAL_TOKENS,
-    ])) {
+    encodeGenerator(lineToEncode, { allowedSpecial = new Set(), disallowedSpecial = new Set([exports.ALL_SPECIAL_TOKENS]), } = {}) {
         const specialTokensSet = new Set(this.specialTokenMapping.keys());
-        if (disallowedSpecial.has(exports.DISALLOW_ALL_SPECIAL_TOKENS)) {
+        if (disallowedSpecial.has(exports.ALL_SPECIAL_TOKENS)) {
             disallowedSpecial = new Set(specialTokensSet);
+            allowedSpecial.forEach((val) => disallowedSpecial.delete(val));
             disallowedSpecial.forEach((val) => allowedSpecial.delete(val));
         }
-        if (allowedSpecial.has(exports.DISALLOW_ALL_SPECIAL_TOKENS)) {
+        if (allowedSpecial.has(exports.ALL_SPECIAL_TOKENS)) {
             allowedSpecial = specialTokensSet;
         }
         if (disallowedSpecial.size > 0) {
-            const regexPattern = GptEncoding.specialTokenRegex(disallowedSpecial);
+            const regexPattern = (0, util_js_1.getSpecialTokenRegex)(disallowedSpecial);
             const match = lineToEncode.match(regexPattern);
             if (match !== null) {
                 throw new Error(`Disallowed special token found: ${match[0]}`);
@@ -90,18 +81,72 @@ class GptEncoding {
         }
         return this.bytePairEncodingCoreProcessor.encodeNative(lineToEncode, allowedSpecial);
     }
-    encode(lineToEncode, allowedSpecial = new Set(), disallowedSpecial = new Set([
-        exports.DISALLOW_ALL_SPECIAL_TOKENS,
-    ])) {
-        return [
-            ...this.encodeGenerator(lineToEncode, allowedSpecial, disallowedSpecial),
-        ].flat();
+    encode(lineToEncode, encodeOptions = {}) {
+        return [...this.encodeGenerator(lineToEncode, encodeOptions)].flat();
+    }
+    /**
+     * Progressively tokenizes an OpenAI chat.
+     * Warning: gpt-3.5-turbo and gpt-4 chat format may change over time.
+     * Returns tokens assuming the 'gpt-3.5-turbo-0301' / 'gpt-4-0314' format.
+     * Based on OpenAI's guidelines: https://github.com/openai/openai-python/blob/main/chatml.md
+     * Also mentioned in section 6 of this document: https://github.com/openai/openai-cookbook/blob/main/examples/How_to_count_tokens_with_tiktoken.ipynb
+     */
+    *encodeChatGenerator(chat, model = this.modelName) {
+        if (!model) {
+            throw new Error('Model name must be provided either during initialization or passed in to the method.');
+        }
+        const params = mapping_js_1.chatModelParams[model];
+        const chatStartToken = this.specialTokenMapping.get(specialTokens_js_1.ImStart);
+        const chatEndToken = this.specialTokenMapping.get(specialTokens_js_1.ImEnd);
+        if (!params || chatStartToken === undefined || chatEndToken === undefined) {
+            throw new Error(`Model '${model}' does not support chat.`);
+        }
+        const allowedSpecial = new Set([specialTokens_js_1.ImSep]);
+        const { messageSeparator, roleSeparator } = params;
+        const encodedMessageSeparator = messageSeparator.length > 0 ? this.encode(messageSeparator) : [];
+        const encodedRoleSeparator = roleSeparator.length > 0
+            ? this.encode(roleSeparator, { allowedSpecial })
+            : [];
+        const nameCache = new Map();
+        for (const { role = 'system', name = role, content } of chat) {
+            if (content === undefined) {
+                throw new Error('Content must be defined for all messages.');
+            }
+            yield [chatStartToken];
+            const encodedName = nameCache.get(name) ?? this.encode(name);
+            nameCache.set(name, encodedName);
+            yield encodedName;
+            if (encodedRoleSeparator.length > 0) {
+                yield encodedRoleSeparator;
+            }
+            yield* this.encodeGenerator(content);
+            yield [chatEndToken];
+            yield encodedMessageSeparator;
+        }
+        // every reply is primed with <|start|>assistant<|message|>
+        yield [chatStartToken];
+        yield* this.encodeGenerator('assistant');
+        if (encodedRoleSeparator.length > 0) {
+            yield encodedRoleSeparator;
+        }
+    }
+    /**
+     * Encodes a chat into a single array of tokens.
+     * Warning: gpt-3.5-turbo and gpt-4 chat format may change over time.
+     * Returns tokens assuming the 'gpt-3.5-turbo-0301' / 'gpt-4-0314' format.
+     * Based on OpenAI's guidelines: https://github.com/openai/openai-python/blob/main/chatml.md
+     * Also mentioned in section 6 of this document: https://github.com/openai/openai-cookbook/blob/main/examples/How_to_count_tokens_with_tiktoken.ipynb
+     */
+    encodeChat(chat, model = this.modelName) {
+        return [...this.encodeChatGenerator(chat, model)].flat();
     }
     /**
      * @returns {false | number} false if token limit is exceeded, otherwise the number of tokens
      */
-    isWithinTokenLimit(text, tokenLimit) {
-        const tokenGenerator = this.encodeGenerator(text);
+    isWithinTokenLimit(input, tokenLimit) {
+        const tokenGenerator = typeof input === 'string'
+            ? this.encodeGenerator(input)
+            : this.encodeChatGenerator(input);
         let count = 0;
         for (const tokens of tokenGenerator) {
             count += tokens.length;

package/cjs/GptEncoding.js.map

@@ -1 +1 @@
-{"version":3,"file":"GptEncoding.js","sourceRoot":"","sources":["../src/GptEncoding.ts"],"names":[],"mappings":";;;AAAA,sCAAsC;AACtC,uEAAgE;AAChE,uDAAgD;AAChD,6CAIqB;AACrB,qDAMyB;AACzB,yDAM2B;AAC3B,6CAAiE;AAEpD,QAAA,2BAA2B,GAAG,KAAK,CAAA;AAEhD,MAAa,WAAW;IACtB,MAAM,CAAC,WAAW,GAAG,8BAAW,CAAA;IAChC,MAAM,CAAC,SAAS,GAAG,4BAAS,CAAA;IAC5B,MAAM,CAAC,SAAS,GAAG,4BAAS,CAAA;IAC5B,MAAM,CAAC,SAAS,GAAG,4BAAS,CAAA;IAC5B,MAAM,CAAC,SAAS,GAAG,4BAAS,CAAA;IAE5B,OAAO,GAAG,IAAI,WAAW,CAAC,MAAM,CAAC,CAAA;IACzB,6BAA6B,CAAsB;IACnD,mBAAmB,CAAqB;IAEhD,YAAoB,EAClB,eAAe,EACf,sBAAsB,EACtB,mBAAmB,EACnB,sBAAsB,GACV;QACZ,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,CAC5B,WAAW,CAAC,kBAAkB,CAAC,sBAAsB,CAAC,EACtD,WAAW,CAAC,kBAAkB,CAAC,mBAAmB,CAAC,CACpD,CAAA;QACD,IAAI,CAAC,mBAAmB,GAAG,mBAAmB,CAAA;QAE9C,IAAI,sBAAsB,KAAK,SAAS,EAAE;YACxC,IACE,sBAAsB,CAAC,IAAI,GAAG,mBAAmB,CAAC,IAAI;gBACtD,sBAAsB,EACtB;gBACA,MAAM,IAAI,KAAK,CACb,sFAAsF,CACvF,CAAA;aACF;YAED,IAAI,aAAa,KAAK,sBAAsB,GAAG,CAAC,EAAE;gBAChD,MAAM,IAAI,KAAK,CACb,gEAAgE,CACjE,CAAA;aACF;SACF;QAED,IAAI,CAAC,6BAA6B,GAAG,IAAI,8CAAoB,CAAC;YAC5D,eAAe,EAAE,sBAAsB;YACvC,mBAAmB,EAAE,mBAAmB;YACxC,eAAe;SAChB,CAAC,CAAA;QAEF,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACpC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACpC,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACtD,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACtD,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAChE,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IAC9D,CAAC;IAED,MAAM,CAAC,cAAc,CACnB,YAA0B,EAC1B,iBAAsC;QAEtC,MAAM,WAAW,GAAG,IAAA,+BAAc,EAAC,YAAY,EAAE,iBAAiB,CAAC,CAAA;QACnE,OAAO,IAAI,WAAW,CAAC,WAAW,CAAC,CAAA;IACrC,CAAC;IAED,MAAM,CAAC,sBAAsB,CAC3B,SAAoB,EACpB,iBAAsC;QAEtC,MAAM,YAAY,GAAG,+BAAkB,CAAC,SAAS,CAAC,CAAA;QAClD,OAAO,WAAW,CAAC,cAAc,CAAC,YAAY,EAAE,iBAAiB,CAAC,CAAA;IACpE,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,mBAAmB,CAC9B,YAA0B,EAC1B,iBAA2C;QAE3C,MAAM,WAAW,GAAG,MAAM,IAAA,oCAAmB,EAC3C,YAAY,EACZ,iBAAiB,CAClB,CAAA;QACD,OAAO,IAAI,WAAW,CAAC,WAAW,CAAC,CAAA;IACrC,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,2BAA2B,CACtC,SAAoB,EACpB,iBAA2C;QAE3C,MAAM,YAAY,GAAG,+BAAkB,CAAC,SAAS,CAAC,CAAA;QAClD,OAAO,WAAW,CAAC,mBAAmB,CAAC,YAAY,EAAE,iBAAiB,CAAC,CAAA;IACzE,CAAC;IAEO,MAAM,CAAC,kBAAkB,CAAC,GAAyB;QACzD,IAAI,GAAG,GAAG,CAAC,CAAA;QACX,GAAG,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE;YAClB,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,CAAA;QAC1B,CAAC,CAAC,CAAA;QACF,OAAO,GAAG,CAAA;IACZ,CAAC;IAEO,MAAM,CAAC,iBAAiB,CAAC,MAAmB;QAClD,MAAM,aAAa,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,8BAAY,CAAC,CAAA;QACnD,MAAM,KAAK,GAAG,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QACrC,OAAO,IAAI,MAAM,CAAC,IAAI,KAAK,GAAG,CAAC,CAAA;IACjC,CAAC;IAED,eAAe,CACb,YAAoB,EACpB,iBAAiB,IAAI,GAAG,EAAU,EAClC,oBAAiC,IAAI,GAAG,CAAS;QAC/C,mCAA2B;KAC5B,CAAC;QAEF,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAAS,IAAI,CAAC,mBAAmB,CAAC,IAAI,EAAE,CAAC,CAAA;QAEzE,IAAI,iBAAiB,CAAC,GAAG,CAAC,mCAA2B,CAAC,EAAE;YACtD,iBAAiB,GAAG,IAAI,GAAG,CAAS,gBAAgB,CAAC,CAAA;YACrD,iBAAiB,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,cAAc,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;SAC/D;QAED,IAAI,cAAc,CAAC,GAAG,CAAC,mCAA2B,CAAC,EAAE;YACnD,cAAc,GAAG,gBAAgB,CAAA;SAClC;QAED,IAAI,iBAAiB,CAAC,IAAI,GAAG,CAAC,EAAE;YAC9B,MAAM,YAAY,GAAG,WAAW,CAAC,iBAAiB,CAAC,iBAAiB,CAAC,CAAA;YACrE,MAAM,KAAK,GAAG,YAAY,CAAC,KAAK,CAAC,YAAY,CAAC,CAAA;YAC9C,IAAI,KAAK,KAAK,IAAI,EAAE;gBAClB,MAAM,IAAI,KAAK,CAAC,mCAAmC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;aAC/D;SACF;QAED,OAAO,IAAI,CAAC,6BAA6B,CAAC,YAAY,CACpD,YAAY,EACZ,cAAc,CACf,CAAA;IACH,CAAC;IAED,MAAM,CACJ,YAAoB,EACpB,iBAAiB,IAAI,GAAG,EAAU,EAClC,oBAAiC,IAAI,GAAG,CAAS;QAC/C,mCAA2B;KAC5B,CAAC;QAEF,OAAO;YACL,GAAG,IAAI,CAAC,eAAe,CAAC,YAAY,EAAE,cAAc,EAAE,iBAAiB,CAAC;SACzE,CAAC,IAAI,EAAE,CAAA;IACV,CAAC;IAED;;OAEG;IACH,kBAAkB,CAAC,IAAY,EAAE,UAAkB;QACjD,MAAM,cAAc,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,CAAA;QACjD,IAAI,KAAK,GAAG,CAAC,CAAA;QACb,KAAK,MAAM,MAAM,IAAI,cAAc,EAAE;YACnC,KAAK,IAAI,MAAM,CAAC,MAAM,CAAA;YACtB,IAAI,KAAK,GAAG,UAAU,EAAE;gBACtB,OAAO,KAAK,CAAA;aACb;SACF;QACD,OAAO,KAAK,CAAA;IACd,CAAC;IAED,CAAC,eAAe,CACd,mBAAqC;QAErC,MAAM,oBAAoB,GACxB,IAAI,CAAC,6BAA6B,CAAC,YAAY,CAAC,mBAAmB,CAAC,CAAA;QAEtE,IAAI,MAAM,GAAG,EAAE,CAAA;QAEf,KAAK,MAAM,WAAW,IAAI,oBAAoB,EAAE;YAC9C,MAAM,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;YAE5D,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,IAAA,+CAAkC,EAAC,MAAM,CAAC,EAAE;gBACrE,yEAAyE;gBACzE,uCAAuC;gBACvC,SAAQ;aACT;iBAAM;gBACL,MAAM,MAAM,CAAA;gBACZ,eAAe;gBACf,MAAM,GAAG,EAAE,CAAA;aACZ;SACF;QAED,+CAA+C;QAC/C,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE;YACrB,MAAM,MAAM,CAAA;SACb;IACH,CAAC;IAED,KAAK,CAAC,CAAC,oBAAoB,CACzB,mBAA0C;QAE1C,MAAM,oBAAoB,GACxB,IAAI,CAAC,6BAA6B,CAAC,iBAAiB,CAAC,mBAAmB,CAAC,CAAA;QAE3E,IAAI,MAAM,GAAG,EAAE,CAAA;QAEf,IAAI,KAAK,EAAE,MAAM,WAAW,IAAI,oBAAoB,EAAE;YACpD,MAAM,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;YAE5D,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,IAAA,+CAAkC,EAAC,MAAM,CAAC,EAAE;gBACrE,yEAAyE;gBACzE,uCAAuC;gBACvC,SAAQ;aACT;iBAAM;gBACL,MAAM,MAAM,CAAA;gBACZ,eAAe;gBACf,MAAM,GAAG,EAAE,CAAA;aACZ;SACF;QAED,+CAA+C;QAC/C,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE;YACrB,MAAM,MAAM,CAAA;SACb;IACH,CAAC;IAED,MAAM,CAAC,mBAAqC;QAC1C,OAAO,CAAC,GAAG,IAAI,CAAC,eAAe,CAAC,mBAAmB,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IAChE,CAAC;;AA5NU,kCAAW"}
+{"version":3,"file":"GptEncoding.js","sourceRoot":"","sources":["../src/GptEncoding.ts"],"names":[],"mappings":";;;AAAA,sCAAsC;AACtC,uEAAgE;AAChE,6CAKqB;AACrB,qDAMyB;AACzB,yDAS2B;AAC3B,6CAAiE;AACjE,uCAAoE;AAEvD,QAAA,kBAAkB,GAAG,KAAK,CAAA;AAiBvC,MAAa,WAAW;IACtB,MAAM,CAAC,WAAW,GAAG,8BAAW,CAAA;IAChC,MAAM,CAAC,SAAS,GAAG,4BAAS,CAAA;IAC5B,MAAM,CAAC,SAAS,GAAG,4BAAS,CAAA;IAC5B,MAAM,CAAC,SAAS,GAAG,4BAAS,CAAA;IAC5B,MAAM,CAAC,SAAS,GAAG,4BAAS,CAAA;IAE5B,OAAO,GAAG,IAAI,WAAW,CAAC,MAAM,CAAC,CAAA;IACjC,SAAS,CAAY;IACb,6BAA6B,CAAsB;IACnD,mBAAmB,CAAqB;IAEhD,YAAoB,EAClB,eAAe,EACf,sBAAsB,EACtB,mBAAmB,EACnB,sBAAsB,EACtB,SAAS,GACM;QACf,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,CAC5B,IAAA,4BAAkB,EAAC,sBAAsB,CAAC,EAC1C,IAAA,4BAAkB,EAAC,mBAAmB,CAAC,CACxC,CAAA;QACD,IAAI,CAAC,mBAAmB,GAAG,mBAAmB,CAAA;QAE9C,IAAI,sBAAsB,KAAK,SAAS,EAAE;YACxC,IACE,sBAAsB,CAAC,IAAI,GAAG,mBAAmB,CAAC,IAAI;gBACtD,sBAAsB,EACtB;gBACA,MAAM,IAAI,KAAK,CACb,sFAAsF,CACvF,CAAA;aACF;YAED,IAAI,aAAa,KAAK,sBAAsB,GAAG,CAAC,EAAE;gBAChD,MAAM,IAAI,KAAK,CACb,gEAAgE,CACjE,CAAA;aACF;SACF;QAED,IAAI,CAAC,6BAA6B,GAAG,IAAI,8CAAoB,CAAC;YAC5D,eAAe,EAAE,sBAAsB;YACvC,mBAAmB,EAAE,mBAAmB;YACxC,eAAe;SAChB,CAAC,CAAA;QAEF,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACpC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACpC,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACtD,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACtD,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAChE,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC5D,IAAI,CAAC,SAAS,GAAG,SAAS,CAAA;IAC5B,CAAC;IAED,MAAM,CAAC,cAAc,CACnB,YAA0B,EAC1B,iBAAsC;QAEtC,MAAM,WAAW,GAAG,IAAA,kCAAiB,EAAC,YAAY,EAAE,iBAAiB,CAAC,CAAA;QACtE,OAAO,IAAI,WAAW,CAAC,WAAW,CAAC,CAAA;IACrC,CAAC;IAED,MAAM,CAAC,sBAAsB,CAC3B,SAAoB,EACpB,iBAAsC;QAEtC,MAAM,YAAY,GAAG,+BAAkB,CAAC,SAAS,CAAC,CAAA;QAClD,MAAM,WAAW,GAAG,IAAA,kCAAiB,EAAC,YAAY,EAAE,iBAAiB,CAAC,CAAA;QACtE,OAAO,IAAI,WAAW,CAAC,EAAE,GAAG,WAAW,EAAE,SAAS,EAAE,CAAC,CAAA;IACvD,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,mBAAmB,CAC9B,YAA0B,EAC1B,iBAA2C;QAE3C,MAAM,WAAW,GAAG,MAAM,IAAA,oCAAmB,EAC3C,YAAY,EACZ,iBAAiB,CAClB,CAAA;QACD,OAAO,IAAI,WAAW,CAAC,WAAW,CAAC,CAAA;IACrC,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,2BAA2B,CACtC,SAAoB,EACpB,iBAA2C;QAE3C,MAAM,YAAY,GAAG,+BAAkB,CAAC,SAAS,CAAC,CAAA;QAClD,MAAM,WAAW,GAAG,MAAM,IAAA,oCAAmB,EAC3C,YAAY,EACZ,iBAAiB,CAClB,CAAA;QACD,OAAO,IAAI,WAAW,CAAC,EAAE,GAAG,WAAW,EAAE,SAAS,EAAE,CAAC,CAAA;IACvD,CAAC;IAED,eAAe,CACb,YAAoB,EACpB,EACE,cAAc,GAAG,IAAI,GAAG,EAAU,EAClC,iBAAiB,GAAG,IAAI,GAAG,CAAS,CAAC,0BAAkB,CAAC,CAAC,MACxC,EAAE;QAErB,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAAS,IAAI,CAAC,mBAAmB,CAAC,IAAI,EAAE,CAAC,CAAA;QAEzE,IAAI,iBAAiB,CAAC,GAAG,CAAC,0BAAkB,CAAC,EAAE;YAC7C,iBAAiB,GAAG,IAAI,GAAG,CAAS,gBAAgB,CAAC,CAAA;YACrD,cAAc,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,iBAAiB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;YAC9D,iBAAiB,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,cAAc,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;SAC/D;QAED,IAAI,cAAc,CAAC,GAAG,CAAC,0BAAkB,CAAC,EAAE;YAC1C,cAAc,GAAG,gBAAgB,CAAA;SAClC;QAED,IAAI,iBAAiB,CAAC,IAAI,GAAG,CAAC,EAAE;YAC9B,MAAM,YAAY,GAAG,IAAA,8BAAoB,EAAC,iBAAiB,CAAC,CAAA;YAC5D,MAAM,KAAK,GAAG,YAAY,CAAC,KAAK,CAAC,YAAY,CAAC,CAAA;YAC9C,IAAI,KAAK,KAAK,IAAI,EAAE;gBAClB,MAAM,IAAI,KAAK,CAAC,mCAAmC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;aAC/D;SACF;QAED,OAAO,IAAI,CAAC,6BAA6B,CAAC,YAAY,CACpD,YAAY,EACZ,cAAc,CACf,CAAA;IACH,CAAC;IAED,MAAM,CAAC,YAAoB,EAAE,gBAA+B,EAAE;QAC5D,OAAO,CAAC,GAAG,IAAI,CAAC,eAAe,CAAC,YAAY,EAAE,aAAa,CAAC,CAAC,CAAC,IAAI,EAAE,CAAA;IACtE,CAAC;IAED;;;;;;OAMG;IACH,CAAC,mBAAmB,CAClB,IAA2B,EAC3B,KAAK,GAAG,IAAI,CAAC,SAAS;QAEtB,IAAI,CAAC,KAAK,EAAE;YACV,MAAM,IAAI,KAAK,CACb,sFAAsF,CACvF,CAAA;SACF;QACD,MAAM,MAAM,GAAG,4BAAe,CAAC,KAAK,CAAC,CAAA;QACrC,MAAM,cAAc,GAAG,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,0BAAO,CAAC,CAAA;QAC5D,MAAM,YAAY,GAAG,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,wBAAK,CAAC,CAAA;QAExD,IAAI,CAAC,MAAM,IAAI,cAAc,KAAK,SAAS,IAAI,YAAY,KAAK,SAAS,EAAE;YACzE,MAAM,IAAI,KAAK,CAAC,UAAU,KAAK,0BAA0B,CAAC,CAAA;SAC3D;QACD,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,CAAC,wBAAK,CAAC,CAAC,CAAA;QACvC,MAAM,EAAE,gBAAgB,EAAE,aAAa,EAAE,GAAG,MAAM,CAAA;QAClD,MAAM,uBAAuB,GAC3B,gBAAgB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;QAClE,MAAM,oBAAoB,GACxB,aAAa,CAAC,MAAM,GAAG,CAAC;YACtB,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,aAAa,EAAE,EAAE,cAAc,EAAE,CAAC;YAChD,CAAC,CAAC,EAAE,CAAA;QACR,MAAM,SAAS,GAAG,IAAI,GAAG,EAAoB,CAAA;QAE7C,KAAK,MAAM,EAAE,IAAI,GAAG,QAAQ,EAAE,IAAI,GAAG,IAAI,EAAE,OAAO,EAAE,IAAI,IAAI,EAAE;YAC5D,IAAI,OAAO,KAAK,SAAS,EAAE;gBACzB,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAA;aAC7D;YAED,MAAM,CAAC,cAAc,CAAC,CAAA;YACtB,MAAM,WAAW,GAAG,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;YAC5D,SAAS,CAAC,GAAG,CAAC,IAAI,EAAE,WAAW,CAAC,CAAA;YAChC,MAAM,WAAW,CAAA;YACjB,IAAI,oBAAoB,CAAC,MAAM,GAAG,CAAC,EAAE;gBACnC,MAAM,oBAAoB,CAAA;aAC3B;YACD,KAAK,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,CAAA;YACpC,MAAM,CAAC,YAAY,CAAC,CAAA;YACpB,MAAM,uBAAuB,CAAA;SAC9B;QAED,2DAA2D;QAC3D,MAAM,CAAC,cAAc,CAAC,CAAA;QACtB,KAAK,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,WAAW,CAAC,CAAA;QACxC,IAAI,oBAAoB,CAAC,MAAM,GAAG,CAAC,EAAE;YACnC,MAAM,oBAAoB,CAAA;SAC3B;IACH,CAAC;IAED;;;;;;OAMG;IACH,UAAU,CAAC,IAA4B,EAAE,KAAK,GAAG,IAAI,CAAC,SAAS;QAC7D,OAAO,CAAC,GAAG,IAAI,CAAC,mBAAmB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAA;IAC1D,CAAC;IAED;;OAEG;IACH,kBAAkB,CAChB,KAAqC,EACrC,UAAkB;QAElB,MAAM,cAAc,GAClB,OAAO,KAAK,KAAK,QAAQ;YACvB,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC;YAC7B,CAAC,CAAC,IAAI,CAAC,mBAAmB,CAAC,KAAK,CAAC,CAAA;QACrC,IAAI,KAAK,GAAG,CAAC,CAAA;QACb,KAAK,MAAM,MAAM,IAAI,cAAc,EAAE;YACnC,KAAK,IAAI,MAAM,CAAC,MAAM,CAAA;YACtB,IAAI,KAAK,GAAG,UAAU,EAAE;gBACtB,OAAO,KAAK,CAAA;aACb;SACF;QACD,OAAO,KAAK,CAAA;IACd,CAAC;IAED,CAAC,eAAe,CACd,mBAAqC;QAErC,MAAM,oBAAoB,GACxB,IAAI,CAAC,6BAA6B,CAAC,YAAY,CAAC,mBAAmB,CAAC,CAAA;QAEtE,IAAI,MAAM,GAAG,EAAE,CAAA;QAEf,KAAK,MAAM,WAAW,IAAI,oBAAoB,EAAE;YAC9C,MAAM,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;YAE5D,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,IAAA,+CAAkC,EAAC,MAAM,CAAC,EAAE;gBACrE,yEAAyE;gBACzE,uCAAuC;gBACvC,SAAQ;aACT;iBAAM;gBACL,MAAM,MAAM,CAAA;gBACZ,eAAe;gBACf,MAAM,GAAG,EAAE,CAAA;aACZ;SACF;QAED,+CAA+C;QAC/C,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE;YACrB,MAAM,MAAM,CAAA;SACb;IACH,CAAC;IAED,KAAK,CAAC,CAAC,oBAAoB,CACzB,mBAA0C;QAE1C,MAAM,oBAAoB,GACxB,IAAI,CAAC,6BAA6B,CAAC,iBAAiB,CAAC,mBAAmB,CAAC,CAAA;QAE3E,IAAI,MAAM,GAAG,EAAE,CAAA;QAEf,IAAI,KAAK,EAAE,MAAM,WAAW,IAAI,oBAAoB,EAAE;YACpD,MAAM,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;YAE5D,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,IAAA,+CAAkC,EAAC,MAAM,CAAC,EAAE;gBACrE,yEAAyE;gBACzE,uCAAuC;gBACvC,SAAQ;aACT;iBAAM;gBACL,MAAM,MAAM,CAAA;gBACZ,eAAe;gBACf,MAAM,GAAG,EAAE,CAAA;aACZ;SACF;QAED,+CAA+C;QAC/C,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE;YACrB,MAAM,MAAM,CAAA;SACb;IACH,CAAC;IAED,MAAM,CAAC,mBAAqC;QAC1C,OAAO,CAAC,GAAG,IAAI,CAAC,eAAe,CAAC,mBAAmB,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IAChE,CAAC;;AA1RU,kCAAW"}

package/cjs/GptEncoding.test.js

@@ -134,6 +134,60 @@ describe.each(mapping_js_1.encodingNames)('%s', (encodingName) => {
         });
     });
 });
+const chatModelNames = Object.keys(mapping_js_1.chatModelParams);
+const exampleMessages = [
+    {
+        role: 'system',
+        content: 'You are a helpful, pattern-following assistant that translates corporate jargon into plain English.',
+    },
+    {
+        role: 'system',
+        name: 'example_user',
+        content: 'New synergies will help drive top-line growth.',
+    },
+    {
+        role: 'system',
+        name: 'example_assistant',
+        content: 'Things working well together will increase revenue.',
+    },
+    {
+        role: 'system',
+        name: 'example_user',
+        content: "Let's circle back when we have more bandwidth to touch base on opportunities for increased leverage.",
+    },
+    {
+        role: 'system',
+        name: 'example_assistant',
+        content: "Let's talk later when we're less busy about how to do better.",
+    },
+    {
+        role: 'user',
+        content: "This late pivot means we don't have time to boil the ocean for the client deliverable.",
+    },
+];
+describe.each(chatModelNames)('%s', (modelName) => {
+    const encoding = GptEncoding_js_1.GptEncoding.getEncodingApiForModel(modelName, resolveEncoding_js_1.resolveEncoding);
+    const expectedEncodedLength = modelName.startsWith('gpt-3.5-turbo')
+        ? 127
+        : 121;
+    describe('chat functionality', () => {
+        it('encodes a chat correctly', () => {
+            const encoded = encoding.encodeChat(exampleMessages);
+            expect(encoded).toMatchSnapshot();
+            expect(encoded).toHaveLength(expectedEncodedLength);
+            const decoded = encoding.decode(encoded);
+            expect(decoded).toMatchSnapshot();
+        });
+        it('isWithinTokenLimit: false', () => {
+            const isWithinTokenLimit = encoding.isWithinTokenLimit(exampleMessages, 50);
+            expect(isWithinTokenLimit).toBe(false);
+        });
+        it('isWithinTokenLimit: true (number)', () => {
+            const isWithinTokenLimit = encoding.isWithinTokenLimit(exampleMessages, 150);
+            expect(isWithinTokenLimit).toBe(expectedEncodedLength);
+        });
+    });
+});
 function loadTestPlans() {
     const testPlanPath = path_1.default.join(__dirname, '../data/TestPlans.txt');
     const testPlanData = fs_1.default.readFileSync(testPlanPath, 'utf8');

package/cjs/GptEncoding.test.js.map

@@ -1 +1 @@
-{"version":3,"file":"GptEncoding.test.js","sourceRoot":"","sources":["../src/GptEncoding.test.ts"],"names":[],"mappings":";;;;;AAAA,4CAAmB;AACnB,gDAAuB;AACvB,qDAA8C;AAC9C,6CAA+D;AAC/D,6DAAsD;AAEtD,MAAM,aAAa,GAAG;IACpB,KAAK,EAAE,CAAC,GAAG,CAAC;IACZ,GAAG,EAAE,CAAC,GAAG,CAAC;IACV,mBAAmB,EAAE,CAAC,IAAK,EAAE,GAAG,EAAE,GAAG,EAAE,IAAK,CAAC;IAC7C,WAAW,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,KAAM,CAAC;IAC/B,mBAAmB,EAAE,CAAC,KAAM,EAAE,KAAM,EAAE,GAAG,EAAE,GAAG,EAAE,KAAM,EAAE,GAAG,EAAE,GAAG,CAAC;IACjE,uBAAuB,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,EAAE,IAAI,CAAC;IAClE,6CAA6C,EAAE;QAC7C,IAAK,EAAE,KAAM,EAAE,KAAM,EAAE,GAAG,EAAE,KAAM,EAAE,KAAM,EAAE,IAAK,EAAE,IAAK;KACzD;IACD,8CAA8C,EAAE;QAC9C,KAAM,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,IAAK,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE;QAC1E,EAAE,EAAE,EAAE,EAAE,EAAE;KACX;CACF,CAAA;AAED,MAAM,OAAO,GAAG;IACd,WAAW,EAAE;QACX,KAAK,EAAE,CAAC,GAAG,CAAC;QACZ,GAAG,EAAE,CAAC,GAAG,CAAC;QACV,mBAAmB,EAAE,CAAC,IAAK,EAAE,GAAG,EAAE,IAAK,EAAE,IAAK,CAAC;QAC/C,WAAW,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,KAAM,CAAC;QAC/B,mBAAmB,EAAE,CAAC,KAAM,EAAE,KAAM,EAAE,GAAG,EAAE,IAAK,EAAE,KAAM,EAAE,GAAG,EAAE,GAAG,CAAC;QACnE,uBAAuB,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,EAAE,IAAI,CAAC;QAClE,6CAA6C,EAAE;YAC7C,IAAK,EAAE,IAAK,EAAE,GAAG,EAAE,KAAM,EAAE,GAAG,EAAE,IAAK;SACtC;QACD,8CAA8C,EAAE;YAC9C,KAAM,EAAE,EAAE,EAAE,GAAG,EAAE,IAAK,EAAE,GAAG,EAAE,IAAK,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,KAAM,EAAE,EAAE,EAAE,GAAG;YACtE,IAAK,EAAE,KAAM;SACd;KACF;IACD,SAAS,EAAE,aAAa;IACxB,SAAS,EAAE,aAAa;IACxB,SAAS,EAAE,aAAa;CACe,CAAA;AAEzC,mEAAmE;AACnE,MAAM,SAAS,GAAG,aAAa,EAAE,CAAA;AAEjC,QAAQ,CAAC,IAAI,CAAC,0BAAa,CAAC,CAAC,IAAI,EAAE,CAAC,YAA0B,EAAE,EAAE;IAChE,MAAM,QAAQ,GAAG,4BAAW,CAAC,cAAc,CAAC,YAAY,EAAE,oCAAe,CAAC,CAAA;IAE1E,MAAM,EACJ,MAAM,EACN,eAAe,EACf,oBAAoB,EACpB,MAAM,EACN,kBAAkB,GACnB,GAAG,QAAQ,CAAA;IAEZ,MAAM,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC,CAAA;IAEpC,QAAQ,CAAC,qBAAqB,EAAE,GAAG,EAAE;QACnC,EAAE,CAAC,cAAc,EAAE,GAAG,EAAE;YACtB,MAAM,GAAG,GAAG,EAAE,CAAA;YACd,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;YAC/B,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;YACxC,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YAC1C,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QAC5C,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;YACf,MAAM,GAAG,GAAG,GAAG,CAAA;YACf,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;YACzC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;YACxC,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YAC1C,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QAChD,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;YACb,MAAM,GAAG,GAAG,IAAI,CAAA;YAChB,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;YACvC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;QAC1C,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,aAAa,EAAE,GAAG,EAAE;YACrB,MAAM,GAAG,GAAG,mBAAmB,CAAA;YAC/B,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;YACxC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;YACxC,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YAC9C,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAA;QAC7D,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,kBAAkB,EAAE,GAAG,EAAE;YAC1B,MAAM,GAAG,GAAG,aAAa,CAAA;YACzB,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,WAAW,CAAC,CAAA;YAC/C,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;YACxC,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,CAAA;QACpE,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,QAAQ,EAAE,GAAG,EAAE;YAChB,MAAM,GAAG,GAAG,mBAAmB,CAAA;YAC/B,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;YACxC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;YACxC,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YAC9C,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAA;QAC/D,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,qCAAqC,EAAE,GAAG,EAAE;YAC7C,MAAM,GAAG,GAAG,mBAAmB,CAAA;YAC/B,MAAM,SAAS,GAAG,eAAe,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;YAC9C,MAAM,CAAC,uBAAuB,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE;gBAC/C,MAAM,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YAC5C,CAAC,CAAC,CAAA;QACJ,CAAC,CAAC,CAAA;QAEF,KAAK,SAAS,CAAC,CAAC,wBAAwB;YACtC,MAAM,GAAG,GAAG,mBAAmB,CAAA;YAC/B,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,GAAG,CAAC,EAAE;gBAC/B,4CAA4C;gBAC5C,MAAM,MAAM,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;aACnC;QACH,CAAC;QAED,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;YACzD,MAAM,SAAS,GAAG,oBAAoB,CAAC,wBAAwB,EAAE,CAAC,CAAA;YAClE,MAAM,OAAO,GAAG,CAAC,GAAG,MAAM,CAAC,uBAAuB,CAAC,CAAA;YACnD,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,SAAS,EAAE;gBACnC,MAAM,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,CAAA;aACvC;QACH,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,sBAAsB,EAAE,GAAG,EAAE;YAC9B,MAAM,GAAG,GAAG,6CAA6C,CAAA;YAEzD,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;YACxC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;QAC1C,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,kBAAkB,EAAE,GAAG,EAAE;YAC1B,MAAM,GAAG,GAAG,8CAA8C,CAAA;YAC1D,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;YACxC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,aAAa,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;YAC9C,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YACnE,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAA;QAC/D,CAAC,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEF,QAAQ,CAAC,WAAW,EAAE,GAAG,EAAE;QACzB,SAAS,CAAC,YAAY,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,EAAE,EAAE;YACtD,EAAE,CAAC,WAAW,MAAM,EAAE,EAAE,GAAG,EAAE;gBAC3B,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAA;YACzC,CAAC,CAAC,CAAA;YACF,EAAE,CAAC,WAAW,MAAM,EAAE,EAAE,GAAG,EAAE;gBAC3B,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAA;YACzC,CAAC,CAAC,CAAA;QACJ,CAAC,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;AACJ,CAAC,CAAC,CAAA;AAEF,SAAS,aAAa;IACpB,MAAM,YAAY,GAAG,cAAI,CAAC,IAAI,CAAC,SAAS,EAAE,uBAAuB,CAAC,CAAA;IAClE,MAAM,YAAY,GAAG,YAAE,CAAC,YAAY,CAAC,YAAY,EAAE,MAAM,CAAC,CAAA;IAC1D,MAAM,KAAK,GAGP;QACF,WAAW,EAAE,EAAE;QACf,SAAS,EAAE,EAAE;QACb,SAAS,EAAE,EAAE;QACb,SAAS,EAAE,EAAE;KACd,CAAA;IACD,YAAY,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,EAAE;QAC9C,MAAM,CAAC,gBAAgB,EAAE,UAAU,EAAE,WAAW,CAAC,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;QACxE,IAAI,CAAC,gBAAgB,IAAI,CAAC,UAAU,IAAI,CAAC,WAAW;YAAE,OAAM;QAC5D,MAAM,YAAY,GAAG,gBAAgB,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAiB,CAAA;QACpE,KAAK,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC;YACvB,MAAM,EAAE,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;YACxD,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC;SACxD,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IACF,OAAO,KAAK,CAAA;AACd,CAAC"}
+{"version":3,"file":"GptEncoding.test.js","sourceRoot":"","sources":["../src/GptEncoding.test.ts"],"names":[],"mappings":";;;;;AAAA,4CAAmB;AACnB,gDAAuB;AACvB,qDAA8C;AAC9C,6CAKqB;AACrB,6DAAsD;AAEtD,MAAM,aAAa,GAAG;IACpB,KAAK,EAAE,CAAC,GAAG,CAAC;IACZ,GAAG,EAAE,CAAC,GAAG,CAAC;IACV,mBAAmB,EAAE,CAAC,IAAK,EAAE,GAAG,EAAE,GAAG,EAAE,IAAK,CAAC;IAC7C,WAAW,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,KAAM,CAAC;IAC/B,mBAAmB,EAAE,CAAC,KAAM,EAAE,KAAM,EAAE,GAAG,EAAE,GAAG,EAAE,KAAM,EAAE,GAAG,EAAE,GAAG,CAAC;IACjE,uBAAuB,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,EAAE,IAAI,CAAC;IAClE,6CAA6C,EAAE;QAC7C,IAAK,EAAE,KAAM,EAAE,KAAM,EAAE,GAAG,EAAE,KAAM,EAAE,KAAM,EAAE,IAAK,EAAE,IAAK;KACzD;IACD,8CAA8C,EAAE;QAC9C,KAAM,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,IAAK,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE;QAC1E,EAAE,EAAE,EAAE,EAAE,EAAE;KACX;CACF,CAAA;AAED,MAAM,OAAO,GAAG;IACd,WAAW,EAAE;QACX,KAAK,EAAE,CAAC,GAAG,CAAC;QACZ,GAAG,EAAE,CAAC,GAAG,CAAC;QACV,mBAAmB,EAAE,CAAC,IAAK,EAAE,GAAG,EAAE,IAAK,EAAE,IAAK,CAAC;QAC/C,WAAW,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,KAAM,CAAC;QAC/B,mBAAmB,EAAE,CAAC,KAAM,EAAE,KAAM,EAAE,GAAG,EAAE,IAAK,EAAE,KAAM,EAAE,GAAG,EAAE,GAAG,CAAC;QACnE,uBAAuB,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,EAAE,IAAI,CAAC;QAClE,6CAA6C,EAAE;YAC7C,IAAK,EAAE,IAAK,EAAE,GAAG,EAAE,KAAM,EAAE,GAAG,EAAE,IAAK;SACtC;QACD,8CAA8C,EAAE;YAC9C,KAAM,EAAE,EAAE,EAAE,GAAG,EAAE,IAAK,EAAE,GAAG,EAAE,IAAK,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,KAAM,EAAE,EAAE,EAAE,GAAG;YACtE,IAAK,EAAE,KAAM;SACd;KACF;IACD,SAAS,EAAE,aAAa;IACxB,SAAS,EAAE,aAAa;IACxB,SAAS,EAAE,aAAa;CACe,CAAA;AAEzC,mEAAmE;AACnE,MAAM,SAAS,GAAG,aAAa,EAAE,CAAA;AAEjC,QAAQ,CAAC,IAAI,CAAC,0BAAa,CAAC,CAAC,IAAI,EAAE,CAAC,YAA0B,EAAE,EAAE;IAChE,MAAM,QAAQ,GAAG,4BAAW,CAAC,cAAc,CAAC,YAAY,EAAE,oCAAe,CAAC,CAAA;IAE1E,MAAM,EACJ,MAAM,EACN,eAAe,EACf,oBAAoB,EACpB,MAAM,EACN,kBAAkB,GACnB,GAAG,QAAQ,CAAA;IAEZ,MAAM,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC,CAAA;IAEpC,QAAQ,CAAC,qBAAqB,EAAE,GAAG,EAAE;QACnC,EAAE,CAAC,cAAc,EAAE,GAAG,EAAE;YACtB,MAAM,GAAG,GAAG,EAAE,CAAA;YACd,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;YAC/B,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;YACxC,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YAC1C,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QAC5C,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;YACf,MAAM,GAAG,GAAG,GAAG,CAAA;YACf,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;YACzC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;YACxC,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YAC1C,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QAChD,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;YACb,MAAM,GAAG,GAAG,IAAI,CAAA;YAChB,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;YACvC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;QAC1C,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,aAAa,EAAE,GAAG,EAAE;YACrB,MAAM,GAAG,GAAG,mBAAmB,CAAA;YAC/B,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;YACxC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;YACxC,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YAC9C,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAA;QAC7D,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,kBAAkB,EAAE,GAAG,EAAE;YAC1B,MAAM,GAAG,GAAG,aAAa,CAAA;YACzB,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,WAAW,CAAC,CAAA;YAC/C,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;YACxC,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,CAAA;QACpE,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,QAAQ,EAAE,GAAG,EAAE;YAChB,MAAM,GAAG,GAAG,mBAAmB,CAAA;YAC/B,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;YACxC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;YACxC,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YAC9C,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAA;QAC/D,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,qCAAqC,EAAE,GAAG,EAAE;YAC7C,MAAM,GAAG,GAAG,mBAAmB,CAAA;YAC/B,MAAM,SAAS,GAAG,eAAe,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;YAC9C,MAAM,CAAC,uBAAuB,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE;gBAC/C,MAAM,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YAC5C,CAAC,CAAC,CAAA;QACJ,CAAC,CAAC,CAAA;QAEF,KAAK,SAAS,CAAC,CAAC,wBAAwB;YACtC,MAAM,GAAG,GAAG,mBAAmB,CAAA;YAC/B,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,GAAG,CAAC,EAAE;gBAC/B,4CAA4C;gBAC5C,MAAM,MAAM,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;aACnC;QACH,CAAC;QAED,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;YACzD,MAAM,SAAS,GAAG,oBAAoB,CAAC,wBAAwB,EAAE,CAAC,CAAA;YAClE,MAAM,OAAO,GAAG,CAAC,GAAG,MAAM,CAAC,uBAAuB,CAAC,CAAA;YACnD,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,SAAS,EAAE;gBACnC,MAAM,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,CAAA;aACvC;QACH,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,sBAAsB,EAAE,GAAG,EAAE;YAC9B,MAAM,GAAG,GAAG,6CAA6C,CAAA;YAEzD,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;YACxC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;QAC1C,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,kBAAkB,EAAE,GAAG,EAAE;YAC1B,MAAM,GAAG,GAAG,8CAA8C,CAAA;YAC1D,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;YACxC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,aAAa,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;YAC9C,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YACnE,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAA;QAC/D,CAAC,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEF,QAAQ,CAAC,WAAW,EAAE,GAAG,EAAE;QACzB,SAAS,CAAC,YAAY,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,EAAE,EAAE;YACtD,EAAE,CAAC,WAAW,MAAM,EAAE,EAAE,GAAG,EAAE;gBAC3B,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAA;YACzC,CAAC,CAAC,CAAA;YACF,EAAE,CAAC,WAAW,MAAM,EAAE,EAAE,GAAG,EAAE;gBAC3B,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAA;YACzC,CAAC,CAAC,CAAA;QACJ,CAAC,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;AACJ,CAAC,CAAC,CAAA;AAEF,MAAM,cAAc,GAAG,MAAM,CAAC,IAAI,CAAC,4BAAe,CAA6B,CAAA;AAE/E,MAAM,eAAe,GAAG;IACtB;QACE,IAAI,EAAE,QAAQ;QACd,OAAO,EACL,qGAAqG;KACxG;IACD;QACE,IAAI,EAAE,QAAQ;QACd,IAAI,EAAE,cAAc;QACpB,OAAO,EAAE,gDAAgD;KAC1D;IACD;QACE,IAAI,EAAE,QAAQ;QACd,IAAI,EAAE,mBAAmB;QACzB,OAAO,EAAE,qDAAqD;KAC/D;IACD;QACE,IAAI,EAAE,QAAQ;QACd,IAAI,EAAE,cAAc;QACpB,OAAO,EACL,sGAAsG;KACzG;IACD;QACE,IAAI,EAAE,QAAQ;QACd,IAAI,EAAE,mBAAmB;QACzB,OAAO,EAAE,+DAA+D;KACzE;IACD;QACE,IAAI,EAAE,MAAM;QACZ,OAAO,EACL,wFAAwF;KAC3F;CACO,CAAA;AAEV,QAAQ,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,IAAI,EAAE,CAAC,SAAS,EAAE,EAAE;IAChD,MAAM,QAAQ,GAAG,4BAAW,CAAC,sBAAsB,CACjD,SAAS,EACT,oCAAe,CAChB,CAAA;IACD,MAAM,qBAAqB,GAAG,SAAS,CAAC,UAAU,CAAC,eAAe,CAAC;QACjE,CAAC,CAAC,GAAG;QACL,CAAC,CAAC,GAAG,CAAA;IAEP,QAAQ,CAAC,oBAAoB,EAAE,GAAG,EAAE;QAClC,EAAE,CAAC,0BAA0B,EAAE,GAAG,EAAE;YAClC,MAAM,OAAO,GAAG,QAAQ,CAAC,UAAU,CAAC,eAAe,CAAC,CAAA;YACpD,MAAM,CAAC,OAAO,CAAC,CAAC,eAAe,EAAE,CAAA;YACjC,MAAM,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,qBAAqB,CAAC,CAAA;YAEnD,MAAM,OAAO,GAAG,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;YACxC,MAAM,CAAC,OAAO,CAAC,CAAC,eAAe,EAAE,CAAA;QACnC,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,2BAA2B,EAAE,GAAG,EAAE;YACnC,MAAM,kBAAkB,GAAG,QAAQ,CAAC,kBAAkB,CACpD,eAAe,EACf,EAAE,CACH,CAAA;YACD,MAAM,CAAC,kBAAkB,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QACxC,CAAC,CAAC,CAAA;QACF,EAAE,CAAC,mCAAmC,EAAE,GAAG,EAAE;YAC3C,MAAM,kBAAkB,GAAG,QAAQ,CAAC,kBAAkB,CACpD,eAAe,EACf,GAAG,CACJ,CAAA;YACD,MAAM,CAAC,kBAAkB,CAAC,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAA;QACxD,CAAC,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;AACJ,CAAC,CAAC,CAAA;AAEF,SAAS,aAAa;IACpB,MAAM,YAAY,GAAG,cAAI,CAAC,IAAI,CAAC,SAAS,EAAE,uBAAuB,CAAC,CAAA;IAClE,MAAM,YAAY,GAAG,YAAE,CAAC,YAAY,CAAC,YAAY,EAAE,MAAM,CAAC,CAAA;IAC1D,MAAM,KAAK,GAGP;QACF,WAAW,EAAE,EAAE;QACf,SAAS,EAAE,EAAE;QACb,SAAS,EAAE,EAAE;QACb,SAAS,EAAE,EAAE;KACd,CAAA;IACD,YAAY,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,EAAE;QAC9C,MAAM,CAAC,gBAAgB,EAAE,UAAU,EAAE,WAAW,CAAC,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;QACxE,IAAI,CAAC,gBAAgB,IAAI,CAAC,UAAU,IAAI,CAAC,WAAW;YAAE,OAAM;QAC5D,MAAM,YAAY,GAAG,gBAAgB,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAiB,CAAA;QACpE,KAAK,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC;YACvB,MAAM,EAAE,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;YACxD,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC;SACxD,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IACF,OAAO,KAAK,CAAA;AACd,CAAC"}

package/cjs/encoding/cl100k_base.d.ts

@@ -1,6 +1,6 @@
 import { GptEncoding } from '../GptEncoding.js';
 export * from '../specialTokens.js';
 declare const api: GptEncoding;
-declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void, unknown>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, unknown>, encode: (lineToEncode: string, allowedSpecial?: Set<string>, disallowedSpecial?: Set<string>) => number[], encodeGenerator: (lineToEncode: string, allowedSpecial?: Set<string>, disallowedSpecial?: Set<string>) => Generator<number[], number, unknown>, isWithinTokenLimit: (text: string, tokenLimit: number) => number | false;
-export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, };
+declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void, unknown>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, unknown>, encode: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, { allowedSpecial, disallowedSpecial, }?: import("../GptEncoding.js").EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, tokenLimit: number) => number | false, encodeChat: (chat: readonly import("../GptEncoding.js").ChatMessage[], model?: "gpt-4" | "gpt-4-32k" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo" | "gpt-3.5-turbo-0301" | "text-davinci-003" | "text-davinci-002" | "text-davinci-001" | "text-curie-001" | "text-babbage-001" | "text-ada-001" | "davinci" | "curie" | "babbage" | "ada" | "code-davinci-002" | "code-davinci-001" | "code-cushman-002" | "code-cushman-001" | "davinci-codex" | "cushman-codex" | "text-davinci-edit-001" | "code-davinci-edit-001" | "text-embedding-ada-002" | "text-similarity-davinci-001" | "text-similarity-curie-001" | "text-similarity-babbage-001" | "text-similarity-ada-001" | "text-search-davinci-doc-001" | "text-search-curie-doc-001" | "text-search-babbage-doc-001" | "text-search-ada-doc-001" | "code-search-babbage-code-001" | "code-search-ada-code-001" | undefined) => number[], encodeChatGenerator: (chat: Iterable<import("../GptEncoding.js").ChatMessage>, model?: "gpt-4" | "gpt-4-32k" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo" | "gpt-3.5-turbo-0301" | "text-davinci-003" | "text-davinci-002" | "text-davinci-001" | "text-curie-001" | "text-babbage-001" | "text-ada-001" | "davinci" | "curie" | "babbage" | "ada" | "code-davinci-002" | "code-davinci-001" | "code-cushman-002" | "code-cushman-001" | "davinci-codex" | "cushman-codex" | "text-davinci-edit-001" | "code-davinci-edit-001" | "text-embedding-ada-002" | "text-similarity-davinci-001" | "text-similarity-curie-001" | "text-similarity-babbage-001" | "text-similarity-ada-001" | "text-search-davinci-doc-001" | "text-search-curie-doc-001" | "text-search-babbage-doc-001" | "text-search-ada-doc-001" | "code-search-babbage-code-001" | "code-search-ada-code-001" | undefined) => Generator<number[], void, undefined>;
+export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, };
 export default api;

package/cjs/encoding/cl100k_base.js

@@ -17,20 +17,22 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.isWithinTokenLimit = exports.encodeGenerator = exports.encode = exports.decodeGenerator = exports.decodeAsyncGenerator = exports.decode = void 0;
+exports.isWithinTokenLimit = exports.encodeGenerator = exports.encodeChatGenerator = exports.encodeChat = exports.encode = exports.decodeGenerator = exports.decodeAsyncGenerator = exports.decode = void 0;
 /* eslint-disable import/extensions */
 const convertTokenBytePairEncodingFromTuples_js_1 = require("../convertTokenBytePairEncodingFromTuples.js");
 const cl100k_base_js_1 = __importDefault(require("../encodings/cl100k_base.js"));
 const GptEncoding_js_1 = require("../GptEncoding.js");
 __exportStar(require("../specialTokens.js"), exports);
 const api = GptEncoding_js_1.GptEncoding.getEncodingApi('cl100k_base', () => (0, convertTokenBytePairEncodingFromTuples_js_1.convertTokenBytePairEncodingFromTuples)(cl100k_base_js_1.default));
-const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, } = api;
+const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, encodeChat, encodeChatGenerator, } = api;
 exports.decode = decode;
 exports.decodeAsyncGenerator = decodeAsyncGenerator;
 exports.decodeGenerator = decodeGenerator;
 exports.encode = encode;
 exports.encodeGenerator = encodeGenerator;
 exports.isWithinTokenLimit = isWithinTokenLimit;
+exports.encodeChat = encodeChat;
+exports.encodeChatGenerator = encodeChatGenerator;
 // eslint-disable-next-line import/no-default-export
 exports.default = api;
 //# sourceMappingURL=cl100k_base.js.map

package/cjs/encoding/cl100k_base.js.map

@@ -1 +1 @@
-{"version":3,"file":"cl100k_base.js","sourceRoot":"","sources":["../../src/encoding/cl100k_base.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA,sCAAsC;AACtC,4GAAqG;AACrG,iFAAiD;AACjD,sDAA+C;AAE/C,sDAAmC;AAEnC,MAAM,GAAG,GAAG,4BAAW,CAAC,cAAc,CAAC,aAAa,EAAE,GAAG,EAAE,CACzD,IAAA,kFAAsC,EAAC,wBAAO,CAAC,CAChD,CAAA;AACD,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,GACnB,GAAG,GAAG,CAAA;AAEL,wBAAM;AACN,oDAAoB;AACpB,0CAAe;AACf,wBAAM;AACN,0CAAe;AACf,gDAAkB;AAEpB,oDAAoD;AACpD,kBAAe,GAAG,CAAA"}
+{"version":3,"file":"cl100k_base.js","sourceRoot":"","sources":["../../src/encoding/cl100k_base.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA,sCAAsC;AACtC,4GAAqG;AACrG,iFAAiD;AACjD,sDAA+C;AAE/C,sDAAmC;AAEnC,MAAM,GAAG,GAAG,4BAAW,CAAC,cAAc,CAAC,aAAa,EAAE,GAAG,EAAE,CACzD,IAAA,kFAAsC,EAAC,wBAAO,CAAC,CAChD,CAAA;AACD,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,EAClB,UAAU,EACV,mBAAmB,GACpB,GAAG,GAAG,CAAA;AAEL,wBAAM;AACN,oDAAoB;AACpB,0CAAe;AACf,wBAAM;AAGN,0CAAe;AACf,gDAAkB;AAHlB,gCAAU;AACV,kDAAmB;AAIrB,oDAAoD;AACpD,kBAAe,GAAG,CAAA"}

package/cjs/encoding/p50k_base.d.ts

@@ -1,6 +1,6 @@
 import { GptEncoding } from '../GptEncoding.js';
 export * from '../specialTokens.js';
 declare const api: GptEncoding;
-declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void, unknown>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, unknown>, encode: (lineToEncode: string, allowedSpecial?: Set<string>, disallowedSpecial?: Set<string>) => number[], encodeGenerator: (lineToEncode: string, allowedSpecial?: Set<string>, disallowedSpecial?: Set<string>) => Generator<number[], number, unknown>, isWithinTokenLimit: (text: string, tokenLimit: number) => number | false;
+declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void, unknown>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, unknown>, encode: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, { allowedSpecial, disallowedSpecial, }?: import("../GptEncoding.js").EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, tokenLimit: number) => number | false;
 export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, };
 export default api;

package/cjs/encoding/p50k_edit.d.ts

@@ -1,6 +1,6 @@
 import { GptEncoding } from '../GptEncoding.js';
 export * from '../specialTokens.js';
 declare const api: GptEncoding;
-declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void, unknown>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, unknown>, encode: (lineToEncode: string, allowedSpecial?: Set<string>, disallowedSpecial?: Set<string>) => number[], encodeGenerator: (lineToEncode: string, allowedSpecial?: Set<string>, disallowedSpecial?: Set<string>) => Generator<number[], number, unknown>, isWithinTokenLimit: (text: string, tokenLimit: number) => number | false;
+declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void, unknown>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, unknown>, encode: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, { allowedSpecial, disallowedSpecial, }?: import("../GptEncoding.js").EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, tokenLimit: number) => number | false;
 export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, };
 export default api;

package/cjs/encoding/r50k_base.d.ts

@@ -1,6 +1,6 @@
 import { GptEncoding } from '../GptEncoding.js';
 export * from '../specialTokens.js';
 declare const api: GptEncoding;
-declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void, unknown>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, unknown>, encode: (lineToEncode: string, allowedSpecial?: Set<string>, disallowedSpecial?: Set<string>) => number[], encodeGenerator: (lineToEncode: string, allowedSpecial?: Set<string>, disallowedSpecial?: Set<string>) => Generator<number[], number, unknown>, isWithinTokenLimit: (text: string, tokenLimit: number) => number | false;
+declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void, unknown>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, unknown>, encode: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, { allowedSpecial, disallowedSpecial, }?: import("../GptEncoding.js").EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, tokenLimit: number) => number | false;
 export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, };
 export default api;

package/cjs/main.d.ts

@@ -1 +1,2 @@
+export { default } from './encoding/cl100k_base.js';
 export * from './encoding/cl100k_base.js';

package/cjs/main.js

@@ -13,6 +13,13 @@ var __createBinding = (this && this.__createBinding) || (Object.create ? (functi
 var __exportStar = (this && this.__exportStar) || function(m, exports) {
     for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
 };
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = void 0;
+// eslint-disable-next-line no-restricted-exports, import/no-default-export
+var cl100k_base_js_1 = require("./encoding/cl100k_base.js");
+Object.defineProperty(exports, "default", { enumerable: true, get: function () { return __importDefault(cl100k_base_js_1).default; } });
 __exportStar(require("./encoding/cl100k_base.js"), exports);
 //# sourceMappingURL=main.js.map

package/cjs/main.js.map

@@ -1 +1 @@
-{"version":3,"file":"main.js","sourceRoot":"","sources":["../src/main.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,4DAAyC"}
+{"version":3,"file":"main.js","sourceRoot":"","sources":["../src/main.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA,2EAA2E;AAC3E,4DAAmD;AAA1C,0HAAA,OAAO,OAAA;AAChB,4DAAyC"}