gpt-tokenizer 2.0.0 → 2.1.1

package/esm/GptEncoding.d.ts

@@ -1,6 +1,18 @@
 import { type EncodingName, type ModelName } from './mapping.js';
 import { type GetMergeableRanksAsyncFn, type GetMergeableRanksFn } from './modelParams.js';
-export declare const DISALLOW_ALL_SPECIAL_TOKENS = "all";
+export declare const ALL_SPECIAL_TOKENS = "all";
+export interface EncodeOptions {
+    allowedSpecial?: Set<string>;
+    disallowedSpecial?: Set<string>;
+}
+export interface ChatMessage {
+    role?: 'system' | 'user' | 'assistant';
+    name?: string;
+    content: string;
+}
+export interface EncodeChatOptions {
+    primeWithAssistantResponse?: string;
+}
 export declare class GptEncoding {
     static EndOfPrompt: string;
     static EndOfText: string;
@@ -8,6 +20,7 @@ export declare class GptEncoding {
     static FimPrefix: string;
     static FimSuffix: string;
     decoder: TextDecoder;
+    modelName?: ModelName;
     private bytePairEncodingCoreProcessor;
     private specialTokenMapping;
     private constructor();
@@ -15,14 +28,28 @@ export declare class GptEncoding {
     static getEncodingApiForModel(modelName: ModelName, getMergeableRanks: GetMergeableRanksFn): GptEncoding;
     static getEncodingApiAsync(encodingName: EncodingName, getMergeableRanks: GetMergeableRanksAsyncFn): Promise<GptEncoding>;
     static getEncodingApiForModelAsync(modelName: ModelName, getMergeableRanks: GetMergeableRanksAsyncFn): Promise<GptEncoding>;
-    private static getMaxValueFromMap;
-    private static specialTokenRegex;
-    encodeGenerator(lineToEncode: string, allowedSpecial?: Set<string>, disallowedSpecial?: Set<string>): Generator<number[], number>;
-    encode(lineToEncode: string, allowedSpecial?: Set<string>, disallowedSpecial?: Set<string>): number[];
+    encodeGenerator(lineToEncode: string, { allowedSpecial, disallowedSpecial, }?: EncodeOptions): Generator<number[], number, undefined>;
+    encode(lineToEncode: string, encodeOptions?: EncodeOptions): number[];
+    /**
+     * Progressively tokenizes an OpenAI chat.
+     * Warning: gpt-3.5-turbo and gpt-4 chat format may change over time.
+     * Returns tokens assuming the 'gpt-3.5-turbo-0301' / 'gpt-4-0314' format.
+     * Based on OpenAI's guidelines: https://github.com/openai/openai-python/blob/main/chatml.md
+     * Also mentioned in section 6 of this document: https://github.com/openai/openai-cookbook/blob/main/examples/How_to_count_tokens_with_tiktoken.ipynb
+     */
+    encodeChatGenerator(chat: Iterable<ChatMessage>, model?: "gpt-4" | "gpt-4-32k" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo" | "gpt-3.5-turbo-0301" | "text-davinci-003" | "text-davinci-002" | "text-davinci-001" | "text-curie-001" | "text-babbage-001" | "text-ada-001" | "davinci" | "curie" | "babbage" | "ada" | "code-davinci-002" | "code-davinci-001" | "code-cushman-002" | "code-cushman-001" | "davinci-codex" | "cushman-codex" | "text-davinci-edit-001" | "code-davinci-edit-001" | "text-embedding-ada-002" | "text-similarity-davinci-001" | "text-similarity-curie-001" | "text-similarity-babbage-001" | "text-similarity-ada-001" | "text-search-davinci-doc-001" | "text-search-curie-doc-001" | "text-search-babbage-doc-001" | "text-search-ada-doc-001" | "code-search-babbage-code-001" | "code-search-ada-code-001" | undefined): Generator<number[], void, undefined>;
+    /**
+     * Encodes a chat into a single array of tokens.
+     * Warning: gpt-3.5-turbo and gpt-4 chat format may change over time.
+     * Returns tokens assuming the 'gpt-3.5-turbo-0301' / 'gpt-4-0314' format.
+     * Based on OpenAI's guidelines: https://github.com/openai/openai-python/blob/main/chatml.md
+     * Also mentioned in section 6 of this document: https://github.com/openai/openai-cookbook/blob/main/examples/How_to_count_tokens_with_tiktoken.ipynb
+     */
+    encodeChat(chat: readonly ChatMessage[], model?: "gpt-4" | "gpt-4-32k" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo" | "gpt-3.5-turbo-0301" | "text-davinci-003" | "text-davinci-002" | "text-davinci-001" | "text-curie-001" | "text-babbage-001" | "text-ada-001" | "davinci" | "curie" | "babbage" | "ada" | "code-davinci-002" | "code-davinci-001" | "code-cushman-002" | "code-cushman-001" | "davinci-codex" | "cushman-codex" | "text-davinci-edit-001" | "code-davinci-edit-001" | "text-embedding-ada-002" | "text-similarity-davinci-001" | "text-similarity-curie-001" | "text-similarity-babbage-001" | "text-similarity-ada-001" | "text-search-davinci-doc-001" | "text-search-curie-doc-001" | "text-search-babbage-doc-001" | "text-search-ada-doc-001" | "code-search-babbage-code-001" | "code-search-ada-code-001" | undefined): number[];
     /**
      * @returns {false | number} false if token limit is exceeded, otherwise the number of tokens
      */
-    isWithinTokenLimit(text: string, tokenLimit: number): false | number;
+    isWithinTokenLimit(input: string | Iterable<ChatMessage>, tokenLimit: number): false | number;
     decodeGenerator(inputTokensToDecode: Iterable<number>): Generator<string, void>;
     decodeAsyncGenerator(inputTokensToDecode: AsyncIterable<number>): AsyncGenerator<string, void>;
     decode(inputTokensToDecode: Iterable<number>): string;

package/esm/GptEncoding.js

@@ -1,11 +1,11 @@
 /* eslint-disable no-param-reassign */
 import { BytePairEncodingCore } from './BytePairEncodingCore.js';
-import { escapeRegExp } from './escapeRegExp.js';
-import { modelToEncodingMap, } from './mapping.js';
-import { getModelParams, getModelParamsAsync, } from './modelParams.js';
-import { EndOfPrompt, EndOfText, FimMiddle, FimPrefix, FimSuffix, } from './specialTokens.js';
+import { chatModelParams, modelToEncodingMap, } from './mapping.js';
+import { getEncodingParams, getModelParamsAsync, } from './modelParams.js';
+import { EndOfPrompt, EndOfText, FimMiddle, FimPrefix, FimSuffix, ImEnd, ImSep, ImStart, } from './specialTokens.js';
 import { endsWithIncompleteUtfPairSurrogate } from './utfUtil.js';
-export const DISALLOW_ALL_SPECIAL_TOKENS = 'all';
+import { getMaxValueFromMap, getSpecialTokenRegex } from './util.js';
+export const ALL_SPECIAL_TOKENS = 'all';
 class GptEncoding {
     static EndOfPrompt = EndOfPrompt;
     static EndOfText = EndOfText;
@@ -13,10 +13,11 @@ class GptEncoding {
     static FimPrefix = FimPrefix;
     static FimSuffix = FimSuffix;
     decoder = new TextDecoder('utf8');
+    modelName;
     bytePairEncodingCoreProcessor;
     specialTokenMapping;
-    constructor({ tokenSplitRegex, mergeableBytePairRanks, specialTokenMapping, expectedVocabularySize, }) {
-        const maxTokenValue = Math.max(GptEncoding.getMaxValueFromMap(mergeableBytePairRanks), GptEncoding.getMaxValueFromMap(specialTokenMapping));
+    constructor({ tokenSplitRegex, mergeableBytePairRanks, specialTokenMapping, expectedVocabularySize, modelName, }) {
+        const maxTokenValue = Math.max(getMaxValueFromMap(mergeableBytePairRanks), getMaxValueFromMap(specialTokenMapping));
         this.specialTokenMapping = specialTokenMapping;
         if (expectedVocabularySize !== undefined) {
             if (mergeableBytePairRanks.size + specialTokenMapping.size !==
@@ -38,14 +39,16 @@ class GptEncoding {
         this.decodeGenerator = this.decodeGenerator.bind(this);
         this.decodeAsyncGenerator = this.decodeAsyncGenerator.bind(this);
         this.isWithinTokenLimit = this.isWithinTokenLimit.bind(this);
+        this.modelName = modelName;
     }
     static getEncodingApi(encodingName, getMergeableRanks) {
-        const modelParams = getModelParams(encodingName, getMergeableRanks);
+        const modelParams = getEncodingParams(encodingName, getMergeableRanks);
         return new GptEncoding(modelParams);
     }
     static getEncodingApiForModel(modelName, getMergeableRanks) {
         const encodingName = modelToEncodingMap[modelName];
-        return GptEncoding.getEncodingApi(encodingName, getMergeableRanks);
+        const modelParams = getEncodingParams(encodingName, getMergeableRanks);
+        return new GptEncoding({ ...modelParams, modelName });
     }
     static async getEncodingApiAsync(encodingName, getMergeableRanks) {
         const modelParams = await getModelParamsAsync(encodingName, getMergeableRanks);
@@ -53,33 +56,21 @@ class GptEncoding {
     }
     static async getEncodingApiForModelAsync(modelName, getMergeableRanks) {
         const encodingName = modelToEncodingMap[modelName];
-        return GptEncoding.getEncodingApiAsync(encodingName, getMergeableRanks);
-    }
-    static getMaxValueFromMap(map) {
-        let max = 0;
-        map.forEach((val) => {
-            max = Math.max(max, val);
-        });
-        return max;
-    }
-    static specialTokenRegex(tokens) {
-        const escapedTokens = [...tokens].map(escapeRegExp);
-        const inner = escapedTokens.join('|');
-        return new RegExp(`(${inner})`);
+        const modelParams = await getModelParamsAsync(encodingName, getMergeableRanks);
+        return new GptEncoding({ ...modelParams, modelName });
     }
-    encodeGenerator(lineToEncode, allowedSpecial = new Set(), disallowedSpecial = new Set([
-        DISALLOW_ALL_SPECIAL_TOKENS,
-    ])) {
+    encodeGenerator(lineToEncode, { allowedSpecial = new Set(), disallowedSpecial = new Set([ALL_SPECIAL_TOKENS]), } = {}) {
         const specialTokensSet = new Set(this.specialTokenMapping.keys());
-        if (disallowedSpecial.has(DISALLOW_ALL_SPECIAL_TOKENS)) {
+        if (disallowedSpecial.has(ALL_SPECIAL_TOKENS)) {
             disallowedSpecial = new Set(specialTokensSet);
+            allowedSpecial.forEach((val) => disallowedSpecial.delete(val));
             disallowedSpecial.forEach((val) => allowedSpecial.delete(val));
         }
-        if (allowedSpecial.has(DISALLOW_ALL_SPECIAL_TOKENS)) {
+        if (allowedSpecial.has(ALL_SPECIAL_TOKENS)) {
             allowedSpecial = specialTokensSet;
         }
         if (disallowedSpecial.size > 0) {
-            const regexPattern = GptEncoding.specialTokenRegex(disallowedSpecial);
+            const regexPattern = getSpecialTokenRegex(disallowedSpecial);
             const match = lineToEncode.match(regexPattern);
             if (match !== null) {
                 throw new Error(`Disallowed special token found: ${match[0]}`);
@@ -87,18 +78,72 @@ class GptEncoding {
         }
         return this.bytePairEncodingCoreProcessor.encodeNative(lineToEncode, allowedSpecial);
     }
-    encode(lineToEncode, allowedSpecial = new Set(), disallowedSpecial = new Set([
-        DISALLOW_ALL_SPECIAL_TOKENS,
-    ])) {
-        return [
-            ...this.encodeGenerator(lineToEncode, allowedSpecial, disallowedSpecial),
-        ].flat();
+    encode(lineToEncode, encodeOptions = {}) {
+        return [...this.encodeGenerator(lineToEncode, encodeOptions)].flat();
+    }
+    /**
+     * Progressively tokenizes an OpenAI chat.
+     * Warning: gpt-3.5-turbo and gpt-4 chat format may change over time.
+     * Returns tokens assuming the 'gpt-3.5-turbo-0301' / 'gpt-4-0314' format.
+     * Based on OpenAI's guidelines: https://github.com/openai/openai-python/blob/main/chatml.md
+     * Also mentioned in section 6 of this document: https://github.com/openai/openai-cookbook/blob/main/examples/How_to_count_tokens_with_tiktoken.ipynb
+     */
+    *encodeChatGenerator(chat, model = this.modelName) {
+        if (!model) {
+            throw new Error('Model name must be provided either during initialization or passed in to the method.');
+        }
+        const params = chatModelParams[model];
+        const chatStartToken = this.specialTokenMapping.get(ImStart);
+        const chatEndToken = this.specialTokenMapping.get(ImEnd);
+        if (!params || chatStartToken === undefined || chatEndToken === undefined) {
+            throw new Error(`Model '${model}' does not support chat.`);
+        }
+        const allowedSpecial = new Set([ImSep]);
+        const { messageSeparator, roleSeparator } = params;
+        const encodedMessageSeparator = messageSeparator.length > 0 ? this.encode(messageSeparator) : [];
+        const encodedRoleSeparator = roleSeparator.length > 0
+            ? this.encode(roleSeparator, { allowedSpecial })
+            : [];
+        const nameCache = new Map();
+        for (const { role = 'system', name = role, content } of chat) {
+            if (content === undefined) {
+                throw new Error('Content must be defined for all messages.');
+            }
+            yield [chatStartToken];
+            const encodedName = nameCache.get(name) ?? this.encode(name);
+            nameCache.set(name, encodedName);
+            yield encodedName;
+            if (encodedRoleSeparator.length > 0) {
+                yield encodedRoleSeparator;
+            }
+            yield* this.encodeGenerator(content);
+            yield [chatEndToken];
+            yield encodedMessageSeparator;
+        }
+        // every reply is primed with <|start|>assistant<|message|>
+        yield [chatStartToken];
+        yield* this.encodeGenerator('assistant');
+        if (encodedRoleSeparator.length > 0) {
+            yield encodedRoleSeparator;
+        }
+    }
+    /**
+     * Encodes a chat into a single array of tokens.
+     * Warning: gpt-3.5-turbo and gpt-4 chat format may change over time.
+     * Returns tokens assuming the 'gpt-3.5-turbo-0301' / 'gpt-4-0314' format.
+     * Based on OpenAI's guidelines: https://github.com/openai/openai-python/blob/main/chatml.md
+     * Also mentioned in section 6 of this document: https://github.com/openai/openai-cookbook/blob/main/examples/How_to_count_tokens_with_tiktoken.ipynb
+     */
+    encodeChat(chat, model = this.modelName) {
+        return [...this.encodeChatGenerator(chat, model)].flat();
     }
     /**
      * @returns {false | number} false if token limit is exceeded, otherwise the number of tokens
      */
-    isWithinTokenLimit(text, tokenLimit) {
-        const tokenGenerator = this.encodeGenerator(text);
+    isWithinTokenLimit(input, tokenLimit) {
+        const tokenGenerator = typeof input === 'string'
+            ? this.encodeGenerator(input)
+            : this.encodeChatGenerator(input);
         let count = 0;
         for (const tokens of tokenGenerator) {
             count += tokens.length;

package/esm/GptEncoding.js.map

@@ -1 +1 @@
-{"version":3,"file":"GptEncoding.js","sourceRoot":"","sources":["../src/GptEncoding.ts"],"names":[],"mappings":"AAAA,sCAAsC;AACtC,OAAO,EAAE,oBAAoB,EAAE,MAAM,2BAA2B,CAAA;AAChE,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAA;AAChD,OAAO,EAGL,kBAAkB,GACnB,MAAM,cAAc,CAAA;AACrB,OAAO,EAIL,cAAc,EACd,mBAAmB,GACpB,MAAM,kBAAkB,CAAA;AACzB,OAAO,EACL,WAAW,EACX,SAAS,EACT,SAAS,EACT,SAAS,EACT,SAAS,GACV,MAAM,oBAAoB,CAAA;AAC3B,OAAO,EAAE,kCAAkC,EAAE,MAAM,cAAc,CAAA;AAEjE,MAAM,CAAC,MAAM,2BAA2B,GAAG,KAAK,CAAA;AAEhD,MAAa,WAAW;IACtB,MAAM,CAAC,WAAW,GAAG,WAAW,CAAA;IAChC,MAAM,CAAC,SAAS,GAAG,SAAS,CAAA;IAC5B,MAAM,CAAC,SAAS,GAAG,SAAS,CAAA;IAC5B,MAAM,CAAC,SAAS,GAAG,SAAS,CAAA;IAC5B,MAAM,CAAC,SAAS,GAAG,SAAS,CAAA;IAE5B,OAAO,GAAG,IAAI,WAAW,CAAC,MAAM,CAAC,CAAA;IACzB,6BAA6B,CAAsB;IACnD,mBAAmB,CAAqB;IAEhD,YAAoB,EAClB,eAAe,EACf,sBAAsB,EACtB,mBAAmB,EACnB,sBAAsB,GACV;QACZ,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,CAC5B,WAAW,CAAC,kBAAkB,CAAC,sBAAsB,CAAC,EACtD,WAAW,CAAC,kBAAkB,CAAC,mBAAmB,CAAC,CACpD,CAAA;QACD,IAAI,CAAC,mBAAmB,GAAG,mBAAmB,CAAA;QAE9C,IAAI,sBAAsB,KAAK,SAAS,EAAE;YACxC,IACE,sBAAsB,CAAC,IAAI,GAAG,mBAAmB,CAAC,IAAI;gBACtD,sBAAsB,EACtB;gBACA,MAAM,IAAI,KAAK,CACb,sFAAsF,CACvF,CAAA;aACF;YAED,IAAI,aAAa,KAAK,sBAAsB,GAAG,CAAC,EAAE;gBAChD,MAAM,IAAI,KAAK,CACb,gEAAgE,CACjE,CAAA;aACF;SACF;QAED,IAAI,CAAC,6BAA6B,GAAG,IAAI,oBAAoB,CAAC;YAC5D,eAAe,EAAE,sBAAsB;YACvC,mBAAmB,EAAE,mBAAmB;YACxC,eAAe;SAChB,CAAC,CAAA;QAEF,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACpC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACpC,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACtD,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACtD,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAChE,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IAC9D,CAAC;IAED,MAAM,CAAC,cAAc,CACnB,YAA0B,EAC1B,iBAAsC;QAEtC,MAAM,WAAW,GAAG,cAAc,CAAC,YAAY,EAAE,iBAAiB,CAAC,CAAA;QACnE,OAAO,IAAI,WAAW,CAAC,WAAW,CAAC,CAAA;IACrC,CAAC;IAED,MAAM,CAAC,sBAAsB,CAC3B,SAAoB,EACpB,iBAAsC;QAEtC,MAAM,YAAY,GAAG,kBAAkB,CAAC,SAAS,CAAC,CAAA;QAClD,OAAO,WAAW,CAAC,cAAc,CAAC,YAAY,EAAE,iBAAiB,CAAC,CAAA;IACpE,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,mBAAmB,CAC9B,YAA0B,EAC1B,iBAA2C;QAE3C,MAAM,WAAW,GAAG,MAAM,mBAAmB,CAC3C,YAAY,EACZ,iBAAiB,CAClB,CAAA;QACD,OAAO,IAAI,WAAW,CAAC,WAAW,CAAC,CAAA;IACrC,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,2BAA2B,CACtC,SAAoB,EACpB,iBAA2C;QAE3C,MAAM,YAAY,GAAG,kBAAkB,CAAC,SAAS,CAAC,CAAA;QAClD,OAAO,WAAW,CAAC,mBAAmB,CAAC,YAAY,EAAE,iBAAiB,CAAC,CAAA;IACzE,CAAC;IAEO,MAAM,CAAC,kBAAkB,CAAC,GAAyB;QACzD,IAAI,GAAG,GAAG,CAAC,CAAA;QACX,GAAG,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE;YAClB,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,CAAA;QAC1B,CAAC,CAAC,CAAA;QACF,OAAO,GAAG,CAAA;IACZ,CAAC;IAEO,MAAM,CAAC,iBAAiB,CAAC,MAAmB;QAClD,MAAM,aAAa,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,YAAY,CAAC,CAAA;QACnD,MAAM,KAAK,GAAG,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QACrC,OAAO,IAAI,MAAM,CAAC,IAAI,KAAK,GAAG,CAAC,CAAA;IACjC,CAAC;IAED,eAAe,CACb,YAAoB,EACpB,iBAAiB,IAAI,GAAG,EAAU,EAClC,oBAAiC,IAAI,GAAG,CAAS;QAC/C,2BAA2B;KAC5B,CAAC;QAEF,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAAS,IAAI,CAAC,mBAAmB,CAAC,IAAI,EAAE,CAAC,CAAA;QAEzE,IAAI,iBAAiB,CAAC,GAAG,CAAC,2BAA2B,CAAC,EAAE;YACtD,iBAAiB,GAAG,IAAI,GAAG,CAAS,gBAAgB,CAAC,CAAA;YACrD,iBAAiB,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,cAAc,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;SAC/D;QAED,IAAI,cAAc,CAAC,GAAG,CAAC,2BAA2B,CAAC,EAAE;YACnD,cAAc,GAAG,gBAAgB,CAAA;SAClC;QAED,IAAI,iBAAiB,CAAC,IAAI,GAAG,CAAC,EAAE;YAC9B,MAAM,YAAY,GAAG,WAAW,CAAC,iBAAiB,CAAC,iBAAiB,CAAC,CAAA;YACrE,MAAM,KAAK,GAAG,YAAY,CAAC,KAAK,CAAC,YAAY,CAAC,CAAA;YAC9C,IAAI,KAAK,KAAK,IAAI,EAAE;gBAClB,MAAM,IAAI,KAAK,CAAC,mCAAmC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;aAC/D;SACF;QAED,OAAO,IAAI,CAAC,6BAA6B,CAAC,YAAY,CACpD,YAAY,EACZ,cAAc,CACf,CAAA;IACH,CAAC;IAED,MAAM,CACJ,YAAoB,EACpB,iBAAiB,IAAI,GAAG,EAAU,EAClC,oBAAiC,IAAI,GAAG,CAAS;QAC/C,2BAA2B;KAC5B,CAAC;QAEF,OAAO;YACL,GAAG,IAAI,CAAC,eAAe,CAAC,YAAY,EAAE,cAAc,EAAE,iBAAiB,CAAC;SACzE,CAAC,IAAI,EAAE,CAAA;IACV,CAAC;IAED;;OAEG;IACH,kBAAkB,CAAC,IAAY,EAAE,UAAkB;QACjD,MAAM,cAAc,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,CAAA;QACjD,IAAI,KAAK,GAAG,CAAC,CAAA;QACb,KAAK,MAAM,MAAM,IAAI,cAAc,EAAE;YACnC,KAAK,IAAI,MAAM,CAAC,MAAM,CAAA;YACtB,IAAI,KAAK,GAAG,UAAU,EAAE;gBACtB,OAAO,KAAK,CAAA;aACb;SACF;QACD,OAAO,KAAK,CAAA;IACd,CAAC;IAED,CAAC,eAAe,CACd,mBAAqC;QAErC,MAAM,oBAAoB,GACxB,IAAI,CAAC,6BAA6B,CAAC,YAAY,CAAC,mBAAmB,CAAC,CAAA;QAEtE,IAAI,MAAM,GAAG,EAAE,CAAA;QAEf,KAAK,MAAM,WAAW,IAAI,oBAAoB,EAAE;YAC9C,MAAM,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;YAE5D,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,kCAAkC,CAAC,MAAM,CAAC,EAAE;gBACrE,yEAAyE;gBACzE,uCAAuC;gBACvC,SAAQ;aACT;iBAAM;gBACL,MAAM,MAAM,CAAA;gBACZ,eAAe;gBACf,MAAM,GAAG,EAAE,CAAA;aACZ;SACF;QAED,+CAA+C;QAC/C,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE;YACrB,MAAM,MAAM,CAAA;SACb;IACH,CAAC;IAED,KAAK,CAAC,CAAC,oBAAoB,CACzB,mBAA0C;QAE1C,MAAM,oBAAoB,GACxB,IAAI,CAAC,6BAA6B,CAAC,iBAAiB,CAAC,mBAAmB,CAAC,CAAA;QAE3E,IAAI,MAAM,GAAG,EAAE,CAAA;QAEf,IAAI,KAAK,EAAE,MAAM,WAAW,IAAI,oBAAoB,EAAE;YACpD,MAAM,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;YAE5D,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,kCAAkC,CAAC,MAAM,CAAC,EAAE;gBACrE,yEAAyE;gBACzE,uCAAuC;gBACvC,SAAQ;aACT;iBAAM;gBACL,MAAM,MAAM,CAAA;gBACZ,eAAe;gBACf,MAAM,GAAG,EAAE,CAAA;aACZ;SACF;QAED,+CAA+C;QAC/C,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE;YACrB,MAAM,MAAM,CAAA;SACb;IACH,CAAC;IAED,MAAM,CAAC,mBAAqC;QAC1C,OAAO,CAAC,GAAG,IAAI,CAAC,eAAe,CAAC,mBAAmB,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IAChE,CAAC;;SA5NU,WAAW"}
+{"version":3,"file":"GptEncoding.js","sourceRoot":"","sources":["../src/GptEncoding.ts"],"names":[],"mappings":"AAAA,sCAAsC;AACtC,OAAO,EAAE,oBAAoB,EAAE,MAAM,2BAA2B,CAAA;AAChE,OAAO,EAGL,eAAe,EACf,kBAAkB,GACnB,MAAM,cAAc,CAAA;AACrB,OAAO,EAIL,iBAAiB,EACjB,mBAAmB,GACpB,MAAM,kBAAkB,CAAA;AACzB,OAAO,EACL,WAAW,EACX,SAAS,EACT,SAAS,EACT,SAAS,EACT,SAAS,EACT,KAAK,EACL,KAAK,EACL,OAAO,GACR,MAAM,oBAAoB,CAAA;AAC3B,OAAO,EAAE,kCAAkC,EAAE,MAAM,cAAc,CAAA;AACjE,OAAO,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,MAAM,WAAW,CAAA;AAEpE,MAAM,CAAC,MAAM,kBAAkB,GAAG,KAAK,CAAA;AAiBvC,MAAa,WAAW;IACtB,MAAM,CAAC,WAAW,GAAG,WAAW,CAAA;IAChC,MAAM,CAAC,SAAS,GAAG,SAAS,CAAA;IAC5B,MAAM,CAAC,SAAS,GAAG,SAAS,CAAA;IAC5B,MAAM,CAAC,SAAS,GAAG,SAAS,CAAA;IAC5B,MAAM,CAAC,SAAS,GAAG,SAAS,CAAA;IAE5B,OAAO,GAAG,IAAI,WAAW,CAAC,MAAM,CAAC,CAAA;IACjC,SAAS,CAAY;IACb,6BAA6B,CAAsB;IACnD,mBAAmB,CAAqB;IAEhD,YAAoB,EAClB,eAAe,EACf,sBAAsB,EACtB,mBAAmB,EACnB,sBAAsB,EACtB,SAAS,GACM;QACf,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,CAC5B,kBAAkB,CAAC,sBAAsB,CAAC,EAC1C,kBAAkB,CAAC,mBAAmB,CAAC,CACxC,CAAA;QACD,IAAI,CAAC,mBAAmB,GAAG,mBAAmB,CAAA;QAE9C,IAAI,sBAAsB,KAAK,SAAS,EAAE;YACxC,IACE,sBAAsB,CAAC,IAAI,GAAG,mBAAmB,CAAC,IAAI;gBACtD,sBAAsB,EACtB;gBACA,MAAM,IAAI,KAAK,CACb,sFAAsF,CACvF,CAAA;aACF;YAED,IAAI,aAAa,KAAK,sBAAsB,GAAG,CAAC,EAAE;gBAChD,MAAM,IAAI,KAAK,CACb,gEAAgE,CACjE,CAAA;aACF;SACF;QAED,IAAI,CAAC,6BAA6B,GAAG,IAAI,oBAAoB,CAAC;YAC5D,eAAe,EAAE,sBAAsB;YACvC,mBAAmB,EAAE,mBAAmB;YACxC,eAAe;SAChB,CAAC,CAAA;QAEF,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACpC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACpC,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACtD,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACtD,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAChE,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC5D,IAAI,CAAC,SAAS,GAAG,SAAS,CAAA;IAC5B,CAAC;IAED,MAAM,CAAC,cAAc,CACnB,YAA0B,EAC1B,iBAAsC;QAEtC,MAAM,WAAW,GAAG,iBAAiB,CAAC,YAAY,EAAE,iBAAiB,CAAC,CAAA;QACtE,OAAO,IAAI,WAAW,CAAC,WAAW,CAAC,CAAA;IACrC,CAAC;IAED,MAAM,CAAC,sBAAsB,CAC3B,SAAoB,EACpB,iBAAsC;QAEtC,MAAM,YAAY,GAAG,kBAAkB,CAAC,SAAS,CAAC,CAAA;QAClD,MAAM,WAAW,GAAG,iBAAiB,CAAC,YAAY,EAAE,iBAAiB,CAAC,CAAA;QACtE,OAAO,IAAI,WAAW,CAAC,EAAE,GAAG,WAAW,EAAE,SAAS,EAAE,CAAC,CAAA;IACvD,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,mBAAmB,CAC9B,YAA0B,EAC1B,iBAA2C;QAE3C,MAAM,WAAW,GAAG,MAAM,mBAAmB,CAC3C,YAAY,EACZ,iBAAiB,CAClB,CAAA;QACD,OAAO,IAAI,WAAW,CAAC,WAAW,CAAC,CAAA;IACrC,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,2BAA2B,CACtC,SAAoB,EACpB,iBAA2C;QAE3C,MAAM,YAAY,GAAG,kBAAkB,CAAC,SAAS,CAAC,CAAA;QAClD,MAAM,WAAW,GAAG,MAAM,mBAAmB,CAC3C,YAAY,EACZ,iBAAiB,CAClB,CAAA;QACD,OAAO,IAAI,WAAW,CAAC,EAAE,GAAG,WAAW,EAAE,SAAS,EAAE,CAAC,CAAA;IACvD,CAAC;IAED,eAAe,CACb,YAAoB,EACpB,EACE,cAAc,GAAG,IAAI,GAAG,EAAU,EAClC,iBAAiB,GAAG,IAAI,GAAG,CAAS,CAAC,kBAAkB,CAAC,CAAC,MACxC,EAAE;QAErB,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAAS,IAAI,CAAC,mBAAmB,CAAC,IAAI,EAAE,CAAC,CAAA;QAEzE,IAAI,iBAAiB,CAAC,GAAG,CAAC,kBAAkB,CAAC,EAAE;YAC7C,iBAAiB,GAAG,IAAI,GAAG,CAAS,gBAAgB,CAAC,CAAA;YACrD,cAAc,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,iBAAiB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;YAC9D,iBAAiB,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,cAAc,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;SAC/D;QAED,IAAI,cAAc,CAAC,GAAG,CAAC,kBAAkB,CAAC,EAAE;YAC1C,cAAc,GAAG,gBAAgB,CAAA;SAClC;QAED,IAAI,iBAAiB,CAAC,IAAI,GAAG,CAAC,EAAE;YAC9B,MAAM,YAAY,GAAG,oBAAoB,CAAC,iBAAiB,CAAC,CAAA;YAC5D,MAAM,KAAK,GAAG,YAAY,CAAC,KAAK,CAAC,YAAY,CAAC,CAAA;YAC9C,IAAI,KAAK,KAAK,IAAI,EAAE;gBAClB,MAAM,IAAI,KAAK,CAAC,mCAAmC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;aAC/D;SACF;QAED,OAAO,IAAI,CAAC,6BAA6B,CAAC,YAAY,CACpD,YAAY,EACZ,cAAc,CACf,CAAA;IACH,CAAC;IAED,MAAM,CAAC,YAAoB,EAAE,gBAA+B,EAAE;QAC5D,OAAO,CAAC,GAAG,IAAI,CAAC,eAAe,CAAC,YAAY,EAAE,aAAa,CAAC,CAAC,CAAC,IAAI,EAAE,CAAA;IACtE,CAAC;IAED;;;;;;OAMG;IACH,CAAC,mBAAmB,CAClB,IAA2B,EAC3B,KAAK,GAAG,IAAI,CAAC,SAAS;QAEtB,IAAI,CAAC,KAAK,EAAE;YACV,MAAM,IAAI,KAAK,CACb,sFAAsF,CACvF,CAAA;SACF;QACD,MAAM,MAAM,GAAG,eAAe,CAAC,KAAK,CAAC,CAAA;QACrC,MAAM,cAAc,GAAG,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,OAAO,CAAC,CAAA;QAC5D,MAAM,YAAY,GAAG,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;QAExD,IAAI,CAAC,MAAM,IAAI,cAAc,KAAK,SAAS,IAAI,YAAY,KAAK,SAAS,EAAE;YACzE,MAAM,IAAI,KAAK,CAAC,UAAU,KAAK,0BAA0B,CAAC,CAAA;SAC3D;QACD,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAA;QACvC,MAAM,EAAE,gBAAgB,EAAE,aAAa,EAAE,GAAG,MAAM,CAAA;QAClD,MAAM,uBAAuB,GAC3B,gBAAgB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;QAClE,MAAM,oBAAoB,GACxB,aAAa,CAAC,MAAM,GAAG,CAAC;YACtB,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,aAAa,EAAE,EAAE,cAAc,EAAE,CAAC;YAChD,CAAC,CAAC,EAAE,CAAA;QACR,MAAM,SAAS,GAAG,IAAI,GAAG,EAAoB,CAAA;QAE7C,KAAK,MAAM,EAAE,IAAI,GAAG,QAAQ,EAAE,IAAI,GAAG,IAAI,EAAE,OAAO,EAAE,IAAI,IAAI,EAAE;YAC5D,IAAI,OAAO,KAAK,SAAS,EAAE;gBACzB,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAA;aAC7D;YAED,MAAM,CAAC,cAAc,CAAC,CAAA;YACtB,MAAM,WAAW,GAAG,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;YAC5D,SAAS,CAAC,GAAG,CAAC,IAAI,EAAE,WAAW,CAAC,CAAA;YAChC,MAAM,WAAW,CAAA;YACjB,IAAI,oBAAoB,CAAC,MAAM,GAAG,CAAC,EAAE;gBACnC,MAAM,oBAAoB,CAAA;aAC3B;YACD,KAAK,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,CAAA;YACpC,MAAM,CAAC,YAAY,CAAC,CAAA;YACpB,MAAM,uBAAuB,CAAA;SAC9B;QAED,2DAA2D;QAC3D,MAAM,CAAC,cAAc,CAAC,CAAA;QACtB,KAAK,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,WAAW,CAAC,CAAA;QACxC,IAAI,oBAAoB,CAAC,MAAM,GAAG,CAAC,EAAE;YACnC,MAAM,oBAAoB,CAAA;SAC3B;IACH,CAAC;IAED;;;;;;OAMG;IACH,UAAU,CAAC,IAA4B,EAAE,KAAK,GAAG,IAAI,CAAC,SAAS;QAC7D,OAAO,CAAC,GAAG,IAAI,CAAC,mBAAmB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAA;IAC1D,CAAC;IAED;;OAEG;IACH,kBAAkB,CAChB,KAAqC,EACrC,UAAkB;QAElB,MAAM,cAAc,GAClB,OAAO,KAAK,KAAK,QAAQ;YACvB,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC;YAC7B,CAAC,CAAC,IAAI,CAAC,mBAAmB,CAAC,KAAK,CAAC,CAAA;QACrC,IAAI,KAAK,GAAG,CAAC,CAAA;QACb,KAAK,MAAM,MAAM,IAAI,cAAc,EAAE;YACnC,KAAK,IAAI,MAAM,CAAC,MAAM,CAAA;YACtB,IAAI,KAAK,GAAG,UAAU,EAAE;gBACtB,OAAO,KAAK,CAAA;aACb;SACF;QACD,OAAO,KAAK,CAAA;IACd,CAAC;IAED,CAAC,eAAe,CACd,mBAAqC;QAErC,MAAM,oBAAoB,GACxB,IAAI,CAAC,6BAA6B,CAAC,YAAY,CAAC,mBAAmB,CAAC,CAAA;QAEtE,IAAI,MAAM,GAAG,EAAE,CAAA;QAEf,KAAK,MAAM,WAAW,IAAI,oBAAoB,EAAE;YAC9C,MAAM,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;YAE5D,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,kCAAkC,CAAC,MAAM,CAAC,EAAE;gBACrE,yEAAyE;gBACzE,uCAAuC;gBACvC,SAAQ;aACT;iBAAM;gBACL,MAAM,MAAM,CAAA;gBACZ,eAAe;gBACf,MAAM,GAAG,EAAE,CAAA;aACZ;SACF;QAED,+CAA+C;QAC/C,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE;YACrB,MAAM,MAAM,CAAA;SACb;IACH,CAAC;IAED,KAAK,CAAC,CAAC,oBAAoB,CACzB,mBAA0C;QAE1C,MAAM,oBAAoB,GACxB,IAAI,CAAC,6BAA6B,CAAC,iBAAiB,CAAC,mBAAmB,CAAC,CAAA;QAE3E,IAAI,MAAM,GAAG,EAAE,CAAA;QAEf,IAAI,KAAK,EAAE,MAAM,WAAW,IAAI,oBAAoB,EAAE;YACpD,MAAM,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;YAE5D,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,kCAAkC,CAAC,MAAM,CAAC,EAAE;gBACrE,yEAAyE;gBACzE,uCAAuC;gBACvC,SAAQ;aACT;iBAAM;gBACL,MAAM,MAAM,CAAA;gBACZ,eAAe;gBACf,MAAM,GAAG,EAAE,CAAA;aACZ;SACF;QAED,+CAA+C;QAC/C,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE;YACrB,MAAM,MAAM,CAAA;SACb;IACH,CAAC;IAED,MAAM,CAAC,mBAAqC;QAC1C,OAAO,CAAC,GAAG,IAAI,CAAC,eAAe,CAAC,mBAAmB,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IAChE,CAAC;;SA1RU,WAAW"}

package/esm/GptEncoding.test.js

@@ -1,7 +1,7 @@
 import fs from 'fs';
 import path from 'path';
 import { GptEncoding } from './GptEncoding.js';
-import { encodingNames } from './mapping.js';
+import { chatModelParams, encodingNames, } from './mapping.js';
 import { resolveEncoding } from './resolveEncoding.js';
 const sharedResults = {
     space: [220],
@@ -129,6 +129,60 @@ describe.each(encodingNames)('%s', (encodingName) => {
         });
     });
 });
+const chatModelNames = Object.keys(chatModelParams);
+const exampleMessages = [
+    {
+        role: 'system',
+        content: 'You are a helpful, pattern-following assistant that translates corporate jargon into plain English.',
+    },
+    {
+        role: 'system',
+        name: 'example_user',
+        content: 'New synergies will help drive top-line growth.',
+    },
+    {
+        role: 'system',
+        name: 'example_assistant',
+        content: 'Things working well together will increase revenue.',
+    },
+    {
+        role: 'system',
+        name: 'example_user',
+        content: "Let's circle back when we have more bandwidth to touch base on opportunities for increased leverage.",
+    },
+    {
+        role: 'system',
+        name: 'example_assistant',
+        content: "Let's talk later when we're less busy about how to do better.",
+    },
+    {
+        role: 'user',
+        content: "This late pivot means we don't have time to boil the ocean for the client deliverable.",
+    },
+];
+describe.each(chatModelNames)('%s', (modelName) => {
+    const encoding = GptEncoding.getEncodingApiForModel(modelName, resolveEncoding);
+    const expectedEncodedLength = modelName.startsWith('gpt-3.5-turbo')
+        ? 127
+        : 121;
+    describe('chat functionality', () => {
+        it('encodes a chat correctly', () => {
+            const encoded = encoding.encodeChat(exampleMessages);
+            expect(encoded).toMatchSnapshot();
+            expect(encoded).toHaveLength(expectedEncodedLength);
+            const decoded = encoding.decode(encoded);
+            expect(decoded).toMatchSnapshot();
+        });
+        it('isWithinTokenLimit: false', () => {
+            const isWithinTokenLimit = encoding.isWithinTokenLimit(exampleMessages, 50);
+            expect(isWithinTokenLimit).toBe(false);
+        });
+        it('isWithinTokenLimit: true (number)', () => {
+            const isWithinTokenLimit = encoding.isWithinTokenLimit(exampleMessages, 150);
+            expect(isWithinTokenLimit).toBe(expectedEncodedLength);
+        });
+    });
+});
 function loadTestPlans() {
     const testPlanPath = path.join(__dirname, '../data/TestPlans.txt');
     const testPlanData = fs.readFileSync(testPlanPath, 'utf8');

package/esm/GptEncoding.test.js.map

@@ -1 +1 @@
-{"version":3,"file":"GptEncoding.test.js","sourceRoot":"","sources":["../src/GptEncoding.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,IAAI,CAAA;AACnB,OAAO,IAAI,MAAM,MAAM,CAAA;AACvB,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAA;AAC9C,OAAO,EAAqB,aAAa,EAAE,MAAM,cAAc,CAAA;AAC/D,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAA;AAEtD,MAAM,aAAa,GAAG;IACpB,KAAK,EAAE,CAAC,GAAG,CAAC;IACZ,GAAG,EAAE,CAAC,GAAG,CAAC;IACV,mBAAmB,EAAE,CAAC,IAAK,EAAE,GAAG,EAAE,GAAG,EAAE,IAAK,CAAC;IAC7C,WAAW,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,KAAM,CAAC;IAC/B,mBAAmB,EAAE,CAAC,KAAM,EAAE,KAAM,EAAE,GAAG,EAAE,GAAG,EAAE,KAAM,EAAE,GAAG,EAAE,GAAG,CAAC;IACjE,uBAAuB,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,EAAE,IAAI,CAAC;IAClE,6CAA6C,EAAE;QAC7C,IAAK,EAAE,KAAM,EAAE,KAAM,EAAE,GAAG,EAAE,KAAM,EAAE,KAAM,EAAE,IAAK,EAAE,IAAK;KACzD;IACD,8CAA8C,EAAE;QAC9C,KAAM,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,IAAK,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE;QAC1E,EAAE,EAAE,EAAE,EAAE,EAAE;KACX;CACF,CAAA;AAED,MAAM,OAAO,GAAG;IACd,WAAW,EAAE;QACX,KAAK,EAAE,CAAC,GAAG,CAAC;QACZ,GAAG,EAAE,CAAC,GAAG,CAAC;QACV,mBAAmB,EAAE,CAAC,IAAK,EAAE,GAAG,EAAE,IAAK,EAAE,IAAK,CAAC;QAC/C,WAAW,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,KAAM,CAAC;QAC/B,mBAAmB,EAAE,CAAC,KAAM,EAAE,KAAM,EAAE,GAAG,EAAE,IAAK,EAAE,KAAM,EAAE,GAAG,EAAE,GAAG,CAAC;QACnE,uBAAuB,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,EAAE,IAAI,CAAC;QAClE,6CAA6C,EAAE;YAC7C,IAAK,EAAE,IAAK,EAAE,GAAG,EAAE,KAAM,EAAE,GAAG,EAAE,IAAK;SACtC;QACD,8CAA8C,EAAE;YAC9C,KAAM,EAAE,EAAE,EAAE,GAAG,EAAE,IAAK,EAAE,GAAG,EAAE,IAAK,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,KAAM,EAAE,EAAE,EAAE,GAAG;YACtE,IAAK,EAAE,KAAM;SACd;KACF;IACD,SAAS,EAAE,aAAa;IACxB,SAAS,EAAE,aAAa;IACxB,SAAS,EAAE,aAAa;CACe,CAAA;AAEzC,mEAAmE;AACnE,MAAM,SAAS,GAAG,aAAa,EAAE,CAAA;AAEjC,QAAQ,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,IAAI,EAAE,CAAC,YAA0B,EAAE,EAAE;IAChE,MAAM,QAAQ,GAAG,WAAW,CAAC,cAAc,CAAC,YAAY,EAAE,eAAe,CAAC,CAAA;IAE1E,MAAM,EACJ,MAAM,EACN,eAAe,EACf,oBAAoB,EACpB,MAAM,EACN,kBAAkB,GACnB,GAAG,QAAQ,CAAA;IAEZ,MAAM,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC,CAAA;IAEpC,QAAQ,CAAC,qBAAqB,EAAE,GAAG,EAAE;QACnC,EAAE,CAAC,cAAc,EAAE,GAAG,EAAE;YACtB,MAAM,GAAG,GAAG,EAAE,CAAA;YACd,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;YAC/B,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;YACxC,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YAC1C,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QAC5C,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;YACf,MAAM,GAAG,GAAG,GAAG,CAAA;YACf,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;YACzC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;YACxC,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YAC1C,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QAChD,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;YACb,MAAM,GAAG,GAAG,IAAI,CAAA;YAChB,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;YACvC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;QAC1C,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,aAAa,EAAE,GAAG,EAAE;YACrB,MAAM,GAAG,GAAG,mBAAmB,CAAA;YAC/B,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;YACxC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;YACxC,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YAC9C,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAA;QAC7D,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,kBAAkB,EAAE,GAAG,EAAE;YAC1B,MAAM,GAAG,GAAG,aAAa,CAAA;YACzB,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,WAAW,CAAC,CAAA;YAC/C,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;YACxC,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,CAAA;QACpE,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,QAAQ,EAAE,GAAG,EAAE;YAChB,MAAM,GAAG,GAAG,mBAAmB,CAAA;YAC/B,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;YACxC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;YACxC,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YAC9C,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAA;QAC/D,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,qCAAqC,EAAE,GAAG,EAAE;YAC7C,MAAM,GAAG,GAAG,mBAAmB,CAAA;YAC/B,MAAM,SAAS,GAAG,eAAe,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;YAC9C,MAAM,CAAC,uBAAuB,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE;gBAC/C,MAAM,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YAC5C,CAAC,CAAC,CAAA;QACJ,CAAC,CAAC,CAAA;QAEF,KAAK,SAAS,CAAC,CAAC,wBAAwB;YACtC,MAAM,GAAG,GAAG,mBAAmB,CAAA;YAC/B,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,GAAG,CAAC,EAAE;gBAC/B,4CAA4C;gBAC5C,MAAM,MAAM,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;aACnC;QACH,CAAC;QAED,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;YACzD,MAAM,SAAS,GAAG,oBAAoB,CAAC,wBAAwB,EAAE,CAAC,CAAA;YAClE,MAAM,OAAO,GAAG,CAAC,GAAG,MAAM,CAAC,uBAAuB,CAAC,CAAA;YACnD,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,SAAS,EAAE;gBACnC,MAAM,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,CAAA;aACvC;QACH,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,sBAAsB,EAAE,GAAG,EAAE;YAC9B,MAAM,GAAG,GAAG,6CAA6C,CAAA;YAEzD,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;YACxC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;QAC1C,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,kBAAkB,EAAE,GAAG,EAAE;YAC1B,MAAM,GAAG,GAAG,8CAA8C,CAAA;YAC1D,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;YACxC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,aAAa,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;YAC9C,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YACnE,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAA;QAC/D,CAAC,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEF,QAAQ,CAAC,WAAW,EAAE,GAAG,EAAE;QACzB,SAAS,CAAC,YAAY,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,EAAE,EAAE;YACtD,EAAE,CAAC,WAAW,MAAM,EAAE,EAAE,GAAG,EAAE;gBAC3B,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAA;YACzC,CAAC,CAAC,CAAA;YACF,EAAE,CAAC,WAAW,MAAM,EAAE,EAAE,GAAG,EAAE;gBAC3B,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAA;YACzC,CAAC,CAAC,CAAA;QACJ,CAAC,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;AACJ,CAAC,CAAC,CAAA;AAEF,SAAS,aAAa;IACpB,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,uBAAuB,CAAC,CAAA;IAClE,MAAM,YAAY,GAAG,EAAE,CAAC,YAAY,CAAC,YAAY,EAAE,MAAM,CAAC,CAAA;IAC1D,MAAM,KAAK,GAGP;QACF,WAAW,EAAE,EAAE;QACf,SAAS,EAAE,EAAE;QACb,SAAS,EAAE,EAAE;QACb,SAAS,EAAE,EAAE;KACd,CAAA;IACD,YAAY,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,EAAE;QAC9C,MAAM,CAAC,gBAAgB,EAAE,UAAU,EAAE,WAAW,CAAC,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;QACxE,IAAI,CAAC,gBAAgB,IAAI,CAAC,UAAU,IAAI,CAAC,WAAW;YAAE,OAAM;QAC5D,MAAM,YAAY,GAAG,gBAAgB,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAiB,CAAA;QACpE,KAAK,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC;YACvB,MAAM,EAAE,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;YACxD,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC;SACxD,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IACF,OAAO,KAAK,CAAA;AACd,CAAC"}
+{"version":3,"file":"GptEncoding.test.js","sourceRoot":"","sources":["../src/GptEncoding.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,IAAI,CAAA;AACnB,OAAO,IAAI,MAAM,MAAM,CAAA;AACvB,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAA;AAC9C,OAAO,EAGL,eAAe,EACf,aAAa,GACd,MAAM,cAAc,CAAA;AACrB,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAA;AAEtD,MAAM,aAAa,GAAG;IACpB,KAAK,EAAE,CAAC,GAAG,CAAC;IACZ,GAAG,EAAE,CAAC,GAAG,CAAC;IACV,mBAAmB,EAAE,CAAC,IAAK,EAAE,GAAG,EAAE,GAAG,EAAE,IAAK,CAAC;IAC7C,WAAW,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,KAAM,CAAC;IAC/B,mBAAmB,EAAE,CAAC,KAAM,EAAE,KAAM,EAAE,GAAG,EAAE,GAAG,EAAE,KAAM,EAAE,GAAG,EAAE,GAAG,CAAC;IACjE,uBAAuB,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,EAAE,IAAI,CAAC;IAClE,6CAA6C,EAAE;QAC7C,IAAK,EAAE,KAAM,EAAE,KAAM,EAAE,GAAG,EAAE,KAAM,EAAE,KAAM,EAAE,IAAK,EAAE,IAAK;KACzD;IACD,8CAA8C,EAAE;QAC9C,KAAM,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,IAAK,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE;QAC1E,EAAE,EAAE,EAAE,EAAE,EAAE;KACX;CACF,CAAA;AAED,MAAM,OAAO,GAAG;IACd,WAAW,EAAE;QACX,KAAK,EAAE,CAAC,GAAG,CAAC;QACZ,GAAG,EAAE,CAAC,GAAG,CAAC;QACV,mBAAmB,EAAE,CAAC,IAAK,EAAE,GAAG,EAAE,IAAK,EAAE,IAAK,CAAC;QAC/C,WAAW,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,KAAM,CAAC;QAC/B,mBAAmB,EAAE,CAAC,KAAM,EAAE,KAAM,EAAE,GAAG,EAAE,IAAK,EAAE,KAAM,EAAE,GAAG,EAAE,GAAG,CAAC;QACnE,uBAAuB,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,EAAE,IAAI,CAAC;QAClE,6CAA6C,EAAE;YAC7C,IAAK,EAAE,IAAK,EAAE,GAAG,EAAE,KAAM,EAAE,GAAG,EAAE,IAAK;SACtC;QACD,8CAA8C,EAAE;YAC9C,KAAM,EAAE,EAAE,EAAE,GAAG,EAAE,IAAK,EAAE,GAAG,EAAE,IAAK,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,KAAM,EAAE,EAAE,EAAE,GAAG;YACtE,IAAK,EAAE,KAAM;SACd;KACF;IACD,SAAS,EAAE,aAAa;IACxB,SAAS,EAAE,aAAa;IACxB,SAAS,EAAE,aAAa;CACe,CAAA;AAEzC,mEAAmE;AACnE,MAAM,SAAS,GAAG,aAAa,EAAE,CAAA;AAEjC,QAAQ,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,IAAI,EAAE,CAAC,YAA0B,EAAE,EAAE;IAChE,MAAM,QAAQ,GAAG,WAAW,CAAC,cAAc,CAAC,YAAY,EAAE,eAAe,CAAC,CAAA;IAE1E,MAAM,EACJ,MAAM,EACN,eAAe,EACf,oBAAoB,EACpB,MAAM,EACN,kBAAkB,GACnB,GAAG,QAAQ,CAAA;IAEZ,MAAM,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC,CAAA;IAEpC,QAAQ,CAAC,qBAAqB,EAAE,GAAG,EAAE;QACnC,EAAE,CAAC,cAAc,EAAE,GAAG,EAAE;YACtB,MAAM,GAAG,GAAG,EAAE,CAAA;YACd,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;YAC/B,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;YACxC,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YAC1C,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QAC5C,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;YACf,MAAM,GAAG,GAAG,GAAG,CAAA;YACf,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;YACzC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;YACxC,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YAC1C,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QAChD,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;YACb,MAAM,GAAG,GAAG,IAAI,CAAA;YAChB,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;YACvC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;QAC1C,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,aAAa,EAAE,GAAG,EAAE;YACrB,MAAM,GAAG,GAAG,mBAAmB,CAAA;YAC/B,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;YACxC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;YACxC,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YAC9C,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAA;QAC7D,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,kBAAkB,EAAE,GAAG,EAAE;YAC1B,MAAM,GAAG,GAAG,aAAa,CAAA;YACzB,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,WAAW,CAAC,CAAA;YAC/C,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;YACxC,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,CAAA;QACpE,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,QAAQ,EAAE,GAAG,EAAE;YAChB,MAAM,GAAG,GAAG,mBAAmB,CAAA;YAC/B,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;YACxC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;YACxC,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YAC9C,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAA;QAC/D,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,qCAAqC,EAAE,GAAG,EAAE;YAC7C,MAAM,GAAG,GAAG,mBAAmB,CAAA;YAC/B,MAAM,SAAS,GAAG,eAAe,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;YAC9C,MAAM,CAAC,uBAAuB,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE;gBAC/C,MAAM,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YAC5C,CAAC,CAAC,CAAA;QACJ,CAAC,CAAC,CAAA;QAEF,KAAK,SAAS,CAAC,CAAC,wBAAwB;YACtC,MAAM,GAAG,GAAG,mBAAmB,CAAA;YAC/B,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,GAAG,CAAC,EAAE;gBAC/B,4CAA4C;gBAC5C,MAAM,MAAM,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;aACnC;QACH,CAAC;QAED,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;YACzD,MAAM,SAAS,GAAG,oBAAoB,CAAC,wBAAwB,EAAE,CAAC,CAAA;YAClE,MAAM,OAAO,GAAG,CAAC,GAAG,MAAM,CAAC,uBAAuB,CAAC,CAAA;YACnD,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,SAAS,EAAE;gBACnC,MAAM,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,CAAA;aACvC;QACH,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,sBAAsB,EAAE,GAAG,EAAE;YAC9B,MAAM,GAAG,GAAG,6CAA6C,CAAA;YAEzD,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;YACxC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;QAC1C,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,kBAAkB,EAAE,GAAG,EAAE;YAC1B,MAAM,GAAG,GAAG,8CAA8C,CAAA;YAC1D,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;YACxC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,aAAa,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;YAC9C,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YACnE,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAA;QAC/D,CAAC,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEF,QAAQ,CAAC,WAAW,EAAE,GAAG,EAAE;QACzB,SAAS,CAAC,YAAY,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,EAAE,EAAE;YACtD,EAAE,CAAC,WAAW,MAAM,EAAE,EAAE,GAAG,EAAE;gBAC3B,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAA;YACzC,CAAC,CAAC,CAAA;YACF,EAAE,CAAC,WAAW,MAAM,EAAE,EAAE,GAAG,EAAE;gBAC3B,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAA;YACzC,CAAC,CAAC,CAAA;QACJ,CAAC,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;AACJ,CAAC,CAAC,CAAA;AAEF,MAAM,cAAc,GAAG,MAAM,CAAC,IAAI,CAAC,eAAe,CAA6B,CAAA;AAE/E,MAAM,eAAe,GAAG;IACtB;QACE,IAAI,EAAE,QAAQ;QACd,OAAO,EACL,qGAAqG;KACxG;IACD;QACE,IAAI,EAAE,QAAQ;QACd,IAAI,EAAE,cAAc;QACpB,OAAO,EAAE,gDAAgD;KAC1D;IACD;QACE,IAAI,EAAE,QAAQ;QACd,IAAI,EAAE,mBAAmB;QACzB,OAAO,EAAE,qDAAqD;KAC/D;IACD;QACE,IAAI,EAAE,QAAQ;QACd,IAAI,EAAE,cAAc;QACpB,OAAO,EACL,sGAAsG;KACzG;IACD;QACE,IAAI,EAAE,QAAQ;QACd,IAAI,EAAE,mBAAmB;QACzB,OAAO,EAAE,+DAA+D;KACzE;IACD;QACE,IAAI,EAAE,MAAM;QACZ,OAAO,EACL,wFAAwF;KAC3F;CACO,CAAA;AAEV,QAAQ,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,IAAI,EAAE,CAAC,SAAS,EAAE,EAAE;IAChD,MAAM,QAAQ,GAAG,WAAW,CAAC,sBAAsB,CACjD,SAAS,EACT,eAAe,CAChB,CAAA;IACD,MAAM,qBAAqB,GAAG,SAAS,CAAC,UAAU,CAAC,eAAe,CAAC;QACjE,CAAC,CAAC,GAAG;QACL,CAAC,CAAC,GAAG,CAAA;IAEP,QAAQ,CAAC,oBAAoB,EAAE,GAAG,EAAE;QAClC,EAAE,CAAC,0BAA0B,EAAE,GAAG,EAAE;YAClC,MAAM,OAAO,GAAG,QAAQ,CAAC,UAAU,CAAC,eAAe,CAAC,CAAA;YACpD,MAAM,CAAC,OAAO,CAAC,CAAC,eAAe,EAAE,CAAA;YACjC,MAAM,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,qBAAqB,CAAC,CAAA;YAEnD,MAAM,OAAO,GAAG,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;YACxC,MAAM,CAAC,OAAO,CAAC,CAAC,eAAe,EAAE,CAAA;QACnC,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,2BAA2B,EAAE,GAAG,EAAE;YACnC,MAAM,kBAAkB,GAAG,QAAQ,CAAC,kBAAkB,CACpD,eAAe,EACf,EAAE,CACH,CAAA;YACD,MAAM,CAAC,kBAAkB,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QACxC,CAAC,CAAC,CAAA;QACF,EAAE,CAAC,mCAAmC,EAAE,GAAG,EAAE;YAC3C,MAAM,kBAAkB,GAAG,QAAQ,CAAC,kBAAkB,CACpD,eAAe,EACf,GAAG,CACJ,CAAA;YACD,MAAM,CAAC,kBAAkB,CAAC,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAA;QACxD,CAAC,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;AACJ,CAAC,CAAC,CAAA;AAEF,SAAS,aAAa;IACpB,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,uBAAuB,CAAC,CAAA;IAClE,MAAM,YAAY,GAAG,EAAE,CAAC,YAAY,CAAC,YAAY,EAAE,MAAM,CAAC,CAAA;IAC1D,MAAM,KAAK,GAGP;QACF,WAAW,EAAE,EAAE;QACf,SAAS,EAAE,EAAE;QACb,SAAS,EAAE,EAAE;QACb,SAAS,EAAE,EAAE;KACd,CAAA;IACD,YAAY,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,EAAE;QAC9C,MAAM,CAAC,gBAAgB,EAAE,UAAU,EAAE,WAAW,CAAC,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;QACxE,IAAI,CAAC,gBAAgB,IAAI,CAAC,UAAU,IAAI,CAAC,WAAW;YAAE,OAAM;QAC5D,MAAM,YAAY,GAAG,gBAAgB,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAiB,CAAA;QACpE,KAAK,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC;YACvB,MAAM,EAAE,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;YACxD,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC;SACxD,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IACF,OAAO,KAAK,CAAA;AACd,CAAC"}

package/esm/codegen/generateByModel.js

@@ -0,0 +1,19 @@
+import * as fs from 'fs/promises';
+import * as path from 'path';
+import { fileURLToPath } from 'url';
+import { chatModelParams, modelToEncodingMap } from '../mapping.js';
+// eslint-disable-next-line no-underscore-dangle
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const chatModels = Object.keys(chatModelParams);
+const template = await fs.readFile(path.join(__dirname, '../encoding/cl100k_base.ts'), 'utf8');
+await Promise.all(Object.entries(modelToEncodingMap).map(async ([modelName, encoding]) => {
+    await fs.mkdir(path.join(__dirname, '../model'), { recursive: true });
+    const isChatModel = chatModels.includes(modelName);
+    const content = isChatModel
+        ? template.replace(`getEncodingApi('cl100k_base'`, `getEncodingApiForModel('${modelName}'`)
+        : `// eslint-disable-next-line no-restricted-exports, import/no-default-export\nexport { default } from '../encoding/${encoding}.js'\nexport * from '../encoding/${encoding}.js'\n`;
+    await fs.writeFile(path.join(__dirname, `../model/${modelName}.ts`), content);
+    // eslint-disable-next-line no-console
+    console.log(`encoding/${modelName}.ts`);
+}));
+//# sourceMappingURL=generateByModel.js.map

package/esm/codegen/generateByModel.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"generateByModel.js","sourceRoot":"","sources":["../../src/codegen/generateByModel.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,aAAa,CAAA;AACjC,OAAO,KAAK,IAAI,MAAM,MAAM,CAAA;AAC5B,OAAO,EAAE,aAAa,EAAE,MAAM,KAAK,CAAA;AACnC,OAAO,EAAE,eAAe,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAA;AAEnE,gDAAgD;AAChD,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAA;AAE9D,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,eAAe,CAAC,CAAA;AAE/C,MAAM,QAAQ,GAAG,MAAM,EAAE,CAAC,QAAQ,CAChC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,4BAA4B,CAAC,EAClD,MAAM,CACP,CAAA;AAED,MAAM,OAAO,CAAC,GAAG,CACf,MAAM,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,SAAS,EAAE,QAAQ,CAAC,EAAE,EAAE;IACrE,MAAM,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,UAAU,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IACrE,MAAM,WAAW,GAAG,UAAU,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAA;IAElD,MAAM,OAAO,GAAG,WAAW;QACzB,CAAC,CAAC,QAAQ,CAAC,OAAO,CACd,8BAA8B,EAC9B,2BAA2B,SAAS,GAAG,CACxC;QACH,CAAC,CAAC,qHAAqH,QAAQ,oCAAoC,QAAQ,QAAQ,CAAA;IACrL,MAAM,EAAE,CAAC,SAAS,CAChB,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,YAAY,SAAS,KAAK,CAAC,EAChD,OAAO,CACR,CAAA;IAED,sCAAsC;IACtC,OAAO,CAAC,GAAG,CAAC,YAAY,SAAS,KAAK,CAAC,CAAA;AACzC,CAAC,CAAC,CACH,CAAA"}

package/esm/{generateJsModels.js → codegen/generateJsEncodings.js}

@@ -20,7 +20,7 @@ const processFilesInDirectory = async (directoryPath, fn) => {
 };
 // eslint-disable-next-line no-underscore-dangle
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
-await processFilesInDirectory(path.join(__dirname, '../data'), async (filePath) => {
+await processFilesInDirectory(path.join(__dirname, '../../data'), async (filePath) => {
     if (!filePath.endsWith('.tiktoken'))
         return;
     const modelName = path.basename(filePath, '.tiktoken');
@@ -30,9 +30,9 @@ await processFilesInDirectory(path.join(__dirname, '../data'), async (filePath)
         const [token, rank] = x.split(' ');
         return [token, Number.parseInt(rank, 10)];
     });
-    await fs.mkdir(path.join(__dirname, 'models'), { recursive: true });
-    await fs.writeFile(path.join(__dirname, `models/${modelName}.js`), `/* eslint-disable */\n// @ts-nocheck\n// prettier-ignore\n/** @type {[string, number][]} */\nconst encoder = ${JSON.stringify(encoder)};\nexport default encoder;`);
+    await fs.mkdir(path.join(__dirname, '../encodings'), { recursive: true });
+    await fs.writeFile(path.join(__dirname, `../encodings/${modelName}.js`), `/* eslint-disable */\n// @ts-nocheck\n// prettier-ignore\n/** @type {[string, number][]} */\nconst encoder = ${JSON.stringify(encoder)};\nexport default encoder;`);
     // eslint-disable-next-line no-console
     console.log(`Wrote ${modelName}.js`);
 });
-//# sourceMappingURL=generateJsModels.js.map
+//# sourceMappingURL=generateJsEncodings.js.map

package/esm/codegen/generateJsEncodings.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"generateJsEncodings.js","sourceRoot":"","sources":["../../src/codegen/generateJsEncodings.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,aAAa,CAAA;AACjC,OAAO,KAAK,IAAI,MAAM,MAAM,CAAA;AAC5B,OAAO,EAAE,aAAa,EAAE,MAAM,KAAK,CAAA;AAInC,MAAM,uBAAuB,GAAG,KAAK,EACnC,aAAqB,EACrB,EAAoB,EACL,EAAE;IACjB,IAAI;QACF,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,aAAa,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAA;QAEtE,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE;YACxB,uCAAuC;YACvC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;gBAAE,SAAQ;YAC5B,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,IAAI,CAAC,IAAI,CAAC,CAAA;YACpD,4CAA4C;YAC5C,MAAM,EAAE,CAAC,QAAQ,CAAC,CAAA;SACnB;KACF;IAAC,OAAO,KAAK,EAAE;QACd,sCAAsC;QACtC,OAAO,CAAC,KAAK,CAAC,oBAAoB,EAAE,KAAK,CAAC,CAAA;KAC3C;AACH,CAAC,CAAA;AAED,gDAAgD;AAChD,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAA;AAE9D,MAAM,uBAAuB,CAC3B,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,YAAY,CAAC,EAClC,KAAK,EAAE,QAAQ,EAAE,EAAE;IACjB,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,WAAW,CAAC;QAAE,OAAM;IAE3C,MAAM,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAA;IACtD,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAA;IACnD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;IACjC,MAAM,OAAO,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QAC3C,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;QAClC,OAAO,CAAC,KAAK,EAAE,MAAM,CAAC,QAAQ,CAAC,IAAK,EAAE,EAAE,CAAC,CAAC,CAAA;IAC5C,CAAC,CAAC,CAAA;IAEF,MAAM,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,cAAc,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IACzE,MAAM,EAAE,CAAC,SAAS,CAChB,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,gBAAgB,SAAS,KAAK,CAAC,EACpD,gHAAgH,IAAI,CAAC,SAAS,CAC5H,OAAO,CACR,4BAA4B,CAC9B,CAAA;IAED,sCAAsC;IACtC,OAAO,CAAC,GAAG,CAAC,SAAS,SAAS,KAAK,CAAC,CAAA;AACtC,CAAC,CACF,CAAA"}

package/esm/encoding/cl100k_base.d.ts

@@ -1,6 +1,6 @@
 import { GptEncoding } from '../GptEncoding.js';
 export * from '../specialTokens.js';
 declare const api: GptEncoding;
-declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void, unknown>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, unknown>, encode: (lineToEncode: string, allowedSpecial?: Set<string>, disallowedSpecial?: Set<string>) => number[], encodeGenerator: (lineToEncode: string, allowedSpecial?: Set<string>, disallowedSpecial?: Set<string>) => Generator<number[], number, unknown>, isWithinTokenLimit: (text: string, tokenLimit: number) => number | false;
-export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, };
+declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void, unknown>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, unknown>, encode: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, { allowedSpecial, disallowedSpecial, }?: import("../GptEncoding.js").EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, tokenLimit: number) => number | false, encodeChat: (chat: readonly import("../GptEncoding.js").ChatMessage[], model?: "gpt-4" | "gpt-4-32k" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo" | "gpt-3.5-turbo-0301" | "text-davinci-003" | "text-davinci-002" | "text-davinci-001" | "text-curie-001" | "text-babbage-001" | "text-ada-001" | "davinci" | "curie" | "babbage" | "ada" | "code-davinci-002" | "code-davinci-001" | "code-cushman-002" | "code-cushman-001" | "davinci-codex" | "cushman-codex" | "text-davinci-edit-001" | "code-davinci-edit-001" | "text-embedding-ada-002" | "text-similarity-davinci-001" | "text-similarity-curie-001" | "text-similarity-babbage-001" | "text-similarity-ada-001" | "text-search-davinci-doc-001" | "text-search-curie-doc-001" | "text-search-babbage-doc-001" | "text-search-ada-doc-001" | "code-search-babbage-code-001" | "code-search-ada-code-001" | undefined) => number[], encodeChatGenerator: (chat: Iterable<import("../GptEncoding.js").ChatMessage>, model?: "gpt-4" | "gpt-4-32k" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo" | "gpt-3.5-turbo-0301" | "text-davinci-003" | "text-davinci-002" | "text-davinci-001" | "text-curie-001" | "text-babbage-001" | "text-ada-001" | "davinci" | "curie" | "babbage" | "ada" | "code-davinci-002" | "code-davinci-001" | "code-cushman-002" | "code-cushman-001" | "davinci-codex" | "cushman-codex" | "text-davinci-edit-001" | "code-davinci-edit-001" | "text-embedding-ada-002" | "text-similarity-davinci-001" | "text-similarity-curie-001" | "text-similarity-babbage-001" | "text-similarity-ada-001" | "text-search-davinci-doc-001" | "text-search-curie-doc-001" | "text-search-babbage-doc-001" | "text-search-ada-doc-001" | "code-search-babbage-code-001" | "code-search-ada-code-001" | undefined) => Generator<number[], void, undefined>;
+export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, };
 export default api;

package/esm/encoding/cl100k_base.js

@@ -4,8 +4,8 @@ import encoder from '../encodings/cl100k_base.js';
 import { GptEncoding } from '../GptEncoding.js';
 export * from '../specialTokens.js';
 const api = GptEncoding.getEncodingApi('cl100k_base', () => convertTokenBytePairEncodingFromTuples(encoder));
-const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, } = api;
-export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, };
+const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, encodeChat, encodeChatGenerator, } = api;
+export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, };
 // eslint-disable-next-line import/no-default-export
 export default api;
 //# sourceMappingURL=cl100k_base.js.map

package/esm/encoding/cl100k_base.js.map

@@ -1 +1 @@
-{"version":3,"file":"cl100k_base.js","sourceRoot":"","sources":["../../src/encoding/cl100k_base.ts"],"names":[],"mappings":"AAAA,sCAAsC;AACtC,OAAO,EAAE,sCAAsC,EAAE,MAAM,8CAA8C,CAAA;AACrG,OAAO,OAAO,MAAM,6BAA6B,CAAA;AACjD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAA;AAE/C,cAAc,qBAAqB,CAAA;AAEnC,MAAM,GAAG,GAAG,WAAW,CAAC,cAAc,CAAC,aAAa,EAAE,GAAG,EAAE,CACzD,sCAAsC,CAAC,OAAO,CAAC,CAChD,CAAA;AACD,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,GACnB,GAAG,GAAG,CAAA;AACP,OAAO,EACL,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,GACnB,CAAA;AACD,oDAAoD;AACpD,eAAe,GAAG,CAAA"}
+{"version":3,"file":"cl100k_base.js","sourceRoot":"","sources":["../../src/encoding/cl100k_base.ts"],"names":[],"mappings":"AAAA,sCAAsC;AACtC,OAAO,EAAE,sCAAsC,EAAE,MAAM,8CAA8C,CAAA;AACrG,OAAO,OAAO,MAAM,6BAA6B,CAAA;AACjD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAA;AAE/C,cAAc,qBAAqB,CAAA;AAEnC,MAAM,GAAG,GAAG,WAAW,CAAC,cAAc,CAAC,aAAa,EAAE,GAAG,EAAE,CACzD,sCAAsC,CAAC,OAAO,CAAC,CAChD,CAAA;AACD,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,EAClB,UAAU,EACV,mBAAmB,GACpB,GAAG,GAAG,CAAA;AACP,OAAO,EACL,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,UAAU,EACV,mBAAmB,EACnB,eAAe,EACf,kBAAkB,GACnB,CAAA;AACD,oDAAoD;AACpD,eAAe,GAAG,CAAA"}

package/esm/encoding/p50k_base.d.ts

@@ -1,6 +1,6 @@
 import { GptEncoding } from '../GptEncoding.js';
 export * from '../specialTokens.js';
 declare const api: GptEncoding;
-declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void, unknown>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, unknown>, encode: (lineToEncode: string, allowedSpecial?: Set<string>, disallowedSpecial?: Set<string>) => number[], encodeGenerator: (lineToEncode: string, allowedSpecial?: Set<string>, disallowedSpecial?: Set<string>) => Generator<number[], number, unknown>, isWithinTokenLimit: (text: string, tokenLimit: number) => number | false;
+declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void, unknown>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, unknown>, encode: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, { allowedSpecial, disallowedSpecial, }?: import("../GptEncoding.js").EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, tokenLimit: number) => number | false;
 export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, };
 export default api;

package/esm/encoding/p50k_edit.d.ts

@@ -1,6 +1,6 @@
 import { GptEncoding } from '../GptEncoding.js';
 export * from '../specialTokens.js';
 declare const api: GptEncoding;
-declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void, unknown>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, unknown>, encode: (lineToEncode: string, allowedSpecial?: Set<string>, disallowedSpecial?: Set<string>) => number[], encodeGenerator: (lineToEncode: string, allowedSpecial?: Set<string>, disallowedSpecial?: Set<string>) => Generator<number[], number, unknown>, isWithinTokenLimit: (text: string, tokenLimit: number) => number | false;
+declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void, unknown>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, unknown>, encode: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, { allowedSpecial, disallowedSpecial, }?: import("../GptEncoding.js").EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, tokenLimit: number) => number | false;
 export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, };
 export default api;

package/esm/encoding/r50k_base.d.ts

@@ -1,6 +1,6 @@
 import { GptEncoding } from '../GptEncoding.js';
 export * from '../specialTokens.js';
 declare const api: GptEncoding;
-declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void, unknown>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, unknown>, encode: (lineToEncode: string, allowedSpecial?: Set<string>, disallowedSpecial?: Set<string>) => number[], encodeGenerator: (lineToEncode: string, allowedSpecial?: Set<string>, disallowedSpecial?: Set<string>) => Generator<number[], number, unknown>, isWithinTokenLimit: (text: string, tokenLimit: number) => number | false;
+declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void, unknown>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, unknown>, encode: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, { allowedSpecial, disallowedSpecial, }?: import("../GptEncoding.js").EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, tokenLimit: number) => number | false;
 export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, };
 export default api;

package/esm/main.d.ts

@@ -1 +1,2 @@
+export { default } from './encoding/cl100k_base.js';
 export * from './encoding/cl100k_base.js';

package/esm/main.js

@@ -1,2 +1,4 @@
+// eslint-disable-next-line no-restricted-exports, import/no-default-export
+export { default } from './encoding/cl100k_base.js';
 export * from './encoding/cl100k_base.js';
 //# sourceMappingURL=main.js.map

package/esm/main.js.map

@@ -1 +1 @@
-{"version":3,"file":"main.js","sourceRoot":"","sources":["../src/main.ts"],"names":[],"mappings":"AAAA,cAAc,2BAA2B,CAAA"}
+{"version":3,"file":"main.js","sourceRoot":"","sources":["../src/main.ts"],"names":[],"mappings":"AAAA,2EAA2E;AAC3E,OAAO,EAAE,OAAO,EAAE,MAAM,2BAA2B,CAAA;AACnD,cAAc,2BAA2B,CAAA"}

package/esm/mapping.d.ts

@@ -5,7 +5,11 @@ export declare const r50k_base = "r50k_base";
 export declare const encodingNames: readonly ["cl100k_base", "p50k_base", "r50k_base", "p50k_edit"];
 export declare const modelToEncodingMap: {
     readonly 'gpt-4': "cl100k_base";
+    readonly 'gpt-4-32k': "cl100k_base";
+    readonly 'gpt-4-0314': "cl100k_base";
+    readonly 'gpt-4-32k-0314': "cl100k_base";
     readonly 'gpt-3.5-turbo': "cl100k_base";
+    readonly 'gpt-3.5-turbo-0301': "cl100k_base";
     readonly 'text-davinci-003': "p50k_base";
     readonly 'text-davinci-002': "p50k_base";
     readonly 'text-davinci-001': "r50k_base";
@@ -36,5 +40,38 @@ export declare const modelToEncodingMap: {
     readonly 'code-search-babbage-code-001': "r50k_base";
     readonly 'code-search-ada-code-001': "r50k_base";
 };
+export interface ChatParameters {
+    messageSeparator: string;
+    roleSeparator: string;
+}
+declare const internalChatModelParams: {
+    'gpt-3.5-turbo': {
+        messageSeparator: string;
+        roleSeparator: string;
+    };
+    'gpt-3.5-turbo-0301': {
+        messageSeparator: string;
+        roleSeparator: string;
+    };
+    'gpt-4': {
+        messageSeparator: string;
+        roleSeparator: string;
+    };
+    'gpt-4-0314': {
+        messageSeparator: string;
+        roleSeparator: string;
+    };
+    'gpt-4-32k': {
+        messageSeparator: string;
+        roleSeparator: string;
+    };
+    'gpt-4-32k-0314': {
+        messageSeparator: string;
+        roleSeparator: string;
+    };
+};
+export declare const chatModelParams: Partial<Record<ModelName, ChatParameters>>;
 export type ModelName = keyof typeof modelToEncodingMap;
+export type ChatModelName = keyof typeof internalChatModelParams;
 export type EncodingName = (typeof modelToEncodingMap)[ModelName];
+export {};