gpt-tokenizer 2.8.1 → 2.9.0

package/cjs/GptEncoding.d.ts

@@ -1,6 +1,7 @@
 import { ALL_SPECIAL_TOKENS } from './constants.js';
 import { type EncodingName, type ModelName } from './mapping.js';
 import { type GetMergeableRanksFn } from './modelParams.js';
+import { type CostEstimate } from './models.js';
 export interface EncodeOptions {
     /**
      * A list of special tokens that are allowed in the input.
@@ -49,7 +50,7 @@ export declare class GptEncoding {
      * Based on OpenAI's guidelines: https://github.com/openai/openai-python/blob/main/chatml.md
      * Also mentioned in section 6 of this document: https://github.com/openai/openai-cookbook/blob/main/examples/How_to_count_tokens_with_tiktoken.ipynb
      */
-    encodeChatGenerator(chat: Iterable<ChatMessage>, model?: "gpt-4o" | "gpt-3.5-turbo" | "gpt-3.5-turbo-instruct" | "babbage-002" | "davinci-002" | "text-embedding-3-small" | "o1-preview" | "o1-preview-2024-09-12" | "o1-mini" | "o1-mini-2024-09-12" | "chatgpt-4o-latest" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o-mini" | "gpt-4o-mini-2024-07-18" | "gpt-4o-realtime-preview" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-2024-08-06-finetune" | "gpt-4o-mini-2024-07-18-finetune" | "gpt-4o-mini-training" | "gpt-4o-mini-training-2024-07-18" | "gpt-4-turbo" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo-preview" | "gpt-4-0125-preview" | "gpt-4-1106-preview" | "gpt-4" | "gpt-4-0613" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo-finetune" | "gpt-3.5-turbo-16k" | "gpt-4-32k" | "gpt-4-32k-0613" | "gpt-4-vision-preview" | "gpt-4-1106-vision-preview" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo-0613" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-0301" | "text-embedding-3-large" | "text-embedding-ada-002" | "gpt-3.5-turbo-instruct-0914" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-002" | "code-davinci-001" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-002" | "code-cushman-001" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | undefined): Generator<number[], void, undefined>;
+    encodeChatGenerator(chat: Iterable<ChatMessage>, model?: "gpt-4o" | "gpt-3.5-turbo" | "gpt-3.5-turbo-instruct" | "babbage-002" | "davinci-002" | "text-embedding-3-small" | "o1" | "o1-2024-12-17" | "o1-preview" | "o1-preview-2024-09-12" | "o1-mini" | "o1-mini-2024-09-12" | "o3-mini" | "chatgpt-4o-latest" | "gpt-4o-2024-11-20" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o-mini" | "gpt-4o-mini-2024-07-18" | "gpt-4o-realtime-preview" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-realtime-preview-2024-12-17" | "gpt-4o-mini-realtime-preview" | "gpt-4o-mini-realtime-preview-2024-12-17" | "gpt-4o-audio-preview" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-audio-preview-2024-12-17" | "gpt-4o-mini-audio-preview" | "gpt-4o-mini-audio-preview-2024-12-17" | "gpt-4o-2024-08-06-finetune" | "gpt-4o-mini-2024-07-18-finetune" | "gpt-4o-mini-training" | "gpt-4o-mini-training-2024-07-18" | "davinci-002-finetune" | "babbage-002-finetune" | "gpt-4-turbo" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo-preview" | "gpt-4-0125-preview" | "gpt-4-1106-preview" | "gpt-4" | "gpt-4-0613" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo-finetune" | "gpt-3.5-turbo-16k" | "gpt-4-32k" | "gpt-4-32k-0613" | "gpt-4-vision-preview" | "gpt-4-1106-vision-preview" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo-0613" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-0301" | "text-embedding-3-large" | "text-embedding-ada-002" | "gpt-3.5-turbo-instruct-0914" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-002" | "code-davinci-001" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-002" | "code-cushman-001" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | undefined): Generator<number[], void, undefined>;
     /**
      * Encodes a chat into a single array of tokens.
      * Warning: gpt-3.5-turbo and gpt-4 chat format may change over time.
@@ -57,7 +58,7 @@ export declare class GptEncoding {
      * Based on OpenAI's guidelines: https://github.com/openai/openai-python/blob/main/chatml.md
      * Also mentioned in section 6 of this document: https://github.com/openai/openai-cookbook/blob/main/examples/How_to_count_tokens_with_tiktoken.ipynb
      */
-    encodeChat(chat: readonly ChatMessage[], model?: "gpt-4o" | "gpt-3.5-turbo" | "gpt-3.5-turbo-instruct" | "babbage-002" | "davinci-002" | "text-embedding-3-small" | "o1-preview" | "o1-preview-2024-09-12" | "o1-mini" | "o1-mini-2024-09-12" | "chatgpt-4o-latest" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o-mini" | "gpt-4o-mini-2024-07-18" | "gpt-4o-realtime-preview" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-2024-08-06-finetune" | "gpt-4o-mini-2024-07-18-finetune" | "gpt-4o-mini-training" | "gpt-4o-mini-training-2024-07-18" | "gpt-4-turbo" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo-preview" | "gpt-4-0125-preview" | "gpt-4-1106-preview" | "gpt-4" | "gpt-4-0613" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo-finetune" | "gpt-3.5-turbo-16k" | "gpt-4-32k" | "gpt-4-32k-0613" | "gpt-4-vision-preview" | "gpt-4-1106-vision-preview" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo-0613" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-0301" | "text-embedding-3-large" | "text-embedding-ada-002" | "gpt-3.5-turbo-instruct-0914" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-002" | "code-davinci-001" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-002" | "code-cushman-001" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | undefined): number[];
+    encodeChat(chat: readonly ChatMessage[], model?: "gpt-4o" | "gpt-3.5-turbo" | "gpt-3.5-turbo-instruct" | "babbage-002" | "davinci-002" | "text-embedding-3-small" | "o1" | "o1-2024-12-17" | "o1-preview" | "o1-preview-2024-09-12" | "o1-mini" | "o1-mini-2024-09-12" | "o3-mini" | "chatgpt-4o-latest" | "gpt-4o-2024-11-20" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o-mini" | "gpt-4o-mini-2024-07-18" | "gpt-4o-realtime-preview" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-realtime-preview-2024-12-17" | "gpt-4o-mini-realtime-preview" | "gpt-4o-mini-realtime-preview-2024-12-17" | "gpt-4o-audio-preview" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-audio-preview-2024-12-17" | "gpt-4o-mini-audio-preview" | "gpt-4o-mini-audio-preview-2024-12-17" | "gpt-4o-2024-08-06-finetune" | "gpt-4o-mini-2024-07-18-finetune" | "gpt-4o-mini-training" | "gpt-4o-mini-training-2024-07-18" | "davinci-002-finetune" | "babbage-002-finetune" | "gpt-4-turbo" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo-preview" | "gpt-4-0125-preview" | "gpt-4-1106-preview" | "gpt-4" | "gpt-4-0613" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo-finetune" | "gpt-3.5-turbo-16k" | "gpt-4-32k" | "gpt-4-32k-0613" | "gpt-4-vision-preview" | "gpt-4-1106-vision-preview" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo-0613" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-0301" | "text-embedding-3-large" | "text-embedding-ada-002" | "gpt-3.5-turbo-instruct-0914" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-002" | "code-davinci-001" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-002" | "code-cushman-001" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | undefined): number[];
     /**
      * @returns {false | number} false if token limit is exceeded, otherwise the number of tokens
      */
@@ -73,4 +74,12 @@ export declare class GptEncoding {
     decodeGenerator(inputTokensToDecode: Iterable<number>): Generator<string, void, void>;
     decodeAsyncGenerator(inputTokensToDecode: AsyncIterable<number>): AsyncGenerator<string, void>;
     decodeAsync(inputTokensToDecode: AsyncIterable<number>): Promise<string>;
+    /**
+     * Estimates the cost of processing a given token count using the model's pricing.
+     *
+     * @param tokenCount - The number of tokens to estimate cost for
+     * @param modelName - Optional model name to use for cost calculation (defaults to this.modelName)
+     * @returns Cost estimate object with applicable price components (input, output, batchInput, batchOutput)
+     */
+    estimateCost(tokenCount: number, modelName?: "gpt-4o" | "gpt-3.5-turbo" | "gpt-3.5-turbo-instruct" | "babbage-002" | "davinci-002" | "text-embedding-3-small" | "o1" | "o1-2024-12-17" | "o1-preview" | "o1-preview-2024-09-12" | "o1-mini" | "o1-mini-2024-09-12" | "o3-mini" | "chatgpt-4o-latest" | "gpt-4o-2024-11-20" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o-mini" | "gpt-4o-mini-2024-07-18" | "gpt-4o-realtime-preview" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-realtime-preview-2024-12-17" | "gpt-4o-mini-realtime-preview" | "gpt-4o-mini-realtime-preview-2024-12-17" | "gpt-4o-audio-preview" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-audio-preview-2024-12-17" | "gpt-4o-mini-audio-preview" | "gpt-4o-mini-audio-preview-2024-12-17" | "gpt-4o-2024-08-06-finetune" | "gpt-4o-mini-2024-07-18-finetune" | "gpt-4o-mini-training" | "gpt-4o-mini-training-2024-07-18" | "davinci-002-finetune" | "babbage-002-finetune" | "gpt-4-turbo" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo-preview" | "gpt-4-0125-preview" | "gpt-4-1106-preview" | "gpt-4" | "gpt-4-0613" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo-finetune" | "gpt-3.5-turbo-16k" | "gpt-4-32k" | "gpt-4-32k-0613" | "gpt-4-vision-preview" | "gpt-4-1106-vision-preview" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo-0613" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-0301" | "text-embedding-3-large" | "text-embedding-ada-002" | "gpt-3.5-turbo-instruct-0914" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-002" | "code-davinci-001" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-002" | "code-cushman-001" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | undefined): CostEstimate;
 }

package/cjs/GptEncoding.js

@@ -7,6 +7,7 @@ const BytePairEncodingCore_js_1 = require("./BytePairEncodingCore.js");
 const constants_js_1 = require("./constants.js");
 const mapping_js_1 = require("./mapping.js");
 const modelParams_js_1 = require("./modelParams.js");
+const models_js_1 = require("./models.js");
 const specialTokens_js_1 = require("./specialTokens.js");
 const utfUtil_js_1 = require("./utfUtil.js");
 const util_js_1 = require("./util.js");
@@ -57,6 +58,7 @@ class GptEncoding {
         this.countTokens = this.countTokens.bind(this);
         this.setMergeCacheSize = this.setMergeCacheSize.bind(this);
         this.clearMergeCache = this.clearMergeCache.bind(this);
+        this.estimateCost = this.estimateCost.bind(this);
         this.modelName = modelName;
     }
     static getEncodingApi(encodingName, getMergeableRanks) {
@@ -294,6 +296,43 @@ class GptEncoding {
         }
         return buffer;
     }
+    /**
+     * Estimates the cost of processing a given token count using the model's pricing.
+     *
+     * @param tokenCount - The number of tokens to estimate cost for
+     * @param modelName - Optional model name to use for cost calculation (defaults to this.modelName)
+     * @returns Cost estimate object with applicable price components (input, output, batchInput, batchOutput)
+     */
+    estimateCost(tokenCount, modelName = this.modelName) {
+        if (!modelName) {
+            throw new Error('Model name must be provided either during initialization or passed in to the method.');
+        }
+        const model = models_js_1.models[modelName];
+        if (!model) {
+            throw new Error(`Unknown model: ${modelName}`);
+        }
+        if (!model.cost) {
+            throw new Error(`No cost information available for model: ${modelName}`);
+        }
+        const costPerMillion = model.cost;
+        const result = {};
+        // Calculate cost per token and multiply by token count
+        // eslint-disable-next-line no-magic-numbers
+        const millionTokens = tokenCount / 1_000_000;
+        if (costPerMillion.input !== undefined) {
+            result.input = costPerMillion.input * millionTokens;
+        }
+        if (costPerMillion.output !== undefined) {
+            result.output = costPerMillion.output * millionTokens;
+        }
+        if (costPerMillion.batchInput !== undefined) {
+            result.batchInput = costPerMillion.batchInput * millionTokens;
+        }
+        if (costPerMillion.batchOutput !== undefined) {
+            result.batchOutput = costPerMillion.batchOutput * millionTokens;
+        }
+        return result;
+    }
 }
 exports.GptEncoding = GptEncoding;
 //# sourceMappingURL=GptEncoding.js.map

package/cjs/GptEncoding.js.map

@@ -1 +1 @@
-{"version":3,"file":"GptEncoding.js","sourceRoot":"","sources":["../src/GptEncoding.ts"],"names":[],"mappings":";;;AAAA,uDAAuD;AACvD,sCAAsC;AACtC,uEAAyE;AACzE,iDAAmD;AACnD,6CAOqB;AACrB,qDAIyB;AACzB,yDAS2B;AAC3B,6CAAiE;AACjE,uCAAoE;AAgCpE,MAAa,WAAW;IACtB,MAAM,CAAC,WAAW,GAAG,8BAAW,CAAA;IAChC,MAAM,CAAC,SAAS,GAAG,4BAAS,CAAA;IAC5B,MAAM,CAAC,SAAS,GAAG,4BAAS,CAAA;IAC5B,MAAM,CAAC,SAAS,GAAG,4BAAS,CAAA;IAC5B,MAAM,CAAC,SAAS,GAAG,4BAAS,CAAA;IAE5B,SAAS,CAAY;IACb,6BAA6B,CAAsB;IACnD,oBAAoB,CAAqB;IACzC,gBAAgB,CAAa;IAC7B,oBAAoB,CAAQ;IAC5B,yBAAyB,CAAoB;IAE5C,cAAc,CAAQ;IAE/B,YAAoB,EAClB,mBAAmB,EAAE,sBAAsB,EAC3C,oBAAoB,EACpB,sBAAsB,EACtB,SAAS,EACT,GAAG,IAAI,EACQ;QACf,IAAI,CAAC,oBAAoB,GAAG,oBAAoB,CAAA;QAChD,IAAI,CAAC,gBAAgB,GAAG,IAAI,GAAG,CAAS,IAAI,CAAC,oBAAoB,CAAC,IAAI,EAAE,CAAC,CAAA;QACzE,IAAI,CAAC,oBAAoB,GAAG,IAAA,8BAAoB,EAAC,IAAI,CAAC,gBAAgB,CAAC,CAAA;QAEvE,IAAI,CAAC,6BAA6B,GAAG,IAAI,8CAAoB,CAAC;YAC5D,mBAAmB,EAAE,sBAAsB;YAC3C,oBAAoB;YACpB,GAAG,IAAI;SACR,CAAC,CAAA;QACF,IAAI,CAAC,yBAAyB,GAAG,IAAI,CAAC,oBAAoB,EAAE,CAAA;QAE5D,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,CAC5B,sBAAsB,CAAC,MAAM,GAAG,CAAC,EACjC,IAAA,4BAAkB,EAAC,oBAAoB,CAAC,CACzC,CAAA;QAED,IAAI,CAAC,cAAc;YACjB,IAAI,CAAC,6BAA6B,CAAC,0BAA0B;gBAC7D,oBAAoB,CAAC,IAAI,CAAA;QAE3B,IAAI,sBAAsB,KAAK,SAAS,EAAE,CAAC;YACzC,IAAI,IAAI,CAAC,cAAc,KAAK,sBAAsB,EAAE,CAAC;gBACnD,MAAM,IAAI,KAAK,CACb,4FAA4F,CAC7F,CAAA;YACH,CAAC;YAED,IAAI,aAAa,KAAK,sBAAsB,GAAG,CAAC,EAAE,CAAC;gBACjD,MAAM,IAAI,KAAK,CACb,mHAAmH,aAAa,cAC9H,sBAAsB,GAAG,CAC3B,EAAE,CACH,CAAA;YACH,CAAC;QACH,CAAC;QAED,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACpC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACpC,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACtD,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACtD,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAChE,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC9C,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC5D,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC5C,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC9D,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC9C,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC1D,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACtD,IAAI,CAAC,SAAS,GAAG,SAAS,CAAA;IAC5B,CAAC;IAED,MAAM,CAAC,cAAc,CACnB,YAA0B,EAC1B,iBAAsC;QAEtC,MAAM,WAAW,GAAG,IAAA,kCAAiB,EAAC,YAAY,EAAE,iBAAiB,CAAC,CAAA;QACtE,OAAO,IAAI,WAAW,CAAC,WAAW,CAAC,CAAA;IACrC,CAAC;IAED,MAAM,CAAC,sBAAsB,CAC3B,SAAoB,EACpB,iBAAsC;QAEtC,MAAM,YAAY,GAAG,+BAAkB,CAAC,SAAS,CAAC,CAAA;QAClD,MAAM,WAAW,GAAG,IAAA,kCAAiB,EAAC,YAAY,EAAE,iBAAiB,CAAC,CAAA;QACtE,OAAO,IAAI,WAAW,CAAC,EAAE,GAAG,WAAW,EAAE,SAAS,EAAE,CAAC,CAAA;IACvD,CAAC;IAEO,oBAAoB,CAAC,EAC3B,cAAc,EACd,iBAAiB,MACA,EAAE;QACnB,IAAI,YAAgC,CAAA;QAEpC,IACE,cAAc,KAAK,iCAAkB;YACrC,cAAc,EAAE,GAAG,CAAC,iCAAkB,CAAC,EACvC,CAAC;YACD,cAAc,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAA;YAC/C,MAAM,iBAAiB,GAAG,cAAc,CAAA;YACxC,IAAI,iBAAiB,KAAK,iCAAkB,EAAE,CAAC;gBAC7C,MAAM,IAAI,KAAK,CACb,mEAAmE,CACpE,CAAA;YACH,CAAC;YACD,IAAI,OAAO,iBAAiB,KAAK,QAAQ,EAAE,CAAC;gBAC1C,gDAAgD;gBAChD,iBAAiB,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,iBAAiB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;YACnE,CAAC;iBAAM,CAAC;gBACN,yEAAyE;gBACzE,iBAAiB,GAAG,IAAI,GAAG,EAAE,CAAA;YAC/B,CAAC;QACH,CAAC;QAED,IACE,CAAC,iBAAiB;YAClB,iBAAiB,KAAK,iCAAkB;YACxC,iBAAiB,CAAC,GAAG,CAAC,iCAAkB,CAAC,EACzC,CAAC;YACD,gDAAgD;YAChD,iBAAiB,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAA;YAClD,MAAM,oBAAoB,GAAG,iBAAiB,CAAA;YAC9C,IAAI,cAAc,EAAE,IAAI,EAAE,CAAC;gBACzB,cAAc,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,oBAAoB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;gBACjE,2CAA2C;gBAC3C,iBAAiB,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,cAAc,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;gBAC9D,IAAI,iBAAiB,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;oBAC/B,YAAY,GAAG,IAAA,8BAAoB,EAAC,iBAAiB,CAAC,CAAA;gBACxD,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,YAAY,GAAG,IAAI,CAAC,oBAAoB,CAAA;YAC1C,CAAC;QACH,CAAC;QAED,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,CAAA;IACzC,CAAC;IAED,eAAe,CACb,YAAoB,EACpB,aAA6B;QAE7B,MAAM,kBAAkB,GAAG,aAAa;YACtC,CAAC,CAAC,IAAI,CAAC,oBAAoB,CAAC,aAAa,CAAC;YAC1C,CAAC,CAAC,IAAI,CAAC,yBAAyB,CAAA;QAElC,IAAI,kBAAkB,CAAC,YAAY,EAAE,CAAC;YACpC,MAAM,KAAK,GAAG,YAAY,CAAC,KAAK,CAAC,kBAAkB,CAAC,YAAY,CAAC,CAAA;YACjE,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;gBACnB,MAAM,IAAI,KAAK,CAAC,mCAAmC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;YAChE,CAAC;QACH,CAAC;QAED,OAAO,IAAI,CAAC,6BAA6B,CAAC,qBAAqB,CAC7D,YAAY,EACZ,kBAAkB,CAAC,cAAc,CAClC,CAAA;IACH,CAAC;IAED,MAAM,CAAC,YAAoB,EAAE,aAA6B;QACxD,MAAM,kBAAkB,GAAG,aAAa;YACtC,CAAC,CAAC,IAAI,CAAC,oBAAoB,CAAC,aAAa,CAAC;YAC1C,CAAC,CAAC,IAAI,CAAC,yBAAyB,CAAA;QAElC,IAAI,kBAAkB,CAAC,YAAY,EAAE,CAAC;YACpC,MAAM,KAAK,GAAG,YAAY,CAAC,KAAK,CAAC,kBAAkB,CAAC,YAAY,CAAC,CAAA;YACjE,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;gBACnB,MAAM,IAAI,KAAK,CAAC,mCAAmC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;YAChE,CAAC;QACH,CAAC;QAED,OAAO,IAAI,CAAC,6BAA6B,CAAC,YAAY,CACpD,YAAY,EACZ,kBAAkB,CAAC,cAAc,CAClC,CAAA;IACH,CAAC;IAED;;;;;;OAMG;IACH,CAAC,mBAAmB,CAClB,IAA2B,EAC3B,KAAK,GAAG,IAAI,CAAC,SAAS;QAEtB,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,MAAM,IAAI,KAAK,CACb,sFAAsF,CACvF,CAAA;QACH,CAAC;QAED,MAAM,MAAM,GACV,4BAAe,CAAC,KAAsB,CAAC,CAAA;QACzC,MAAM,cAAc,GAAG,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,0BAAO,CAAC,CAAA;QAC7D,MAAM,YAAY,GAAG,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,wBAAK,CAAC,CAAA;QAEzD,IAAI,CAAC,MAAM,IAAI,cAAc,KAAK,SAAS,IAAI,YAAY,KAAK,SAAS,EAAE,CAAC;YAC1E,MAAM,IAAI,KAAK,CAAC,UAAU,KAAK,0BAA0B,CAAC,CAAA;QAC5D,CAAC;QACD,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,CAAC,wBAAK,CAAC,CAAC,CAAA;QACvC,MAAM,EAAE,gBAAgB,EAAE,aAAa,EAAE,GAAG,MAAM,CAAA;QAClD,MAAM,uBAAuB,GAC3B,gBAAgB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;QAClE,MAAM,oBAAoB,GACxB,aAAa,CAAC,MAAM,GAAG,CAAC;YACtB,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,aAAa,EAAE,EAAE,cAAc,EAAE,CAAC;YAChD,CAAC,CAAC,EAAE,CAAA;QACR,MAAM,SAAS,GAAG,IAAI,GAAG,EAAoB,CAAA;QAE7C,KAAK,MAAM,EAAE,IAAI,GAAG,QAAQ,EAAE,IAAI,GAAG,IAAI,EAAE,OAAO,EAAE,IAAI,IAAI,EAAE,CAAC;YAC7D,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;gBAC1B,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAA;YAC9D,CAAC;YAED,MAAM,CAAC,cAAc,CAAC,CAAA;YACtB,MAAM,WAAW,GAAG,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;YAC5D,SAAS,CAAC,GAAG,CAAC,IAAI,EAAE,WAAW,CAAC,CAAA;YAChC,MAAM,WAAW,CAAA;YACjB,IAAI,oBAAoB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACpC,MAAM,oBAAoB,CAAA;YAC5B,CAAC;YACD,KAAK,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,CAAA;YACpC,MAAM,CAAC,YAAY,CAAC,CAAA;YACpB,MAAM,uBAAuB,CAAA;QAC/B,CAAC;QAED,2DAA2D;QAC3D,MAAM,CAAC,cAAc,CAAC,CAAA;QACtB,KAAK,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,WAAW,CAAC,CAAA;QACxC,IAAI,oBAAoB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACpC,MAAM,oBAAoB,CAAA;QAC5B,CAAC;IACH,CAAC;IAED;;;;;;OAMG;IACH,UAAU,CAAC,IAA4B,EAAE,KAAK,GAAG,IAAI,CAAC,SAAS;QAC7D,OAAO,CAAC,GAAG,IAAI,CAAC,mBAAmB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAA;IAC1D,CAAC;IAED;;OAEG;IACH,kBAAkB,CAChB,KAAqC,EACrC,UAAkB;QAElB,MAAM,cAAc,GAClB,OAAO,KAAK,KAAK,QAAQ;YACvB,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC;YAC7B,CAAC,CAAC,IAAI,CAAC,mBAAmB,CAAC,KAAK,CAAC,CAAA;QACrC,IAAI,KAAK,GAAG,CAAC,CAAA;QACb,KAAK,MAAM,MAAM,IAAI,cAAc,EAAE,CAAC;YACpC,KAAK,IAAI,MAAM,CAAC,MAAM,CAAA;YACtB,IAAI,KAAK,GAAG,UAAU,EAAE,CAAC;gBACvB,OAAO,KAAK,CAAA;YACd,CAAC;QACH,CAAC;QACD,OAAO,KAAK,CAAA;IACd,CAAC;IAED;;;OAGG;IACH,WAAW,CACT,KAAqC,EACrC,aAA6B;QAE7B,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YAC9B,MAAM,kBAAkB,GAAG,aAAa;gBACtC,CAAC,CAAC,IAAI,CAAC,oBAAoB,CAAC,aAAa,CAAC;gBAC1C,CAAC,CAAC,IAAI,CAAC,yBAAyB,CAAA;YAElC,IAAI,kBAAkB,CAAC,YAAY,EAAE,CAAC;gBACpC,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,kBAAkB,CAAC,YAAY,CAAC,CAAA;gBAC1D,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;oBACnB,MAAM,IAAI,KAAK,CAAC,mCAAmC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;gBAChE,CAAC;YACH,CAAC;YAED,OAAO,IAAI,CAAC,6BAA6B,CAAC,WAAW,CACnD,KAAK,EACL,kBAAkB,CAAC,cAAc,CAClC,CAAA;QACH,CAAC;QAED,MAAM,cAAc,GAAG,IAAI,CAAC,mBAAmB,CAAC,KAAK,CAAC,CAAA;QACtD,IAAI,KAAK,GAAG,CAAC,CAAA;QACb,KAAK,MAAM,MAAM,IAAI,cAAc,EAAE,CAAC;YACpC,KAAK,IAAI,MAAM,CAAC,MAAM,CAAA;QACxB,CAAC;QACD,OAAO,KAAK,CAAA;IACd,CAAC;IAED,iBAAiB,CAAC,IAAY;QAC5B,IAAI,CAAC,6BAA6B,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAA;IAC5D,CAAC;IAED,eAAe;QACb,IAAI,CAAC,6BAA6B,CAAC,eAAe,EAAE,CAAA;IACtD,CAAC;IAED,MAAM,CAAC,mBAAqC;QAC1C,OAAO,IAAI,CAAC,6BAA6B,CAAC,YAAY,CAAC,mBAAmB,CAAC,CAAA;IAC7E,CAAC;IAED,CAAC,eAAe,CACd,mBAAqC;QAErC,MAAM,oBAAoB,GACxB,IAAI,CAAC,6BAA6B,CAAC,qBAAqB,CACtD,mBAAmB,CACpB,CAAA;QAEH,IAAI,MAAM,GAAG,EAAE,CAAA;QAEf,KAAK,MAAM,WAAW,IAAI,oBAAoB,EAAE,CAAC;YAC/C,MAAM;gBACJ,OAAO,WAAW,KAAK,QAAQ;oBAC7B,CAAC,CAAC,WAAW;oBACb,CAAC,CAAC,iCAAO,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;YAEnD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,IAAA,+CAAkC,EAAC,MAAM,CAAC,EAAE,CAAC;gBACtE,yEAAyE;gBACzE,uCAAuC;gBACvC,SAAQ;YACV,CAAC;iBAAM,CAAC;gBACN,MAAM,MAAM,CAAA;gBACZ,eAAe;gBACf,MAAM,GAAG,EAAE,CAAA;YACb,CAAC;QACH,CAAC;QAED,+CAA+C;QAC/C,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtB,MAAM,MAAM,CAAA;QACd,CAAC;IACH,CAAC;IAED,KAAK,CAAC,CAAC,oBAAoB,CACzB,mBAA0C;QAE1C,MAAM,oBAAoB,GACxB,IAAI,CAAC,6BAA6B,CAAC,yBAAyB,CAC1D,mBAAmB,CACpB,CAAA;QAEH,IAAI,MAAM,GAAG,EAAE,CAAA;QAEf,IAAI,KAAK,EAAE,MAAM,WAAW,IAAI,oBAAoB,EAAE,CAAC;YACrD,MAAM;gBACJ,OAAO,WAAW,KAAK,QAAQ;oBAC7B,CAAC,CAAC,WAAW;oBACb,CAAC,CAAC,iCAAO,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;YAEnD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,IAAA,+CAAkC,EAAC,MAAM,CAAC,EAAE,CAAC;gBACtE,yEAAyE;gBACzE,uCAAuC;gBACvC,SAAQ;YACV,CAAC;iBAAM,CAAC;gBACN,MAAM,MAAM,CAAA;gBACZ,eAAe;gBACf,MAAM,GAAG,EAAE,CAAA;YACb,CAAC;QACH,CAAC;QAED,+CAA+C;QAC/C,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtB,MAAM,MAAM,CAAA;QACd,CAAC;IACH,CAAC;IAED,KAAK,CAAC,WAAW,CACf,mBAA0C;QAE1C,MAAM,oBAAoB,GACxB,IAAI,CAAC,6BAA6B,CAAC,yBAAyB,CAC1D,mBAAmB,CACpB,CAAA;QAEH,IAAI,MAAM,GAAG,EAAE,CAAA;QAEf,IAAI,KAAK,EAAE,MAAM,WAAW,IAAI,oBAAoB,EAAE,CAAC;YACrD,MAAM;gBACJ,OAAO,WAAW,KAAK,QAAQ;oBAC7B,CAAC,CAAC,WAAW;oBACb,CAAC,CAAC,iCAAO,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;QACrD,CAAC;QAED,OAAO,MAAM,CAAA;IACf,CAAC;;AAjZH,kCAkZC"}
+{"version":3,"file":"GptEncoding.js","sourceRoot":"","sources":["../src/GptEncoding.ts"],"names":[],"mappings":";;;AAAA,uDAAuD;AACvD,sCAAsC;AACtC,uEAAyE;AACzE,iDAAmD;AACnD,6CAOqB;AACrB,qDAIyB;AACzB,2CAAuD;AACvD,yDAS2B;AAC3B,6CAAiE;AACjE,uCAAoE;AAgCpE,MAAa,WAAW;IACtB,MAAM,CAAC,WAAW,GAAG,8BAAW,CAAA;IAChC,MAAM,CAAC,SAAS,GAAG,4BAAS,CAAA;IAC5B,MAAM,CAAC,SAAS,GAAG,4BAAS,CAAA;IAC5B,MAAM,CAAC,SAAS,GAAG,4BAAS,CAAA;IAC5B,MAAM,CAAC,SAAS,GAAG,4BAAS,CAAA;IAE5B,SAAS,CAAY;IACb,6BAA6B,CAAsB;IACnD,oBAAoB,CAAqB;IACzC,gBAAgB,CAAa;IAC7B,oBAAoB,CAAQ;IAC5B,yBAAyB,CAAoB;IAE5C,cAAc,CAAQ;IAE/B,YAAoB,EAClB,mBAAmB,EAAE,sBAAsB,EAC3C,oBAAoB,EACpB,sBAAsB,EACtB,SAAS,EACT,GAAG,IAAI,EACQ;QACf,IAAI,CAAC,oBAAoB,GAAG,oBAAoB,CAAA;QAChD,IAAI,CAAC,gBAAgB,GAAG,IAAI,GAAG,CAAS,IAAI,CAAC,oBAAoB,CAAC,IAAI,EAAE,CAAC,CAAA;QACzE,IAAI,CAAC,oBAAoB,GAAG,IAAA,8BAAoB,EAAC,IAAI,CAAC,gBAAgB,CAAC,CAAA;QAEvE,IAAI,CAAC,6BAA6B,GAAG,IAAI,8CAAoB,CAAC;YAC5D,mBAAmB,EAAE,sBAAsB;YAC3C,oBAAoB;YACpB,GAAG,IAAI;SACR,CAAC,CAAA;QACF,IAAI,CAAC,yBAAyB,GAAG,IAAI,CAAC,oBAAoB,EAAE,CAAA;QAE5D,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,CAC5B,sBAAsB,CAAC,MAAM,GAAG,CAAC,EACjC,IAAA,4BAAkB,EAAC,oBAAoB,CAAC,CACzC,CAAA;QAED,IAAI,CAAC,cAAc;YACjB,IAAI,CAAC,6BAA6B,CAAC,0BAA0B;gBAC7D,oBAAoB,CAAC,IAAI,CAAA;QAE3B,IAAI,sBAAsB,KAAK,SAAS,EAAE,CAAC;YACzC,IAAI,IAAI,CAAC,cAAc,KAAK,sBAAsB,EAAE,CAAC;gBACnD,MAAM,IAAI,KAAK,CACb,4FAA4F,CAC7F,CAAA;YACH,CAAC;YAED,IAAI,aAAa,KAAK,sBAAsB,GAAG,CAAC,EAAE,CAAC;gBACjD,MAAM,IAAI,KAAK,CACb,mHAAmH,aAAa,cAC9H,sBAAsB,GAAG,CAC3B,EAAE,CACH,CAAA;YACH,CAAC;QACH,CAAC;QAED,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACpC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACpC,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACtD,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACtD,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAChE,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC9C,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC5D,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC5C,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC9D,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC9C,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC1D,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACtD,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAChD,IAAI,CAAC,SAAS,GAAG,SAAS,CAAA;IAC5B,CAAC;IAED,MAAM,CAAC,cAAc,CACnB,YAA0B,EAC1B,iBAAsC;QAEtC,MAAM,WAAW,GAAG,IAAA,kCAAiB,EAAC,YAAY,EAAE,iBAAiB,CAAC,CAAA;QACtE,OAAO,IAAI,WAAW,CAAC,WAAW,CAAC,CAAA;IACrC,CAAC;IAED,MAAM,CAAC,sBAAsB,CAC3B,SAAoB,EACpB,iBAAsC;QAEtC,MAAM,YAAY,GAAG,+BAAkB,CAAC,SAAS,CAAC,CAAA;QAClD,MAAM,WAAW,GAAG,IAAA,kCAAiB,EAAC,YAAY,EAAE,iBAAiB,CAAC,CAAA;QACtE,OAAO,IAAI,WAAW,CAAC,EAAE,GAAG,WAAW,EAAE,SAAS,EAAE,CAAC,CAAA;IACvD,CAAC;IAEO,oBAAoB,CAAC,EAC3B,cAAc,EACd,iBAAiB,MACA,EAAE;QACnB,IAAI,YAAgC,CAAA;QAEpC,IACE,cAAc,KAAK,iCAAkB;YACrC,cAAc,EAAE,GAAG,CAAC,iCAAkB,CAAC,EACvC,CAAC;YACD,cAAc,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAA;YAC/C,MAAM,iBAAiB,GAAG,cAAc,CAAA;YACxC,IAAI,iBAAiB,KAAK,iCAAkB,EAAE,CAAC;gBAC7C,MAAM,IAAI,KAAK,CACb,mEAAmE,CACpE,CAAA;YACH,CAAC;YACD,IAAI,OAAO,iBAAiB,KAAK,QAAQ,EAAE,CAAC;gBAC1C,gDAAgD;gBAChD,iBAAiB,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,iBAAiB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;YACnE,CAAC;iBAAM,CAAC;gBACN,yEAAyE;gBACzE,iBAAiB,GAAG,IAAI,GAAG,EAAE,CAAA;YAC/B,CAAC;QACH,CAAC;QAED,IACE,CAAC,iBAAiB;YAClB,iBAAiB,KAAK,iCAAkB;YACxC,iBAAiB,CAAC,GAAG,CAAC,iCAAkB,CAAC,EACzC,CAAC;YACD,gDAAgD;YAChD,iBAAiB,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAA;YAClD,MAAM,oBAAoB,GAAG,iBAAiB,CAAA;YAC9C,IAAI,cAAc,EAAE,IAAI,EAAE,CAAC;gBACzB,cAAc,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,oBAAoB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;gBACjE,2CAA2C;gBAC3C,iBAAiB,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,cAAc,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;gBAC9D,IAAI,iBAAiB,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;oBAC/B,YAAY,GAAG,IAAA,8BAAoB,EAAC,iBAAiB,CAAC,CAAA;gBACxD,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,YAAY,GAAG,IAAI,CAAC,oBAAoB,CAAA;YAC1C,CAAC;QACH,CAAC;QAED,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,CAAA;IACzC,CAAC;IAED,eAAe,CACb,YAAoB,EACpB,aAA6B;QAE7B,MAAM,kBAAkB,GAAG,aAAa;YACtC,CAAC,CAAC,IAAI,CAAC,oBAAoB,CAAC,aAAa,CAAC;YAC1C,CAAC,CAAC,IAAI,CAAC,yBAAyB,CAAA;QAElC,IAAI,kBAAkB,CAAC,YAAY,EAAE,CAAC;YACpC,MAAM,KAAK,GAAG,YAAY,CAAC,KAAK,CAAC,kBAAkB,CAAC,YAAY,CAAC,CAAA;YACjE,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;gBACnB,MAAM,IAAI,KAAK,CAAC,mCAAmC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;YAChE,CAAC;QACH,CAAC;QAED,OAAO,IAAI,CAAC,6BAA6B,CAAC,qBAAqB,CAC7D,YAAY,EACZ,kBAAkB,CAAC,cAAc,CAClC,CAAA;IACH,CAAC;IAED,MAAM,CAAC,YAAoB,EAAE,aAA6B;QACxD,MAAM,kBAAkB,GAAG,aAAa;YACtC,CAAC,CAAC,IAAI,CAAC,oBAAoB,CAAC,aAAa,CAAC;YAC1C,CAAC,CAAC,IAAI,CAAC,yBAAyB,CAAA;QAElC,IAAI,kBAAkB,CAAC,YAAY,EAAE,CAAC;YACpC,MAAM,KAAK,GAAG,YAAY,CAAC,KAAK,CAAC,kBAAkB,CAAC,YAAY,CAAC,CAAA;YACjE,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;gBACnB,MAAM,IAAI,KAAK,CAAC,mCAAmC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;YAChE,CAAC;QACH,CAAC;QAED,OAAO,IAAI,CAAC,6BAA6B,CAAC,YAAY,CACpD,YAAY,EACZ,kBAAkB,CAAC,cAAc,CAClC,CAAA;IACH,CAAC;IAED;;;;;;OAMG;IACH,CAAC,mBAAmB,CAClB,IAA2B,EAC3B,KAAK,GAAG,IAAI,CAAC,SAAS;QAEtB,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,MAAM,IAAI,KAAK,CACb,sFAAsF,CACvF,CAAA;QACH,CAAC;QAED,MAAM,MAAM,GACV,4BAAe,CAAC,KAAsB,CAAC,CAAA;QACzC,MAAM,cAAc,GAAG,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,0BAAO,CAAC,CAAA;QAC7D,MAAM,YAAY,GAAG,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,wBAAK,CAAC,CAAA;QAEzD,IAAI,CAAC,MAAM,IAAI,cAAc,KAAK,SAAS,IAAI,YAAY,KAAK,SAAS,EAAE,CAAC;YAC1E,MAAM,IAAI,KAAK,CAAC,UAAU,KAAK,0BAA0B,CAAC,CAAA;QAC5D,CAAC;QACD,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,CAAC,wBAAK,CAAC,CAAC,CAAA;QACvC,MAAM,EAAE,gBAAgB,EAAE,aAAa,EAAE,GAAG,MAAM,CAAA;QAClD,MAAM,uBAAuB,GAC3B,gBAAgB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;QAClE,MAAM,oBAAoB,GACxB,aAAa,CAAC,MAAM,GAAG,CAAC;YACtB,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,aAAa,EAAE,EAAE,cAAc,EAAE,CAAC;YAChD,CAAC,CAAC,EAAE,CAAA;QACR,MAAM,SAAS,GAAG,IAAI,GAAG,EAAoB,CAAA;QAE7C,KAAK,MAAM,EAAE,IAAI,GAAG,QAAQ,EAAE,IAAI,GAAG,IAAI,EAAE,OAAO,EAAE,IAAI,IAAI,EAAE,CAAC;YAC7D,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;gBAC1B,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAA;YAC9D,CAAC;YAED,MAAM,CAAC,cAAc,CAAC,CAAA;YACtB,MAAM,WAAW,GAAG,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;YAC5D,SAAS,CAAC,GAAG,CAAC,IAAI,EAAE,WAAW,CAAC,CAAA;YAChC,MAAM,WAAW,CAAA;YACjB,IAAI,oBAAoB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACpC,MAAM,oBAAoB,CAAA;YAC5B,CAAC;YACD,KAAK,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,CAAA;YACpC,MAAM,CAAC,YAAY,CAAC,CAAA;YACpB,MAAM,uBAAuB,CAAA;QAC/B,CAAC;QAED,2DAA2D;QAC3D,MAAM,CAAC,cAAc,CAAC,CAAA;QACtB,KAAK,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,WAAW,CAAC,CAAA;QACxC,IAAI,oBAAoB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACpC,MAAM,oBAAoB,CAAA;QAC5B,CAAC;IACH,CAAC;IAED;;;;;;OAMG;IACH,UAAU,CAAC,IAA4B,EAAE,KAAK,GAAG,IAAI,CAAC,SAAS;QAC7D,OAAO,CAAC,GAAG,IAAI,CAAC,mBAAmB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAA;IAC1D,CAAC;IAED;;OAEG;IACH,kBAAkB,CAChB,KAAqC,EACrC,UAAkB;QAElB,MAAM,cAAc,GAClB,OAAO,KAAK,KAAK,QAAQ;YACvB,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC;YAC7B,CAAC,CAAC,IAAI,CAAC,mBAAmB,CAAC,KAAK,CAAC,CAAA;QACrC,IAAI,KAAK,GAAG,CAAC,CAAA;QACb,KAAK,MAAM,MAAM,IAAI,cAAc,EAAE,CAAC;YACpC,KAAK,IAAI,MAAM,CAAC,MAAM,CAAA;YACtB,IAAI,KAAK,GAAG,UAAU,EAAE,CAAC;gBACvB,OAAO,KAAK,CAAA;YACd,CAAC;QACH,CAAC;QACD,OAAO,KAAK,CAAA;IACd,CAAC;IAED;;;OAGG;IACH,WAAW,CACT,KAAqC,EACrC,aAA6B;QAE7B,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YAC9B,MAAM,kBAAkB,GAAG,aAAa;gBACtC,CAAC,CAAC,IAAI,CAAC,oBAAoB,CAAC,aAAa,CAAC;gBAC1C,CAAC,CAAC,IAAI,CAAC,yBAAyB,CAAA;YAElC,IAAI,kBAAkB,CAAC,YAAY,EAAE,CAAC;gBACpC,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,kBAAkB,CAAC,YAAY,CAAC,CAAA;gBAC1D,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;oBACnB,MAAM,IAAI,KAAK,CAAC,mCAAmC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;gBAChE,CAAC;YACH,CAAC;YAED,OAAO,IAAI,CAAC,6BAA6B,CAAC,WAAW,CACnD,KAAK,EACL,kBAAkB,CAAC,cAAc,CAClC,CAAA;QACH,CAAC;QAED,MAAM,cAAc,GAAG,IAAI,CAAC,mBAAmB,CAAC,KAAK,CAAC,CAAA;QACtD,IAAI,KAAK,GAAG,CAAC,CAAA;QACb,KAAK,MAAM,MAAM,IAAI,cAAc,EAAE,CAAC;YACpC,KAAK,IAAI,MAAM,CAAC,MAAM,CAAA;QACxB,CAAC;QACD,OAAO,KAAK,CAAA;IACd,CAAC;IAED,iBAAiB,CAAC,IAAY;QAC5B,IAAI,CAAC,6BAA6B,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAA;IAC5D,CAAC;IAED,eAAe;QACb,IAAI,CAAC,6BAA6B,CAAC,eAAe,EAAE,CAAA;IACtD,CAAC;IAED,MAAM,CAAC,mBAAqC;QAC1C,OAAO,IAAI,CAAC,6BAA6B,CAAC,YAAY,CAAC,mBAAmB,CAAC,CAAA;IAC7E,CAAC;IAED,CAAC,eAAe,CACd,mBAAqC;QAErC,MAAM,oBAAoB,GACxB,IAAI,CAAC,6BAA6B,CAAC,qBAAqB,CACtD,mBAAmB,CACpB,CAAA;QAEH,IAAI,MAAM,GAAG,EAAE,CAAA;QAEf,KAAK,MAAM,WAAW,IAAI,oBAAoB,EAAE,CAAC;YAC/C,MAAM;gBACJ,OAAO,WAAW,KAAK,QAAQ;oBAC7B,CAAC,CAAC,WAAW;oBACb,CAAC,CAAC,iCAAO,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;YAEnD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,IAAA,+CAAkC,EAAC,MAAM,CAAC,EAAE,CAAC;gBACtE,yEAAyE;gBACzE,uCAAuC;gBACvC,SAAQ;YACV,CAAC;iBAAM,CAAC;gBACN,MAAM,MAAM,CAAA;gBACZ,eAAe;gBACf,MAAM,GAAG,EAAE,CAAA;YACb,CAAC;QACH,CAAC;QAED,+CAA+C;QAC/C,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtB,MAAM,MAAM,CAAA;QACd,CAAC;IACH,CAAC;IAED,KAAK,CAAC,CAAC,oBAAoB,CACzB,mBAA0C;QAE1C,MAAM,oBAAoB,GACxB,IAAI,CAAC,6BAA6B,CAAC,yBAAyB,CAC1D,mBAAmB,CACpB,CAAA;QAEH,IAAI,MAAM,GAAG,EAAE,CAAA;QAEf,IAAI,KAAK,EAAE,MAAM,WAAW,IAAI,oBAAoB,EAAE,CAAC;YACrD,MAAM;gBACJ,OAAO,WAAW,KAAK,QAAQ;oBAC7B,CAAC,CAAC,WAAW;oBACb,CAAC,CAAC,iCAAO,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;YAEnD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,IAAA,+CAAkC,EAAC,MAAM,CAAC,EAAE,CAAC;gBACtE,yEAAyE;gBACzE,uCAAuC;gBACvC,SAAQ;YACV,CAAC;iBAAM,CAAC;gBACN,MAAM,MAAM,CAAA;gBACZ,eAAe;gBACf,MAAM,GAAG,EAAE,CAAA;YACb,CAAC;QACH,CAAC;QAED,+CAA+C;QAC/C,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtB,MAAM,MAAM,CAAA;QACd,CAAC;IACH,CAAC;IAED,KAAK,CAAC,WAAW,CACf,mBAA0C;QAE1C,MAAM,oBAAoB,GACxB,IAAI,CAAC,6BAA6B,CAAC,yBAAyB,CAC1D,mBAAmB,CACpB,CAAA;QAEH,IAAI,MAAM,GAAG,EAAE,CAAA;QAEf,IAAI,KAAK,EAAE,MAAM,WAAW,IAAI,oBAAoB,EAAE,CAAC;YACrD,MAAM;gBACJ,OAAO,WAAW,KAAK,QAAQ;oBAC7B,CAAC,CAAC,WAAW;oBACb,CAAC,CAAC,iCAAO,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;QACrD,CAAC;QAED,OAAO,MAAM,CAAA;IACf,CAAC;IAED;;;;;;OAMG;IACH,YAAY,CAAC,UAAkB,EAAE,SAAS,GAAG,IAAI,CAAC,SAAS;QACzD,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CACb,sFAAsF,CACvF,CAAA;QACH,CAAC;QAED,MAAM,KAAK,GAAG,kBAAM,CAAC,SAAS,CAAC,CAAA;QAC/B,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,MAAM,IAAI,KAAK,CAAC,kBAAkB,SAAS,EAAE,CAAC,CAAA;QAChD,CAAC;QAED,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;YAChB,MAAM,IAAI,KAAK,CAAC,4CAA4C,SAAS,EAAE,CAAC,CAAA;QAC1E,CAAC;QAED,MAAM,cAAc,GAAG,KAAK,CAAC,IAAI,CAAA;QACjC,MAAM,MAAM,GAAiB,EAAE,CAAA;QAE/B,uDAAuD;QACvD,4CAA4C;QAC5C,MAAM,aAAa,GAAG,UAAU,GAAG,SAAS,CAAA;QAE5C,IAAI,cAAc,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YACvC,MAAM,CAAC,KAAK,GAAG,cAAc,CAAC,KAAK,GAAG,aAAa,CAAA;QACrD,CAAC;QAED,IAAI,cAAc,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YACxC,MAAM,CAAC,MAAM,GAAG,cAAc,CAAC,MAAM,GAAG,aAAa,CAAA;QACvD,CAAC;QAED,IAAI,cAAc,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;YAC5C,MAAM,CAAC,UAAU,GAAG,cAAc,CAAC,UAAU,GAAG,aAAa,CAAA;QAC/D,CAAC;QAED,IAAI,cAAc,CAAC,WAAW,KAAK,SAAS,EAAE,CAAC;YAC7C,MAAM,CAAC,WAAW,GAAG,cAAc,CAAC,WAAW,GAAG,aAAa,CAAA;QACjE,CAAC;QAED,OAAO,MAAM,CAAA;IACf,CAAC;;AAncH,kCAocC"}

package/cjs/GptEncoding.test.js

@@ -5,10 +5,14 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 Object.defineProperty(exports, "__esModule", { value: true });
 const fs_1 = __importDefault(require("fs"));
 const path_1 = __importDefault(require("path"));
+// eslint-disable-next-line import/no-extraneous-dependencies
+const vitest_1 = require("vitest");
 const constants_js_1 = require("./constants.js");
 const GptEncoding_js_1 = require("./GptEncoding.js");
 const mapping_js_1 = require("./mapping.js");
+const models_js_1 = require("./models.js");
 const resolveEncoding_js_1 = require("./resolveEncoding.js");
+const specialTokens_js_1 = require("./specialTokens.js");
 const sharedResults = {
     space: [220],
     tab: [197],
@@ -62,8 +66,8 @@ const results = {
 const offsetPrompts = [
     // Basic prompt with "hello world"
     'hello world',
-    // Basic prompt with special token "<|endoftext|>"
-    'hello world<|endoftext|> green cow',
+    // Basic prompt with special token end of text token
+    `hello world${specialTokens_js_1.EndOfText} green cow`,
     // Chinese text: "我非常渴望与人工智能一起工作"
     '我非常渴望与人工智能一起工作',
     // Contains the interesting tokens b'\xe0\xae\xbf\xe0\xae' and b'\xe0\xaf\x8d\xe0\xae'
@@ -75,68 +79,68 @@ const offsetPrompts = [
 ];
 // eslint-disable-next-line @typescript-eslint/no-use-before-define
 const testPlans = loadTestPlans();
-describe.each(mapping_js_1.encodingNames)('%s', (encodingName) => {
+vitest_1.describe.each(mapping_js_1.encodingNames)('%s', (encodingName) => {
     const encoding = GptEncoding_js_1.GptEncoding.getEncodingApi(encodingName, resolveEncoding_js_1.resolveEncoding);
     const { decode, decodeGenerator, decodeAsyncGenerator, encode, isWithinTokenLimit, } = encoding;
-    describe('encode and decode', () => {
-        it.each(offsetPrompts)('offset prompt: %s', (str) => {
-            expect(decode(encode(str, { allowedSpecial: constants_js_1.ALL_SPECIAL_TOKENS }))).toEqual(str);
+    (0, vitest_1.describe)('encode and decode', () => {
+        vitest_1.test.each(offsetPrompts)('offset prompt: %s', (str) => {
+            (0, vitest_1.expect)(decode(encode(str, { allowedSpecial: constants_js_1.ALL_SPECIAL_TOKENS }))).toEqual(str);
         });
     });
-    describe('basic functionality', () => {
+    (0, vitest_1.describe)('basic functionality', () => {
         const result = results[encodingName];
-        it('empty string', () => {
+        (0, vitest_1.test)('empty string', () => {
             const str = '';
-            expect(encode(str)).toEqual([]);
-            expect(decode(encode(str))).toEqual(str);
-            expect(isWithinTokenLimit(str, 0)).toBe(0);
-            expect(isWithinTokenLimit(str, 3)).toBe(0);
+            (0, vitest_1.expect)(encode(str)).toEqual([]);
+            (0, vitest_1.expect)(decode(encode(str))).toEqual(str);
+            (0, vitest_1.expect)(isWithinTokenLimit(str, 0)).toBe(0);
+            (0, vitest_1.expect)(isWithinTokenLimit(str, 3)).toBe(0);
         });
-        it('space', () => {
+        (0, vitest_1.test)('space', () => {
             const str = ' ';
-            expect(encode(str)).toEqual(result.space);
-            expect(decode(encode(str))).toEqual(str);
-            expect(isWithinTokenLimit(str, 3)).toBe(1);
-            expect(isWithinTokenLimit(str, 0)).toBe(false);
+            (0, vitest_1.expect)(encode(str)).toEqual(result.space);
+            (0, vitest_1.expect)(decode(encode(str))).toEqual(str);
+            (0, vitest_1.expect)(isWithinTokenLimit(str, 3)).toBe(1);
+            (0, vitest_1.expect)(isWithinTokenLimit(str, 0)).toBe(false);
         });
-        it('tab', () => {
+        (0, vitest_1.test)('tab', () => {
             const str = '\t';
-            expect(encode(str)).toEqual(result.tab);
-            expect(decode(encode(str))).toEqual(str);
+            (0, vitest_1.expect)(encode(str)).toEqual(result.tab);
+            (0, vitest_1.expect)(decode(encode(str))).toEqual(str);
         });
-        it('simple text', () => {
+        (0, vitest_1.test)('simple text', () => {
             const str = 'This is some text';
-            expect(encode(str)).toEqual(result[str]);
-            expect(decode(encode(str))).toEqual(str);
-            expect(isWithinTokenLimit(str, 3)).toBe(false);
-            expect(isWithinTokenLimit(str, 5)).toBe(result[str].length);
+            (0, vitest_1.expect)(encode(str)).toEqual(result[str]);
+            (0, vitest_1.expect)(decode(encode(str))).toEqual(str);
+            (0, vitest_1.expect)(isWithinTokenLimit(str, 3)).toBe(false);
+            (0, vitest_1.expect)(isWithinTokenLimit(str, 5)).toBe(result[str].length);
         });
-        it('multi-token word', () => {
+        (0, vitest_1.test)('multi-token word', () => {
             const str = 'indivisible';
-            expect(encode(str)).toEqual(result.indivisible);
-            expect(decode(encode(str))).toEqual(str);
-            expect(isWithinTokenLimit(str, 3)).toBe(result.indivisible.length);
+            (0, vitest_1.expect)(encode(str)).toEqual(result.indivisible);
+            (0, vitest_1.expect)(decode(encode(str))).toEqual(str);
+            (0, vitest_1.expect)(isWithinTokenLimit(str, 3)).toBe(result.indivisible.length);
         });
-        it('emojis', () => {
+        (0, vitest_1.test)('emojis', () => {
             const str = 'hello 👋 world 🌍';
-            expect(encode(str)).toEqual(result[str]);
-            expect(decode(encode(str))).toEqual(str);
-            expect(isWithinTokenLimit(str, 4)).toBe(false);
-            expect(isWithinTokenLimit(str, 400)).toBe(result[str].length);
+            (0, vitest_1.expect)(encode(str)).toEqual(result[str]);
+            (0, vitest_1.expect)(decode(encode(str))).toEqual(str);
+            (0, vitest_1.expect)(isWithinTokenLimit(str, 4)).toBe(false);
+            (0, vitest_1.expect)(isWithinTokenLimit(str, 400)).toBe(result[str].length);
         });
-        it('decode token-by-token via generator', () => {
+        (0, vitest_1.test)('decode token-by-token via generator', () => {
             const str = 'hello 👋 world 🌍';
             const generator = decodeGenerator(result[str]);
             result.decodedHelloWorldTokens.forEach((token) => {
-                expect(generator.next().value).toBe(token);
+                (0, vitest_1.expect)(generator.next().value).toBe(token);
             });
         });
-        it('encodes and decodes special tokens', () => {
-            const str = 'hello <|endoftext|> world';
+        (0, vitest_1.test)('encodes and decodes special tokens', () => {
+            const str = `hello ${specialTokens_js_1.EndOfText} world`;
             const encoded = encode(str, {
                 allowedSpecial: constants_js_1.ALL_SPECIAL_TOKENS,
             });
-            expect(decode(encoded)).toEqual(str);
+            (0, vitest_1.expect)(decode(encoded)).toEqual(str);
         });
         async function* getHelloWorldTokensAsync() {
             const str = 'hello 👋 world 🌍';
@@ -145,33 +149,33 @@ describe.each(mapping_js_1.encodingNames)('%s', (encodingName) => {
                 yield await Promise.resolve(token);
             }
         }
-        it('decode token-by-token via async generator', async () => {
+        (0, vitest_1.test)('decode token-by-token via async generator', async () => {
             const generator = decodeAsyncGenerator(getHelloWorldTokensAsync());
             const decoded = [...result.decodedHelloWorldTokens];
             for await (const value of generator) {
-                expect(value).toEqual(decoded.shift());
+                (0, vitest_1.expect)(value).toEqual(decoded.shift());
             }
         });
-        it('properties of Object', () => {
+        (0, vitest_1.test)('properties of Object', () => {
             const str = 'toString constructor hasOwnProperty valueOf';
-            expect(encode(str)).toEqual(result[str]);
-            expect(decode(encode(str))).toEqual(str);
+            (0, vitest_1.expect)(encode(str)).toEqual(result[str]);
+            (0, vitest_1.expect)(decode(encode(str))).toEqual(str);
         });
-        it('text with commas', () => {
+        (0, vitest_1.test)('text with commas', () => {
             const str = 'hello, I am a text, and I have commas. a,b,c';
-            expect(decode(encode(str))).toEqual(str);
-            expect(encode(str)).toStrictEqual(result[str]);
-            expect(isWithinTokenLimit(str, result[str].length - 1)).toBe(false);
-            expect(isWithinTokenLimit(str, 300)).toBe(result[str].length);
+            (0, vitest_1.expect)(decode(encode(str))).toEqual(str);
+            (0, vitest_1.expect)(encode(str)).toStrictEqual(result[str]);
+            (0, vitest_1.expect)(isWithinTokenLimit(str, result[str].length - 1)).toBe(false);
+            (0, vitest_1.expect)(isWithinTokenLimit(str, 300)).toBe(result[str].length);
         });
     });
-    describe('test plan', () => {
+    (0, vitest_1.describe)('test plan', () => {
         testPlans[encodingName].forEach(({ sample, encoded }) => {
-            it(`encodes ${sample}`, () => {
-                expect(encode(sample)).toEqual(encoded);
+            (0, vitest_1.test)(`encodes ${sample}`, () => {
+                (0, vitest_1.expect)(encode(sample)).toEqual(encoded);
             });
-            it(`decodes ${sample}`, () => {
-                expect(decode(encoded)).toEqual(sample);
+            (0, vitest_1.test)(`decodes ${sample}`, () => {
+                (0, vitest_1.expect)(decode(encoded)).toEqual(sample);
             });
         });
     });
@@ -207,31 +211,84 @@ const exampleMessages = [
         content: "This late pivot means we don't have time to boil the ocean for the client deliverable.",
     },
 ];
-describe.each(chatModelNames)('%s', (modelName) => {
+vitest_1.describe.each(chatModelNames)('%s', (modelName) => {
     const encoding = GptEncoding_js_1.GptEncoding.getEncodingApiForModel(modelName, resolveEncoding_js_1.resolveEncoding);
     const expectedEncodedLength = modelName.startsWith('gpt-3.5-turbo')
         ? 127
         : modelName.startsWith('gpt-4o')
             ? 120
             : 121;
-    describe('chat functionality', () => {
-        it('encodes a chat correctly', () => {
+    (0, vitest_1.describe)('chat functionality', () => {
+        (0, vitest_1.test)('encodes a chat correctly', () => {
             const encoded = encoding.encodeChat(exampleMessages);
-            expect(encoded).toMatchSnapshot();
-            expect(encoded).toHaveLength(expectedEncodedLength);
+            (0, vitest_1.expect)(encoded).toMatchSnapshot();
+            (0, vitest_1.expect)(encoded).toHaveLength(expectedEncodedLength);
             const decoded = encoding.decode(encoded);
-            expect(decoded).toMatchSnapshot();
+            (0, vitest_1.expect)(decoded).toMatchSnapshot();
         });
-        it('isWithinTokenLimit: false', () => {
+        (0, vitest_1.test)('isWithinTokenLimit: false', () => {
             const isWithinTokenLimit = encoding.isWithinTokenLimit(exampleMessages, 50);
-            expect(isWithinTokenLimit).toBe(false);
+            (0, vitest_1.expect)(isWithinTokenLimit).toBe(false);
         });
-        it('isWithinTokenLimit: true (number)', () => {
+        (0, vitest_1.test)('isWithinTokenLimit: true (number)', () => {
             const isWithinTokenLimit = encoding.isWithinTokenLimit(exampleMessages, 150);
-            expect(isWithinTokenLimit).toBe(expectedEncodedLength);
+            (0, vitest_1.expect)(isWithinTokenLimit).toBe(expectedEncodedLength);
         });
     });
 });
+(0, vitest_1.describe)('estimateCost functionality', () => {
+    const gpt4oEncoding = GptEncoding_js_1.GptEncoding.getEncodingApiForModel('gpt-4o', resolveEncoding_js_1.resolveEncoding);
+    const gpt35Encoding = GptEncoding_js_1.GptEncoding.getEncodingApiForModel('gpt-3.5-turbo', resolveEncoding_js_1.resolveEncoding);
+    (0, vitest_1.test)('estimates cost correctly for gpt-4o model', () => {
+        const tokenCount = 1_000;
+        const cost = gpt4oEncoding.estimateCost(tokenCount);
+        // gpt-4o has $2.5 per million tokens for input and $10 per million tokens for output
+        (0, vitest_1.expect)(cost.input).toBeCloseTo(0.002_5, 6); // 1000/1M * $2.5
+        (0, vitest_1.expect)(cost.output).toBeCloseTo(0.01, 6); // 1000/1M * $10
+        (0, vitest_1.expect)(cost.batchInput).toBeCloseTo(0.001_25, 6); // 1000/1M * $1.25
+        (0, vitest_1.expect)(cost.batchOutput).toBeCloseTo(0.005, 6); // 1000/1M * $5
+    });
+    (0, vitest_1.test)('estimates cost correctly for gpt-3.5-turbo model', () => {
+        const tokenCount = 1_000;
+        const cost = gpt35Encoding.estimateCost(tokenCount);
+        // gpt-3.5-turbo has $0.5 per million tokens for input and $1.5 per million tokens for output
+        (0, vitest_1.expect)(cost.input).toBeCloseTo(0.000_5, 6); // 1000/1M * $0.5
+        (0, vitest_1.expect)(cost.output).toBeCloseTo(0.001_5, 6); // 1000/1M * $1.5
+        (0, vitest_1.expect)(cost.batchInput).toBeCloseTo(0.000_25, 6); // 1000/1M * $0.25
+        (0, vitest_1.expect)(cost.batchOutput).toBeCloseTo(0.000_75, 6); // 1000/1M * $0.75
+    });
+    (0, vitest_1.test)('allows overriding model name', () => {
+        const tokenCount = 1_000;
+        // Use gpt-4o encoding but override with gpt-3.5-turbo model name
+        const cost = gpt4oEncoding.estimateCost(tokenCount, 'gpt-3.5-turbo');
+        (0, vitest_1.expect)(cost.input).toBeCloseTo(0.000_5, 6); // 1000/1M * $0.5
+        (0, vitest_1.expect)(cost.output).toBeCloseTo(0.001_5, 6); // 1000/1M * $1.5
+    });
+    (0, vitest_1.test)('throws error when model name is not provided', () => {
+        const encoding = GptEncoding_js_1.GptEncoding.getEncodingApi('cl100k_base', resolveEncoding_js_1.resolveEncoding);
+        const tokenCount = 1_000;
+        // No model name was provided during initialization or function call
+        (0, vitest_1.expect)(() => encoding.estimateCost(tokenCount)).toThrow('Model name must be provided either during initialization or passed in to the method.');
+    });
+    (0, vitest_1.test)('throws error for unknown model', () => {
+        const tokenCount = 1_000;
+        (0, vitest_1.expect)(() => gpt4oEncoding.estimateCost(tokenCount, 'non-existent-model')).toThrow('Unknown model: non-existent-model');
+    });
+    (0, vitest_1.test)('only includes properties that exist for the model', () => {
+        // Find a model that only has input cost but no output cost
+        const modelWithInputOnly = Object.entries(models_js_1.models).find(([_, model]) => model.cost?.input !== undefined && model.cost?.output === undefined);
+        if (modelWithInputOnly) {
+            const [modelName] = modelWithInputOnly;
+            const cost = gpt4oEncoding.estimateCost(1_000, modelName);
+            (0, vitest_1.expect)(cost.input).toBeDefined();
+            (0, vitest_1.expect)(cost.output).toBeUndefined();
+        }
+        else {
+            // Skip test if we can't find an appropriate model
+            console.log('Skipping test: no model with input-only cost found');
+        }
+    });
+});
 function loadTestPlans() {
     const testPlanPath = path_1.default.join(__dirname, '../data/TestPlans.txt');
     const testPlanData = fs_1.default.readFileSync(testPlanPath, 'utf8');

package/cjs/GptEncoding.test.js.map

@@ -1 +1 @@
-{"version":3,"file":"GptEncoding.test.js","sourceRoot":"","sources":["../src/GptEncoding.test.ts"],"names":[],"mappings":";;;;;AAAA,4CAAmB;AACnB,gDAAuB;AACvB,iDAAmD;AACnD,qDAAgE;AAChE,6CAKqB;AACrB,6DAAsD;AAEtD,MAAM,aAAa,GAAG;IACpB,KAAK,EAAE,CAAC,GAAG,CAAC;IACZ,GAAG,EAAE,CAAC,GAAG,CAAC;IACV,mBAAmB,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE,GAAG,EAAE,KAAK,CAAC;IAC7C,WAAW,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,MAAM,CAAC;IAC/B,mBAAmB,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,GAAG,CAAC;IACjE,uBAAuB,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,EAAE,IAAI,CAAC;IAClE,6CAA6C,EAAE;QAC7C,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK;KACzD;IACD,8CAA8C,EAAE;QAC9C,MAAM,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE;QAC1E,EAAE,EAAE,EAAE,EAAE,EAAE;KACX;CACF,CAAA;AAED,MAAM,OAAO,GAAG;IACd,UAAU,EAAE;QACV,KAAK,EAAE,CAAC,GAAG,CAAC;QACZ,GAAG,EAAE,CAAC,GAAG,CAAC;QACV,mBAAmB,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,KAAK,CAAC;QAC/C,WAAW,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,OAAO,CAAC;QAChC,mBAAmB,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,CAAC;QAC/D,uBAAuB,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,EAAE,IAAI,CAAC;QAClE,6CAA6C,EAAE;YAC7C,GAAG,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK;SAClD;QACD,8CAA8C,EAAE;YAC9C,MAAM,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,OAAO,EAAE,EAAE,EAAE,GAAG;YACrE,MAAM,EAAE,MAAM;SACf;KACF;IACD,WAAW,EAAE;QACX,KAAK,EAAE,CAAC,GAAG,CAAC;QACZ,GAAG,EAAE,CAAC,GAAG,CAAC;QACV,mBAAmB,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,KAAK,CAAC;QAC/C,WAAW,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,MAAM,CAAC;QAC/B,mBAAmB,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,GAAG,CAAC;QACnE,uBAAuB,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,EAAE,IAAI,CAAC;QAClE,6CAA6C,EAAE;YAC7C,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,KAAK;SACtC;QACD,8CAA8C,EAAE;YAC9C,MAAM,EAAE,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,MAAM,EAAE,EAAE,EAAE,GAAG;YACtE,KAAK,EAAE,MAAM;SACd;KACF;IACD,SAAS,EAAE,aAAa;IACxB,SAAS,EAAE,aAAa;IACxB,SAAS,EAAE,aAAa;CACe,CAAA;AAEzC,MAAM,aAAa,GAAG;IACpB,kCAAkC;IAClC,aAAa;IAEb,kDAAkD;IAClD,oCAAoC;IAEpC,iCAAiC;IACjC,gBAAgB;IAEhB,sFAAsF;IACtF,yDAAyD;IACzD,eAAe;IAEf,qDAAqD;IACrD,yFAAyF;IACzF,KAAK;CACN,CAAA;AAED,mEAAmE;AACnE,MAAM,SAAS,GAAG,aAAa,EAAE,CAAA;AAEjC,QAAQ,CAAC,IAAI,CAAC,0BAAa,CAAC,CAAC,IAAI,EAAE,CAAC,YAA0B,EAAE,EAAE;IAChE,MAAM,QAAQ,GAAG,4BAAW,CAAC,cAAc,CAAC,YAAY,EAAE,oCAAe,CAAC,CAAA;IAE1E,MAAM,EACJ,MAAM,EACN,eAAe,EACf,oBAAoB,EACpB,MAAM,EACN,kBAAkB,GACnB,GAAG,QAAQ,CAAA;IAEZ,QAAQ,CAAC,mBAAmB,EAAE,GAAG,EAAE;QACjC,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,mBAAmB,EAAE,CAAC,GAAG,EAAE,EAAE;YAClD,MAAM,CACJ,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,EAAE,cAAc,EAAE,iCAAkB,EAAE,CAAC,CAAC,CAC5D,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;QAChB,CAAC,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEF,QAAQ,CAAC,qBAAqB,EAAE,GAAG,EAAE;QACnC,MAAM,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC,CAAA;QACpC,EAAE,CAAC,cAAc,EAAE,GAAG,EAAE;YACtB,MAAM,GAAG,GAAG,EAAE,CAAA;YACd,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;YAC/B,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;YACxC,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YAC1C,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QAC5C,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;YACf,MAAM,GAAG,GAAG,GAAG,CAAA;YACf,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;YACzC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;YACxC,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YAC1C,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QAChD,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;YACb,MAAM,GAAG,GAAG,IAAI,CAAA;YAChB,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;YACvC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;QAC1C,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,aAAa,EAAE,GAAG,EAAE;YACrB,MAAM,GAAG,GAAG,mBAAmB,CAAA;YAC/B,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;YACxC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;YACxC,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YAC9C,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAA;QAC7D,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,kBAAkB,EAAE,GAAG,EAAE;YAC1B,MAAM,GAAG,GAAG,aAAa,CAAA;YACzB,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,WAAW,CAAC,CAAA;YAC/C,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;YACxC,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,CAAA;QACpE,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,QAAQ,EAAE,GAAG,EAAE;YAChB,MAAM,GAAG,GAAG,mBAAmB,CAAA;YAC/B,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;YACxC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;YACxC,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YAC9C,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAA;QAC/D,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,qCAAqC,EAAE,GAAG,EAAE;YAC7C,MAAM,GAAG,GAAG,mBAAmB,CAAA;YAC/B,MAAM,SAAS,GAAG,eAAe,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;YAC9C,MAAM,CAAC,uBAAuB,CAAC,OAAO,CAAC,CAAC,KAAa,EAAE,EAAE;gBACvD,MAAM,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YAC5C,CAAC,CAAC,CAAA;QACJ,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,oCAAoC,EAAE,GAAG,EAAE;YAC5C,MAAM,GAAG,GAAG,2BAA2B,CAAA;YACvC,MAAM,OAAO,GAAG,MAAM,CAAC,GAAG,EAAE;gBAC1B,cAAc,EAAE,iCAAkB;aACnC,CAAC,CAAA;YACF,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;QACtC,CAAC,CAAC,CAAA;QAEF,KAAK,SAAS,CAAC,CAAC,wBAAwB;YACtC,MAAM,GAAG,GAAG,mBAAmB,CAAA;YAC/B,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;gBAChC,4CAA4C;gBAC5C,MAAM,MAAM,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;YACpC,CAAC;QACH,CAAC;QAED,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;YACzD,MAAM,SAAS,GAAG,oBAAoB,CAAC,wBAAwB,EAAE,CAAC,CAAA;YAClE,MAAM,OAAO,GAAG,CAAC,GAAG,MAAM,CAAC,uBAAuB,CAAC,CAAA;YACnD,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,SAAS,EAAE,CAAC;gBACpC,MAAM,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,CAAA;YACxC,CAAC;QACH,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,sBAAsB,EAAE,GAAG,EAAE;YAC9B,MAAM,GAAG,GAAG,6CAA6C,CAAA;YAEzD,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;YACxC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;QAC1C,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,kBAAkB,EAAE,GAAG,EAAE;YAC1B,MAAM,GAAG,GAAG,8CAA8C,CAAA;YAC1D,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;YACxC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,aAAa,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;YAC9C,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YACnE,MAAM,CAAC,kBAAkB,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAA;QAC/D,CAAC,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEF,QAAQ,CAAC,WAAW,EAAE,GAAG,EAAE;QACzB,SAAS,CAAC,YAAY,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,EAAE,EAAE;YACtD,EAAE,CAAC,WAAW,MAAM,EAAE,EAAE,GAAG,EAAE;gBAC3B,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAA;YACzC,CAAC,CAAC,CAAA;YACF,EAAE,CAAC,WAAW,MAAM,EAAE,EAAE,GAAG,EAAE;gBAC3B,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAA;YACzC,CAAC,CAAC,CAAA;QACJ,CAAC,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;AACJ,CAAC,CAAC,CAAA;AAEF,MAAM,cAAc,GAAG,MAAM,CAAC,IAAI,CAAC,4BAAe,CAA6B,CAAA;AAE/E,MAAM,eAAe,GAAkB;IACrC;QACE,IAAI,EAAE,QAAQ;QACd,OAAO,EACL,qGAAqG;KACxG;IACD;QACE,IAAI,EAAE,QAAQ;QACd,IAAI,EAAE,cAAc;QACpB,OAAO,EAAE,gDAAgD;KAC1D;IACD;QACE,IAAI,EAAE,QAAQ;QACd,IAAI,EAAE,mBAAmB;QACzB,OAAO,EAAE,qDAAqD;KAC/D;IACD;QACE,IAAI,EAAE,QAAQ;QACd,IAAI,EAAE,cAAc;QACpB,OAAO,EACL,sGAAsG;KACzG;IACD;QACE,IAAI,EAAE,QAAQ;QACd,IAAI,EAAE,mBAAmB;QACzB,OAAO,EAAE,+DAA+D;KACzE;IACD;QACE,IAAI,EAAE,MAAM;QACZ,OAAO,EACL,wFAAwF;KAC3F;CACO,CAAA;AAEV,QAAQ,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,IAAI,EAAE,CAAC,SAAS,EAAE,EAAE;IAChD,MAAM,QAAQ,GAAG,4BAAW,CAAC,sBAAsB,CACjD,SAAS,EACT,oCAAe,CAChB,CAAA;IACD,MAAM,qBAAqB,GAAG,SAAS,CAAC,UAAU,CAAC,eAAe,CAAC;QACjE,CAAC,CAAC,GAAG;QACL,CAAC,CAAC,SAAS,CAAC,UAAU,CAAC,QAAQ,CAAC;YAChC,CAAC,CAAC,GAAG;YACL,CAAC,CAAC,GAAG,CAAA;IAEP,QAAQ,CAAC,oBAAoB,EAAE,GAAG,EAAE;QAClC,EAAE,CAAC,0BAA0B,EAAE,GAAG,EAAE;YAClC,MAAM,OAAO,GAAG,QAAQ,CAAC,UAAU,CAAC,eAAe,CAAC,CAAA;YACpD,MAAM,CAAC,OAAO,CAAC,CAAC,eAAe,EAAE,CAAA;YACjC,MAAM,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,qBAAqB,CAAC,CAAA;YAEnD,MAAM,OAAO,GAAG,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;YACxC,MAAM,CAAC,OAAO,CAAC,CAAC,eAAe,EAAE,CAAA;QACnC,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,2BAA2B,EAAE,GAAG,EAAE;YACnC,MAAM,kBAAkB,GAAG,QAAQ,CAAC,kBAAkB,CACpD,eAAe,EACf,EAAE,CACH,CAAA;YACD,MAAM,CAAC,kBAAkB,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QACxC,CAAC,CAAC,CAAA;QACF,EAAE,CAAC,mCAAmC,EAAE,GAAG,EAAE;YAC3C,MAAM,kBAAkB,GAAG,QAAQ,CAAC,kBAAkB,CACpD,eAAe,EACf,GAAG,CACJ,CAAA;YACD,MAAM,CAAC,kBAAkB,CAAC,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAA;QACxD,CAAC,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;AACJ,CAAC,CAAC,CAAA;AAEF,SAAS,aAAa;IACpB,MAAM,YAAY,GAAG,cAAI,CAAC,IAAI,CAAC,SAAS,EAAE,uBAAuB,CAAC,CAAA;IAClE,MAAM,YAAY,GAAG,YAAE,CAAC,YAAY,CAAC,YAAY,EAAE,MAAM,CAAC,CAAA;IAC1D,MAAM,KAAK,GAGP;QACF,WAAW,EAAE,EAAE;QACf,SAAS,EAAE,EAAE;QACb,SAAS,EAAE,EAAE;QACb,SAAS,EAAE,EAAE;QACb,UAAU,EAAE,EAAE;KACf,CAAA;IACD,YAAY,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,EAAE;QAC9C,MAAM,CAAC,gBAAgB,EAAE,UAAU,EAAE,WAAW,CAAC,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;QACxE,IAAI,CAAC,gBAAgB,IAAI,CAAC,UAAU,IAAI,CAAC,WAAW;YAAE,OAAM;QAC5D,MAAM,YAAY,GAAG,gBAAgB,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAiB,CAAA;QACpE,KAAK,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC;YACvB,MAAM,EAAE,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;YACxD,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC;SACxD,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IACF,OAAO,KAAK,CAAA;AACd,CAAC"}
+{"version":3,"file":"GptEncoding.test.js","sourceRoot":"","sources":["../src/GptEncoding.test.ts"],"names":[],"mappings":";;;;;AAAA,4CAAmB;AACnB,gDAAuB;AACvB,6DAA6D;AAC7D,mCAA+C;AAC/C,iDAAmD;AACnD,qDAAgE;AAChE,6CAKqB;AACrB,2CAAoC;AACpC,6DAAsD;AACtD,yDAA8C;AAE9C,MAAM,aAAa,GAAG;IACpB,KAAK,EAAE,CAAC,GAAG,CAAC;IACZ,GAAG,EAAE,CAAC,GAAG,CAAC;IACV,mBAAmB,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE,GAAG,EAAE,KAAK,CAAC;IAC7C,WAAW,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,MAAM,CAAC;IAC/B,mBAAmB,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,GAAG,CAAC;IACjE,uBAAuB,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,EAAE,IAAI,CAAC;IAClE,6CAA6C,EAAE;QAC7C,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK;KACzD;IACD,8CAA8C,EAAE;QAC9C,MAAM,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE;QAC1E,EAAE,EAAE,EAAE,EAAE,EAAE;KACX;CACF,CAAA;AAED,MAAM,OAAO,GAAG;IACd,UAAU,EAAE;QACV,KAAK,EAAE,CAAC,GAAG,CAAC;QACZ,GAAG,EAAE,CAAC,GAAG,CAAC;QACV,mBAAmB,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,KAAK,CAAC;QAC/C,WAAW,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,OAAO,CAAC;QAChC,mBAAmB,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,CAAC;QAC/D,uBAAuB,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,EAAE,IAAI,CAAC;QAClE,6CAA6C,EAAE;YAC7C,GAAG,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK;SAClD;QACD,8CAA8C,EAAE;YAC9C,MAAM,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,OAAO,EAAE,EAAE,EAAE,GAAG;YACrE,MAAM,EAAE,MAAM;SACf;KACF;IACD,WAAW,EAAE;QACX,KAAK,EAAE,CAAC,GAAG,CAAC;QACZ,GAAG,EAAE,CAAC,GAAG,CAAC;QACV,mBAAmB,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,KAAK,CAAC;QAC/C,WAAW,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,MAAM,CAAC;QAC/B,mBAAmB,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,GAAG,CAAC;QACnE,uBAAuB,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,EAAE,IAAI,CAAC;QAClE,6CAA6C,EAAE;YAC7C,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,KAAK;SACtC;QACD,8CAA8C,EAAE;YAC9C,MAAM,EAAE,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,MAAM,EAAE,EAAE,EAAE,GAAG;YACtE,KAAK,EAAE,MAAM;SACd;KACF;IACD,SAAS,EAAE,aAAa;IACxB,SAAS,EAAE,aAAa;IACxB,SAAS,EAAE,aAAa;CACe,CAAA;AAEzC,MAAM,aAAa,GAAG;IACpB,kCAAkC;IAClC,aAAa;IAEb,oDAAoD;IACpD,cAAc,4BAAS,YAAY;IAEnC,iCAAiC;IACjC,gBAAgB;IAEhB,sFAAsF;IACtF,yDAAyD;IACzD,eAAe;IAEf,qDAAqD;IACrD,yFAAyF;IACzF,KAAK;CACN,CAAA;AAED,mEAAmE;AACnE,MAAM,SAAS,GAAG,aAAa,EAAE,CAAA;AAEjC,iBAAQ,CAAC,IAAI,CAAC,0BAAa,CAAC,CAAC,IAAI,EAAE,CAAC,YAA0B,EAAE,EAAE;IAChE,MAAM,QAAQ,GAAG,4BAAW,CAAC,cAAc,CAAC,YAAY,EAAE,oCAAe,CAAC,CAAA;IAE1E,MAAM,EACJ,MAAM,EACN,eAAe,EACf,oBAAoB,EACpB,MAAM,EACN,kBAAkB,GACnB,GAAG,QAAQ,CAAA;IAEZ,IAAA,iBAAQ,EAAC,mBAAmB,EAAE,GAAG,EAAE;QACjC,aAAI,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,mBAAmB,EAAE,CAAC,GAAG,EAAE,EAAE;YACpD,IAAA,eAAM,EACJ,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,EAAE,cAAc,EAAE,iCAAkB,EAAE,CAAC,CAAC,CAC5D,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;QAChB,CAAC,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEF,IAAA,iBAAQ,EAAC,qBAAqB,EAAE,GAAG,EAAE;QACnC,MAAM,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC,CAAA;QACpC,IAAA,aAAI,EAAC,cAAc,EAAE,GAAG,EAAE;YACxB,MAAM,GAAG,GAAG,EAAE,CAAA;YACd,IAAA,eAAM,EAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;YAC/B,IAAA,eAAM,EAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;YACxC,IAAA,eAAM,EAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YAC1C,IAAA,eAAM,EAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QAC5C,CAAC,CAAC,CAAA;QAEF,IAAA,aAAI,EAAC,OAAO,EAAE,GAAG,EAAE;YACjB,MAAM,GAAG,GAAG,GAAG,CAAA;YACf,IAAA,eAAM,EAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;YACzC,IAAA,eAAM,EAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;YACxC,IAAA,eAAM,EAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YAC1C,IAAA,eAAM,EAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QAChD,CAAC,CAAC,CAAA;QAEF,IAAA,aAAI,EAAC,KAAK,EAAE,GAAG,EAAE;YACf,MAAM,GAAG,GAAG,IAAI,CAAA;YAChB,IAAA,eAAM,EAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;YACvC,IAAA,eAAM,EAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;QAC1C,CAAC,CAAC,CAAA;QAEF,IAAA,aAAI,EAAC,aAAa,EAAE,GAAG,EAAE;YACvB,MAAM,GAAG,GAAG,mBAAmB,CAAA;YAC/B,IAAA,eAAM,EAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;YACxC,IAAA,eAAM,EAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;YACxC,IAAA,eAAM,EAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YAC9C,IAAA,eAAM,EAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAA;QAC7D,CAAC,CAAC,CAAA;QAEF,IAAA,aAAI,EAAC,kBAAkB,EAAE,GAAG,EAAE;YAC5B,MAAM,GAAG,GAAG,aAAa,CAAA;YACzB,IAAA,eAAM,EAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,WAAW,CAAC,CAAA;YAC/C,IAAA,eAAM,EAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;YACxC,IAAA,eAAM,EAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,CAAA;QACpE,CAAC,CAAC,CAAA;QAEF,IAAA,aAAI,EAAC,QAAQ,EAAE,GAAG,EAAE;YAClB,MAAM,GAAG,GAAG,mBAAmB,CAAA;YAC/B,IAAA,eAAM,EAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;YACxC,IAAA,eAAM,EAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;YACxC,IAAA,eAAM,EAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YAC9C,IAAA,eAAM,EAAC,kBAAkB,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAA;QAC/D,CAAC,CAAC,CAAA;QAEF,IAAA,aAAI,EAAC,qCAAqC,EAAE,GAAG,EAAE;YAC/C,MAAM,GAAG,GAAG,mBAAmB,CAAA;YAC/B,MAAM,SAAS,GAAG,eAAe,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;YAC9C,MAAM,CAAC,uBAAuB,CAAC,OAAO,CAAC,CAAC,KAAa,EAAE,EAAE;gBACvD,IAAA,eAAM,EAAC,SAAS,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YAC5C,CAAC,CAAC,CAAA;QACJ,CAAC,CAAC,CAAA;QAEF,IAAA,aAAI,EAAC,oCAAoC,EAAE,GAAG,EAAE;YAC9C,MAAM,GAAG,GAAG,SAAS,4BAAS,QAAQ,CAAA;YACtC,MAAM,OAAO,GAAG,MAAM,CAAC,GAAG,EAAE;gBAC1B,cAAc,EAAE,iCAAkB;aACnC,CAAC,CAAA;YACF,IAAA,eAAM,EAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;QACtC,CAAC,CAAC,CAAA;QAEF,KAAK,SAAS,CAAC,CAAC,wBAAwB;YACtC,MAAM,GAAG,GAAG,mBAAmB,CAAA;YAC/B,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;gBAChC,4CAA4C;gBAC5C,MAAM,MAAM,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;YACpC,CAAC;QACH,CAAC;QAED,IAAA,aAAI,EAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;YAC3D,MAAM,SAAS,GAAG,oBAAoB,CAAC,wBAAwB,EAAE,CAAC,CAAA;YAClE,MAAM,OAAO,GAAG,CAAC,GAAG,MAAM,CAAC,uBAAuB,CAAC,CAAA;YACnD,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,SAAS,EAAE,CAAC;gBACpC,IAAA,eAAM,EAAC,KAAK,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,CAAA;YACxC,CAAC;QACH,CAAC,CAAC,CAAA;QAEF,IAAA,aAAI,EAAC,sBAAsB,EAAE,GAAG,EAAE;YAChC,MAAM,GAAG,GAAG,6CAA6C,CAAA;YAEzD,IAAA,eAAM,EAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;YACxC,IAAA,eAAM,EAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;QAC1C,CAAC,CAAC,CAAA;QAEF,IAAA,aAAI,EAAC,kBAAkB,EAAE,GAAG,EAAE;YAC5B,MAAM,GAAG,GAAG,8CAA8C,CAAA;YAC1D,IAAA,eAAM,EAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;YACxC,IAAA,eAAM,EAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,aAAa,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;YAC9C,IAAA,eAAM,EAAC,kBAAkB,CAAC,GAAG,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YACnE,IAAA,eAAM,EAAC,kBAAkB,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAA;QAC/D,CAAC,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEF,IAAA,iBAAQ,EAAC,WAAW,EAAE,GAAG,EAAE;QACzB,SAAS,CAAC,YAAY,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,EAAE,EAAE;YACtD,IAAA,aAAI,EAAC,WAAW,MAAM,EAAE,EAAE,GAAG,EAAE;gBAC7B,IAAA,eAAM,EAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAA;YACzC,CAAC,CAAC,CAAA;YACF,IAAA,aAAI,EAAC,WAAW,MAAM,EAAE,EAAE,GAAG,EAAE;gBAC7B,IAAA,eAAM,EAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAA;YACzC,CAAC,CAAC,CAAA;QACJ,CAAC,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;AACJ,CAAC,CAAC,CAAA;AAEF,MAAM,cAAc,GAAG,MAAM,CAAC,IAAI,CAAC,4BAAe,CAA6B,CAAA;AAE/E,MAAM,eAAe,GAAkB;IACrC;QACE,IAAI,EAAE,QAAQ;QACd,OAAO,EACL,qGAAqG;KACxG;IACD;QACE,IAAI,EAAE,QAAQ;QACd,IAAI,EAAE,cAAc;QACpB,OAAO,EAAE,gDAAgD;KAC1D;IACD;QACE,IAAI,EAAE,QAAQ;QACd,IAAI,EAAE,mBAAmB;QACzB,OAAO,EAAE,qDAAqD;KAC/D;IACD;QACE,IAAI,EAAE,QAAQ;QACd,IAAI,EAAE,cAAc;QACpB,OAAO,EACL,sGAAsG;KACzG;IACD;QACE,IAAI,EAAE,QAAQ;QACd,IAAI,EAAE,mBAAmB;QACzB,OAAO,EAAE,+DAA+D;KACzE;IACD;QACE,IAAI,EAAE,MAAM;QACZ,OAAO,EACL,wFAAwF;KAC3F;CACO,CAAA;AAEV,iBAAQ,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,IAAI,EAAE,CAAC,SAAS,EAAE,EAAE;IAChD,MAAM,QAAQ,GAAG,4BAAW,CAAC,sBAAsB,CACjD,SAAS,EACT,oCAAe,CAChB,CAAA;IACD,MAAM,qBAAqB,GAAG,SAAS,CAAC,UAAU,CAAC,eAAe,CAAC;QACjE,CAAC,CAAC,GAAG;QACL,CAAC,CAAC,SAAS,CAAC,UAAU,CAAC,QAAQ,CAAC;YAChC,CAAC,CAAC,GAAG;YACL,CAAC,CAAC,GAAG,CAAA;IAEP,IAAA,iBAAQ,EAAC,oBAAoB,EAAE,GAAG,EAAE;QAClC,IAAA,aAAI,EAAC,0BAA0B,EAAE,GAAG,EAAE;YACpC,MAAM,OAAO,GAAG,QAAQ,CAAC,UAAU,CAAC,eAAe,CAAC,CAAA;YACpD,IAAA,eAAM,EAAC,OAAO,CAAC,CAAC,eAAe,EAAE,CAAA;YACjC,IAAA,eAAM,EAAC,OAAO,CAAC,CAAC,YAAY,CAAC,qBAAqB,CAAC,CAAA;YAEnD,MAAM,OAAO,GAAG,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;YACxC,IAAA,eAAM,EAAC,OAAO,CAAC,CAAC,eAAe,EAAE,CAAA;QACnC,CAAC,CAAC,CAAA;QAEF,IAAA,aAAI,EAAC,2BAA2B,EAAE,GAAG,EAAE;YACrC,MAAM,kBAAkB,GAAG,QAAQ,CAAC,kBAAkB,CACpD,eAAe,EACf,EAAE,CACH,CAAA;YACD,IAAA,eAAM,EAAC,kBAAkB,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QACxC,CAAC,CAAC,CAAA;QACF,IAAA,aAAI,EAAC,mCAAmC,EAAE,GAAG,EAAE;YAC7C,MAAM,kBAAkB,GAAG,QAAQ,CAAC,kBAAkB,CACpD,eAAe,EACf,GAAG,CACJ,CAAA;YACD,IAAA,eAAM,EAAC,kBAAkB,CAAC,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAA;QACxD,CAAC,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;AACJ,CAAC,CAAC,CAAA;AAEF,IAAA,iBAAQ,EAAC,4BAA4B,EAAE,GAAG,EAAE;IAC1C,MAAM,aAAa,GAAG,4BAAW,CAAC,sBAAsB,CACtD,QAAQ,EACR,oCAAe,CAChB,CAAA;IACD,MAAM,aAAa,GAAG,4BAAW,CAAC,sBAAsB,CACtD,eAAe,EACf,oCAAe,CAChB,CAAA;IAED,IAAA,aAAI,EAAC,2CAA2C,EAAE,GAAG,EAAE;QACrD,MAAM,UAAU,GAAG,KAAK,CAAA;QACxB,MAAM,IAAI,GAAG,aAAa,CAAC,YAAY,CAAC,UAAU,CAAC,CAAA;QAEnD,qFAAqF;QACrF,IAAA,eAAM,EAAC,IAAI,CAAC,KAAK,CAAC,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC,CAAC,CAAA,CAAC,iBAAiB;QAC5D,IAAA,eAAM,EAAC,IAAI,CAAC,MAAM,CAAC,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC,CAAA,CAAC,gBAAgB;QACzD,IAAA,eAAM,EAAC,IAAI,CAAC,UAAU,CAAC,CAAC,WAAW,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAA,CAAC,kBAAkB;QACnE,IAAA,eAAM,EAAC,IAAI,CAAC,WAAW,CAAC,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC,CAAC,CAAA,CAAC,eAAe;IAChE,CAAC,CAAC,CAAA;IAEF,IAAA,aAAI,EAAC,kDAAkD,EAAE,GAAG,EAAE;QAC5D,MAAM,UAAU,GAAG,KAAK,CAAA;QACxB,MAAM,IAAI,GAAG,aAAa,CAAC,YAAY,CAAC,UAAU,CAAC,CAAA;QAEnD,6FAA6F;QAC7F,IAAA,eAAM,EAAC,IAAI,CAAC,KAAK,CAAC,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC,CAAC,CAAA,CAAC,iBAAiB;QAC5D,IAAA,eAAM,EAAC,IAAI,CAAC,MAAM,CAAC,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC,CAAC,CAAA,CAAC,iBAAiB;QAC7D,IAAA,eAAM,EAAC,IAAI,CAAC,UAAU,CAAC,CAAC,WAAW,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAA,CAAC,kBAAkB;QACnE,IAAA,eAAM,EAAC,IAAI,CAAC,WAAW,CAAC,CAAC,WAAW,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAA,CAAC,kBAAkB;IACtE,CAAC,CAAC,CAAA;IAEF,IAAA,aAAI,EAAC,8BAA8B,EAAE,GAAG,EAAE;QACxC,MAAM,UAAU,GAAG,KAAK,CAAA;QACxB,iEAAiE;QACjE,MAAM,IAAI,GAAG,aAAa,CAAC,YAAY,CAAC,UAAU,EAAE,eAAe,CAAC,CAAA;QAEpE,IAAA,eAAM,EAAC,IAAI,CAAC,KAAK,CAAC,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC,CAAC,CAAA,CAAC,iBAAiB;QAC5D,IAAA,eAAM,EAAC,IAAI,CAAC,MAAM,CAAC,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC,CAAC,CAAA,CAAC,iBAAiB;IAC/D,CAAC,CAAC,CAAA;IAEF,IAAA,aAAI,EAAC,8CAA8C,EAAE,GAAG,EAAE;QACxD,MAAM,QAAQ,GAAG,4BAAW,CAAC,cAAc,CAAC,aAAa,EAAE,oCAAe,CAAC,CAAA;QAC3E,MAAM,UAAU,GAAG,KAAK,CAAA;QAExB,oEAAoE;QACpE,IAAA,eAAM,EAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,YAAY,CAAC,UAAU,CAAC,CAAC,CAAC,OAAO,CACrD,sFAAsF,CACvF,CAAA;IACH,CAAC,CAAC,CAAA;IAEF,IAAA,aAAI,EAAC,gCAAgC,EAAE,GAAG,EAAE;QAC1C,MAAM,UAAU,GAAG,KAAK,CAAA;QACxB,IAAA,eAAM,EAAC,GAAG,EAAE,CACV,aAAa,CAAC,YAAY,CAAC,UAAU,EAAE,oBAA2B,CAAC,CACpE,CAAC,OAAO,CAAC,mCAAmC,CAAC,CAAA;IAChD,CAAC,CAAC,CAAA;IAEF,IAAA,aAAI,EAAC,mDAAmD,EAAE,GAAG,EAAE;QAC7D,2DAA2D;QAC3D,MAAM,kBAAkB,GAAG,MAAM,CAAC,OAAO,CAAC,kBAAM,CAAC,CAAC,IAAI,CACpD,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,EAAE,EAAE,CACb,KAAK,CAAC,IAAI,EAAE,KAAK,KAAK,SAAS,IAAI,KAAK,CAAC,IAAI,EAAE,MAAM,KAAK,SAAS,CACtE,CAAA;QAED,IAAI,kBAAkB,EAAE,CAAC;YACvB,MAAM,CAAC,SAAS,CAAC,GAAG,kBAAkB,CAAA;YACtC,MAAM,IAAI,GAAG,aAAa,CAAC,YAAY,CAAC,KAAK,EAAE,SAAgB,CAAC,CAAA;YAEhE,IAAA,eAAM,EAAC,IAAI,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE,CAAA;YAChC,IAAA,eAAM,EAAC,IAAI,CAAC,MAAM,CAAC,CAAC,aAAa,EAAE,CAAA;QACrC,CAAC;aAAM,CAAC;YACN,kDAAkD;YAClD,OAAO,CAAC,GAAG,CAAC,oDAAoD,CAAC,CAAA;QACnE,CAAC;IACH,CAAC,CAAC,CAAA;AACJ,CAAC,CAAC,CAAA;AAEF,SAAS,aAAa;IACpB,MAAM,YAAY,GAAG,cAAI,CAAC,IAAI,CAAC,SAAS,EAAE,uBAAuB,CAAC,CAAA;IAClE,MAAM,YAAY,GAAG,YAAE,CAAC,YAAY,CAAC,YAAY,EAAE,MAAM,CAAC,CAAA;IAC1D,MAAM,KAAK,GAGP;QACF,WAAW,EAAE,EAAE;QACf,SAAS,EAAE,EAAE;QACb,SAAS,EAAE,EAAE;QACb,SAAS,EAAE,EAAE;QACb,UAAU,EAAE,EAAE;KACf,CAAA;IACD,YAAY,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,EAAE;QAC9C,MAAM,CAAC,gBAAgB,EAAE,UAAU,EAAE,WAAW,CAAC,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;QACxE,IAAI,CAAC,gBAAgB,IAAI,CAAC,UAAU,IAAI,CAAC,WAAW;YAAE,OAAM;QAC5D,MAAM,YAAY,GAAG,gBAAgB,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAiB,CAAA;QACpE,KAAK,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC;YACvB,MAAM,EAAE,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;YACxD,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC;SACxD,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IACF,OAAO,KAAK,CAAA;AACd,CAAC"}

package/cjs/encoding/cl100k_base.d.ts

@@ -2,6 +2,6 @@ import { GptEncoding } from '../GptEncoding.js';
 export * from '../constants.js';
 export * from '../specialTokens.js';
 declare const api: GptEncoding;
-declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, void>, encode: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, tokenLimit: number) => false | number, countTokens: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number, encodeChat: (chat: readonly import("../GptEncoding.js").ChatMessage[], model?: "gpt-4o" | "gpt-3.5-turbo" | "gpt-3.5-turbo-instruct" | "babbage-002" | "davinci-002" | "text-embedding-3-small" | "o1-preview" | "o1-preview-2024-09-12" | "o1-mini" | "o1-mini-2024-09-12" | "chatgpt-4o-latest" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o-mini" | "gpt-4o-mini-2024-07-18" | "gpt-4o-realtime-preview" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-2024-08-06-finetune" | "gpt-4o-mini-2024-07-18-finetune" | "gpt-4o-mini-training" | "gpt-4o-mini-training-2024-07-18" | "gpt-4-turbo" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo-preview" | "gpt-4-0125-preview" | "gpt-4-1106-preview" | "gpt-4" | "gpt-4-0613" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo-finetune" | "gpt-3.5-turbo-16k" | "gpt-4-32k" | "gpt-4-32k-0613" | "gpt-4-vision-preview" | "gpt-4-1106-vision-preview" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo-0613" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-0301" | "text-embedding-3-large" | "text-embedding-ada-002" | "gpt-3.5-turbo-instruct-0914" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-002" | "code-davinci-001" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-002" | "code-cushman-001" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | undefined) => number[], encodeChatGenerator: (chat: Iterable<import("../GptEncoding.js").ChatMessage>, model?: "gpt-4o" | "gpt-3.5-turbo" | "gpt-3.5-turbo-instruct" | "babbage-002" | "davinci-002" | "text-embedding-3-small" | "o1-preview" | "o1-preview-2024-09-12" | "o1-mini" | "o1-mini-2024-09-12" | "chatgpt-4o-latest" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o-mini" | "gpt-4o-mini-2024-07-18" | "gpt-4o-realtime-preview" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-2024-08-06-finetune" | "gpt-4o-mini-2024-07-18-finetune" | "gpt-4o-mini-training" | "gpt-4o-mini-training-2024-07-18" | "gpt-4-turbo" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo-preview" | "gpt-4-0125-preview" | "gpt-4-1106-preview" | "gpt-4" | "gpt-4-0613" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo-finetune" | "gpt-3.5-turbo-16k" | "gpt-4-32k" | "gpt-4-32k-0613" | "gpt-4-vision-preview" | "gpt-4-1106-vision-preview" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo-0613" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-0301" | "text-embedding-3-large" | "text-embedding-ada-002" | "gpt-3.5-turbo-instruct-0914" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-002" | "code-davinci-001" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-002" | "code-cushman-001" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | undefined) => Generator<number[], void, undefined>, vocabularySize: number, setMergeCacheSize: (size: number) => void, clearMergeCache: () => void;
-export { clearMergeCache, countTokens, decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, setMergeCacheSize, vocabularySize, };
+declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, void>, encode: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, tokenLimit: number) => false | number, countTokens: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number, encodeChat: (chat: readonly import("../GptEncoding.js").ChatMessage[], model?: "gpt-4o" | "gpt-3.5-turbo" | "gpt-3.5-turbo-instruct" | "babbage-002" | "davinci-002" | "text-embedding-3-small" | "o1" | "o1-2024-12-17" | "o1-preview" | "o1-preview-2024-09-12" | "o1-mini" | "o1-mini-2024-09-12" | "o3-mini" | "chatgpt-4o-latest" | "gpt-4o-2024-11-20" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o-mini" | "gpt-4o-mini-2024-07-18" | "gpt-4o-realtime-preview" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-realtime-preview-2024-12-17" | "gpt-4o-mini-realtime-preview" | "gpt-4o-mini-realtime-preview-2024-12-17" | "gpt-4o-audio-preview" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-audio-preview-2024-12-17" | "gpt-4o-mini-audio-preview" | "gpt-4o-mini-audio-preview-2024-12-17" | "gpt-4o-2024-08-06-finetune" | "gpt-4o-mini-2024-07-18-finetune" | "gpt-4o-mini-training" | "gpt-4o-mini-training-2024-07-18" | "davinci-002-finetune" | "babbage-002-finetune" | "gpt-4-turbo" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo-preview" | "gpt-4-0125-preview" | "gpt-4-1106-preview" | "gpt-4" | "gpt-4-0613" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo-finetune" | "gpt-3.5-turbo-16k" | "gpt-4-32k" | "gpt-4-32k-0613" | "gpt-4-vision-preview" | "gpt-4-1106-vision-preview" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo-0613" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-0301" | "text-embedding-3-large" | "text-embedding-ada-002" | "gpt-3.5-turbo-instruct-0914" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-002" | "code-davinci-001" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-002" | "code-cushman-001" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | undefined) => number[], encodeChatGenerator: (chat: Iterable<import("../GptEncoding.js").ChatMessage>, model?: "gpt-4o" | "gpt-3.5-turbo" | "gpt-3.5-turbo-instruct" | "babbage-002" | "davinci-002" | "text-embedding-3-small" | "o1" | "o1-2024-12-17" | "o1-preview" | "o1-preview-2024-09-12" | "o1-mini" | "o1-mini-2024-09-12" | "o3-mini" | "chatgpt-4o-latest" | "gpt-4o-2024-11-20" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o-mini" | "gpt-4o-mini-2024-07-18" | "gpt-4o-realtime-preview" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-realtime-preview-2024-12-17" | "gpt-4o-mini-realtime-preview" | "gpt-4o-mini-realtime-preview-2024-12-17" | "gpt-4o-audio-preview" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-audio-preview-2024-12-17" | "gpt-4o-mini-audio-preview" | "gpt-4o-mini-audio-preview-2024-12-17" | "gpt-4o-2024-08-06-finetune" | "gpt-4o-mini-2024-07-18-finetune" | "gpt-4o-mini-training" | "gpt-4o-mini-training-2024-07-18" | "davinci-002-finetune" | "babbage-002-finetune" | "gpt-4-turbo" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo-preview" | "gpt-4-0125-preview" | "gpt-4-1106-preview" | "gpt-4" | "gpt-4-0613" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo-finetune" | "gpt-3.5-turbo-16k" | "gpt-4-32k" | "gpt-4-32k-0613" | "gpt-4-vision-preview" | "gpt-4-1106-vision-preview" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo-0613" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-0301" | "text-embedding-3-large" | "text-embedding-ada-002" | "gpt-3.5-turbo-instruct-0914" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-002" | "code-davinci-001" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-002" | "code-cushman-001" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | undefined) => Generator<number[], void, undefined>, vocabularySize: number, setMergeCacheSize: (size: number) => void, clearMergeCache: () => void, estimateCost: (tokenCount: number, modelName?: "gpt-4o" | "gpt-3.5-turbo" | "gpt-3.5-turbo-instruct" | "babbage-002" | "davinci-002" | "text-embedding-3-small" | "o1" | "o1-2024-12-17" | "o1-preview" | "o1-preview-2024-09-12" | "o1-mini" | "o1-mini-2024-09-12" | "o3-mini" | "chatgpt-4o-latest" | "gpt-4o-2024-11-20" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o-mini" | "gpt-4o-mini-2024-07-18" | "gpt-4o-realtime-preview" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-realtime-preview-2024-12-17" | "gpt-4o-mini-realtime-preview" | "gpt-4o-mini-realtime-preview-2024-12-17" | "gpt-4o-audio-preview" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-audio-preview-2024-12-17" | "gpt-4o-mini-audio-preview" | "gpt-4o-mini-audio-preview-2024-12-17" | "gpt-4o-2024-08-06-finetune" | "gpt-4o-mini-2024-07-18-finetune" | "gpt-4o-mini-training" | "gpt-4o-mini-training-2024-07-18" | "davinci-002-finetune" | "babbage-002-finetune" | "gpt-4-turbo" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo-preview" | "gpt-4-0125-preview" | "gpt-4-1106-preview" | "gpt-4" | "gpt-4-0613" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo-finetune" | "gpt-3.5-turbo-16k" | "gpt-4-32k" | "gpt-4-32k-0613" | "gpt-4-vision-preview" | "gpt-4-1106-vision-preview" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo-0613" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-0301" | "text-embedding-3-large" | "text-embedding-ada-002" | "gpt-3.5-turbo-instruct-0914" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-002" | "code-davinci-001" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-002" | "code-cushman-001" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | undefined) => import("../models.js").CostEstimate;
+export { clearMergeCache, countTokens, decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, estimateCost, isWithinTokenLimit, setMergeCacheSize, vocabularySize, };
 export default api;

package/cjs/encoding/cl100k_base.js

@@ -17,14 +17,14 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.vocabularySize = exports.setMergeCacheSize = exports.isWithinTokenLimit = exports.encodeGenerator = exports.encodeChatGenerator = exports.encodeChat = exports.encode = exports.decodeGenerator = exports.decodeAsyncGenerator = exports.decode = exports.countTokens = exports.clearMergeCache = void 0;
+exports.vocabularySize = exports.setMergeCacheSize = exports.isWithinTokenLimit = exports.estimateCost = exports.encodeGenerator = exports.encodeChatGenerator = exports.encodeChat = exports.encode = exports.decodeGenerator = exports.decodeAsyncGenerator = exports.decode = exports.countTokens = exports.clearMergeCache = void 0;
 /* eslint-disable import/extensions */
 const cl100k_base_js_1 = __importDefault(require("../bpeRanks/cl100k_base.js"));
 const GptEncoding_js_1 = require("../GptEncoding.js");
 __exportStar(require("../constants.js"), exports);
 __exportStar(require("../specialTokens.js"), exports);
 const api = GptEncoding_js_1.GptEncoding.getEncodingApi('cl100k_base', () => cl100k_base_js_1.default);
-const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, countTokens, encodeChat, encodeChatGenerator, vocabularySize, setMergeCacheSize, clearMergeCache, } = api;
+const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, countTokens, encodeChat, encodeChatGenerator, vocabularySize, setMergeCacheSize, clearMergeCache, estimateCost, } = api;
 exports.decode = decode;
 exports.decodeAsyncGenerator = decodeAsyncGenerator;
 exports.decodeGenerator = decodeGenerator;
@@ -37,6 +37,7 @@ exports.encodeChatGenerator = encodeChatGenerator;
 exports.vocabularySize = vocabularySize;
 exports.setMergeCacheSize = setMergeCacheSize;
 exports.clearMergeCache = clearMergeCache;
+exports.estimateCost = estimateCost;
 // eslint-disable-next-line import/no-default-export
 exports.default = api;
 //# sourceMappingURL=cl100k_base.js.map

package/cjs/encoding/cl100k_base.js.map

@@ -1 +1 @@
-{"version":3,"file":"cl100k_base.js","sourceRoot":"","sources":["../../src/encoding/cl100k_base.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA,sCAAsC;AACtC,gFAAiD;AACjD,sDAA+C;AAE/C,kDAA+B;AAC/B,sDAAmC;AAEnC,MAAM,GAAG,GAAG,4BAAW,CAAC,cAAc,CAAC,aAAa,EAAE,GAAG,EAAE,CAAC,wBAAQ,CAAC,CAAA;AACrE,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,EAClB,WAAW,EACX,UAAU,EACV,mBAAmB,EACnB,cAAc,EACd,iBAAiB,EACjB,eAAe,GAChB,GAAG,GAAG,CAAA;AAIL,wBAAM;AACN,oDAAoB;AACpB,0CAAe;AACf,wBAAM;AAGN,0CAAe;AACf,gDAAkB;AARlB,kCAAW;AAKX,gCAAU;AACV,kDAAmB;AAInB,wCAAc;AADd,8CAAiB;AAVjB,0CAAe;AAajB,oDAAoD;AACpD,kBAAe,GAAG,CAAA"}
+{"version":3,"file":"cl100k_base.js","sourceRoot":"","sources":["../../src/encoding/cl100k_base.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA,sCAAsC;AACtC,gFAAiD;AACjD,sDAA+C;AAE/C,kDAA+B;AAC/B,sDAAmC;AAEnC,MAAM,GAAG,GAAG,4BAAW,CAAC,cAAc,CAAC,aAAa,EAAE,GAAG,EAAE,CAAC,wBAAQ,CAAC,CAAA;AACrE,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,EAClB,WAAW,EACX,UAAU,EACV,mBAAmB,EACnB,cAAc,EACd,iBAAiB,EACjB,eAAe,EACf,YAAY,GACb,GAAG,GAAG,CAAA;AAIL,wBAAM;AACN,oDAAoB;AACpB,0CAAe;AACf,wBAAM;AAGN,0CAAe;AAEf,gDAAkB;AATlB,kCAAW;AAKX,gCAAU;AACV,kDAAmB;AAKnB,wCAAc;AADd,8CAAiB;AAXjB,0CAAe;AASf,oCAAY;AAKd,oDAAoD;AACpD,kBAAe,GAAG,CAAA"}