gpt-tokenizer 2.6.0 → 2.6.2

package/esm/model/gpt-4o-realtime-preview-2024-10-01.d.ts

@@ -1,6 +1,7 @@
 import { GptEncoding } from '../GptEncoding.js';
+export * from '../constants.js';
 export * from '../specialTokens.js';
 declare const api: GptEncoding;
-declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, void>, encode: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, tokenLimit: number) => false | number, encodeChat: (chat: readonly import("../GptEncoding.js").ChatMessage[], model?: "gpt-4o" | "gpt-3.5-turbo" | "gpt-3.5-turbo-instruct" | "babbage-002" | "davinci-002" | "text-embedding-3-small" | "o1-preview" | "o1-preview-2024-09-12" | "o1-mini" | "o1-mini-2024-09-12" | "chatgpt-4o-latest" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o-mini" | "gpt-4o-mini-2024-07-18" | "gpt-4o-realtime-preview" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-2024-08-06-finetune" | "gpt-4o-mini-2024-07-18-finetune" | "gpt-4o-mini-training" | "gpt-4o-mini-training-2024-07-18" | "gpt-4-turbo" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo-preview" | "gpt-4-0125-preview" | "gpt-4-1106-preview" | "gpt-4" | "gpt-4-0613" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo-finetune" | "gpt-3.5-turbo-16k" | "gpt-4-32k" | "gpt-4-32k-0613" | "gpt-4-vision-preview" | "gpt-4-1106-vision-preview" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo-0613" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-0301" | "text-embedding-3-large" | "text-embedding-ada-002" | "gpt-3.5-turbo-instruct-0914" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-002" | "code-davinci-001" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-002" | "code-cushman-001" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | undefined) => number[], encodeChatGenerator: (chat: Iterable<import("../GptEncoding.js").ChatMessage>, model?: "gpt-4o" | "gpt-3.5-turbo" | "gpt-3.5-turbo-instruct" | "babbage-002" | "davinci-002" | "text-embedding-3-small" | "o1-preview" | "o1-preview-2024-09-12" | "o1-mini" | "o1-mini-2024-09-12" | "chatgpt-4o-latest" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o-mini" | "gpt-4o-mini-2024-07-18" | "gpt-4o-realtime-preview" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-2024-08-06-finetune" | "gpt-4o-mini-2024-07-18-finetune" | "gpt-4o-mini-training" | "gpt-4o-mini-training-2024-07-18" | "gpt-4-turbo" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo-preview" | "gpt-4-0125-preview" | "gpt-4-1106-preview" | "gpt-4" | "gpt-4-0613" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo-finetune" | "gpt-3.5-turbo-16k" | "gpt-4-32k" | "gpt-4-32k-0613" | "gpt-4-vision-preview" | "gpt-4-1106-vision-preview" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo-0613" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-0301" | "text-embedding-3-large" | "text-embedding-ada-002" | "gpt-3.5-turbo-instruct-0914" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-002" | "code-davinci-001" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-002" | "code-cushman-001" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | undefined) => Generator<number[], void, undefined>;
-export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, };
+declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, void>, encode: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, tokenLimit: number) => false | number, encodeChat: (chat: readonly import("../GptEncoding.js").ChatMessage[], model?: "gpt-4o" | "gpt-3.5-turbo" | "gpt-3.5-turbo-instruct" | "babbage-002" | "davinci-002" | "text-embedding-3-small" | "o1-preview" | "o1-preview-2024-09-12" | "o1-mini" | "o1-mini-2024-09-12" | "chatgpt-4o-latest" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o-mini" | "gpt-4o-mini-2024-07-18" | "gpt-4o-realtime-preview" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-2024-08-06-finetune" | "gpt-4o-mini-2024-07-18-finetune" | "gpt-4o-mini-training" | "gpt-4o-mini-training-2024-07-18" | "gpt-4-turbo" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo-preview" | "gpt-4-0125-preview" | "gpt-4-1106-preview" | "gpt-4" | "gpt-4-0613" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo-finetune" | "gpt-3.5-turbo-16k" | "gpt-4-32k" | "gpt-4-32k-0613" | "gpt-4-vision-preview" | "gpt-4-1106-vision-preview" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo-0613" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-0301" | "text-embedding-3-large" | "text-embedding-ada-002" | "gpt-3.5-turbo-instruct-0914" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-002" | "code-davinci-001" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-002" | "code-cushman-001" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | undefined) => number[], encodeChatGenerator: (chat: Iterable<import("../GptEncoding.js").ChatMessage>, model?: "gpt-4o" | "gpt-3.5-turbo" | "gpt-3.5-turbo-instruct" | "babbage-002" | "davinci-002" | "text-embedding-3-small" | "o1-preview" | "o1-preview-2024-09-12" | "o1-mini" | "o1-mini-2024-09-12" | "chatgpt-4o-latest" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o-mini" | "gpt-4o-mini-2024-07-18" | "gpt-4o-realtime-preview" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-2024-08-06-finetune" | "gpt-4o-mini-2024-07-18-finetune" | "gpt-4o-mini-training" | "gpt-4o-mini-training-2024-07-18" | "gpt-4-turbo" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo-preview" | "gpt-4-0125-preview" | "gpt-4-1106-preview" | "gpt-4" | "gpt-4-0613" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo-finetune" | "gpt-3.5-turbo-16k" | "gpt-4-32k" | "gpt-4-32k-0613" | "gpt-4-vision-preview" | "gpt-4-1106-vision-preview" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo-0613" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-0301" | "text-embedding-3-large" | "text-embedding-ada-002" | "gpt-3.5-turbo-instruct-0914" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-002" | "code-davinci-001" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-002" | "code-cushman-001" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | undefined) => Generator<number[], void, undefined>, vocabularySize: number;
+export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, vocabularySize, };
 export default api;

package/esm/model/gpt-4o-realtime-preview-2024-10-01.js

@@ -1,11 +1,12 @@
 /* eslint-disable import/extensions */
 import bpeRanks from '../bpeRanks/o200k_base.js';
 import { GptEncoding } from '../GptEncoding.js';
+export * from '../constants.js';
 export * from '../specialTokens.js';
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-4o-realtime-preview-2024-10-01', () => bpeRanks);
-const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, encodeChat, encodeChatGenerator, } = api;
-export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, };
+const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, encodeChat, encodeChatGenerator, vocabularySize, } = api;
+export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, vocabularySize, };
 // eslint-disable-next-line import/no-default-export
 export default api;
 //# sourceMappingURL=gpt-4o-realtime-preview-2024-10-01.js.map

package/esm/model/gpt-4o-realtime-preview-2024-10-01.js.map

@@ -1 +1 @@
-{"version":3,"file":"gpt-4o-realtime-preview-2024-10-01.js","sourceRoot":"","sources":["../../src/model/gpt-4o-realtime-preview-2024-10-01.ts"],"names":[],"mappings":"AAAA,sCAAsC;AACtC,OAAO,QAAQ,MAAM,2BAA2B,CAAA;AAChD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAA;AAE/C,cAAc,qBAAqB,CAAA;AACnC,kBAAkB;AAClB,MAAM,GAAG,GAAG,WAAW,CAAC,sBAAsB,CAAC,oCAAoC,EAAE,GAAG,EAAE,CAAC,QAAQ,CAAC,CAAA;AACpG,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,EAClB,UAAU,EACV,mBAAmB,GACpB,GAAG,GAAG,CAAA;AACP,OAAO,EACL,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,UAAU,EACV,mBAAmB,EACnB,eAAe,EACf,kBAAkB,GACnB,CAAA;AACD,oDAAoD;AACpD,eAAe,GAAG,CAAA"}
+{"version":3,"file":"gpt-4o-realtime-preview-2024-10-01.js","sourceRoot":"","sources":["../../src/model/gpt-4o-realtime-preview-2024-10-01.ts"],"names":[],"mappings":"AAAA,sCAAsC;AACtC,OAAO,QAAQ,MAAM,2BAA2B,CAAA;AAChD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAA;AAE/C,cAAc,iBAAiB,CAAA;AAC/B,cAAc,qBAAqB,CAAA;AACnC,kBAAkB;AAClB,MAAM,GAAG,GAAG,WAAW,CAAC,sBAAsB,CAAC,oCAAoC,EAAE,GAAG,EAAE,CAAC,QAAQ,CAAC,CAAA;AACpG,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,EAClB,UAAU,EACV,mBAAmB,EACnB,cAAc,GACf,GAAG,GAAG,CAAA;AACP,OAAO,EACL,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,UAAU,EACV,mBAAmB,EACnB,eAAe,EACf,kBAAkB,EAClB,cAAc,GACf,CAAA;AACD,oDAAoD;AACpD,eAAe,GAAG,CAAA"}

package/esm/model/gpt-4o-realtime-preview.d.ts

@@ -1,6 +1,7 @@
 import { GptEncoding } from '../GptEncoding.js';
+export * from '../constants.js';
 export * from '../specialTokens.js';
 declare const api: GptEncoding;
-declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, void>, encode: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, tokenLimit: number) => false | number, encodeChat: (chat: readonly import("../GptEncoding.js").ChatMessage[], model?: "gpt-4o" | "gpt-3.5-turbo" | "gpt-3.5-turbo-instruct" | "babbage-002" | "davinci-002" | "text-embedding-3-small" | "o1-preview" | "o1-preview-2024-09-12" | "o1-mini" | "o1-mini-2024-09-12" | "chatgpt-4o-latest" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o-mini" | "gpt-4o-mini-2024-07-18" | "gpt-4o-realtime-preview" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-2024-08-06-finetune" | "gpt-4o-mini-2024-07-18-finetune" | "gpt-4o-mini-training" | "gpt-4o-mini-training-2024-07-18" | "gpt-4-turbo" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo-preview" | "gpt-4-0125-preview" | "gpt-4-1106-preview" | "gpt-4" | "gpt-4-0613" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo-finetune" | "gpt-3.5-turbo-16k" | "gpt-4-32k" | "gpt-4-32k-0613" | "gpt-4-vision-preview" | "gpt-4-1106-vision-preview" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo-0613" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-0301" | "text-embedding-3-large" | "text-embedding-ada-002" | "gpt-3.5-turbo-instruct-0914" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-002" | "code-davinci-001" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-002" | "code-cushman-001" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | undefined) => number[], encodeChatGenerator: (chat: Iterable<import("../GptEncoding.js").ChatMessage>, model?: "gpt-4o" | "gpt-3.5-turbo" | "gpt-3.5-turbo-instruct" | "babbage-002" | "davinci-002" | "text-embedding-3-small" | "o1-preview" | "o1-preview-2024-09-12" | "o1-mini" | "o1-mini-2024-09-12" | "chatgpt-4o-latest" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o-mini" | "gpt-4o-mini-2024-07-18" | "gpt-4o-realtime-preview" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-2024-08-06-finetune" | "gpt-4o-mini-2024-07-18-finetune" | "gpt-4o-mini-training" | "gpt-4o-mini-training-2024-07-18" | "gpt-4-turbo" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo-preview" | "gpt-4-0125-preview" | "gpt-4-1106-preview" | "gpt-4" | "gpt-4-0613" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo-finetune" | "gpt-3.5-turbo-16k" | "gpt-4-32k" | "gpt-4-32k-0613" | "gpt-4-vision-preview" | "gpt-4-1106-vision-preview" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo-0613" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-0301" | "text-embedding-3-large" | "text-embedding-ada-002" | "gpt-3.5-turbo-instruct-0914" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-002" | "code-davinci-001" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-002" | "code-cushman-001" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | undefined) => Generator<number[], void, undefined>;
-export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, };
+declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, void>, encode: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, tokenLimit: number) => false | number, encodeChat: (chat: readonly import("../GptEncoding.js").ChatMessage[], model?: "gpt-4o" | "gpt-3.5-turbo" | "gpt-3.5-turbo-instruct" | "babbage-002" | "davinci-002" | "text-embedding-3-small" | "o1-preview" | "o1-preview-2024-09-12" | "o1-mini" | "o1-mini-2024-09-12" | "chatgpt-4o-latest" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o-mini" | "gpt-4o-mini-2024-07-18" | "gpt-4o-realtime-preview" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-2024-08-06-finetune" | "gpt-4o-mini-2024-07-18-finetune" | "gpt-4o-mini-training" | "gpt-4o-mini-training-2024-07-18" | "gpt-4-turbo" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo-preview" | "gpt-4-0125-preview" | "gpt-4-1106-preview" | "gpt-4" | "gpt-4-0613" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo-finetune" | "gpt-3.5-turbo-16k" | "gpt-4-32k" | "gpt-4-32k-0613" | "gpt-4-vision-preview" | "gpt-4-1106-vision-preview" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo-0613" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-0301" | "text-embedding-3-large" | "text-embedding-ada-002" | "gpt-3.5-turbo-instruct-0914" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-002" | "code-davinci-001" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-002" | "code-cushman-001" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | undefined) => number[], encodeChatGenerator: (chat: Iterable<import("../GptEncoding.js").ChatMessage>, model?: "gpt-4o" | "gpt-3.5-turbo" | "gpt-3.5-turbo-instruct" | "babbage-002" | "davinci-002" | "text-embedding-3-small" | "o1-preview" | "o1-preview-2024-09-12" | "o1-mini" | "o1-mini-2024-09-12" | "chatgpt-4o-latest" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o-mini" | "gpt-4o-mini-2024-07-18" | "gpt-4o-realtime-preview" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-2024-08-06-finetune" | "gpt-4o-mini-2024-07-18-finetune" | "gpt-4o-mini-training" | "gpt-4o-mini-training-2024-07-18" | "gpt-4-turbo" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo-preview" | "gpt-4-0125-preview" | "gpt-4-1106-preview" | "gpt-4" | "gpt-4-0613" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo-finetune" | "gpt-3.5-turbo-16k" | "gpt-4-32k" | "gpt-4-32k-0613" | "gpt-4-vision-preview" | "gpt-4-1106-vision-preview" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo-0613" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-0301" | "text-embedding-3-large" | "text-embedding-ada-002" | "gpt-3.5-turbo-instruct-0914" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-002" | "code-davinci-001" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-002" | "code-cushman-001" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | undefined) => Generator<number[], void, undefined>, vocabularySize: number;
+export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, vocabularySize, };
 export default api;

package/esm/model/gpt-4o-realtime-preview.js

@@ -1,11 +1,12 @@
 /* eslint-disable import/extensions */
 import bpeRanks from '../bpeRanks/o200k_base.js';
 import { GptEncoding } from '../GptEncoding.js';
+export * from '../constants.js';
 export * from '../specialTokens.js';
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-4o-realtime-preview', () => bpeRanks);
-const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, encodeChat, encodeChatGenerator, } = api;
-export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, };
+const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, encodeChat, encodeChatGenerator, vocabularySize, } = api;
+export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, vocabularySize, };
 // eslint-disable-next-line import/no-default-export
 export default api;
 //# sourceMappingURL=gpt-4o-realtime-preview.js.map

package/esm/model/gpt-4o-realtime-preview.js.map

@@ -1 +1 @@
-{"version":3,"file":"gpt-4o-realtime-preview.js","sourceRoot":"","sources":["../../src/model/gpt-4o-realtime-preview.ts"],"names":[],"mappings":"AAAA,sCAAsC;AACtC,OAAO,QAAQ,MAAM,2BAA2B,CAAA;AAChD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAA;AAE/C,cAAc,qBAAqB,CAAA;AACnC,kBAAkB;AAClB,MAAM,GAAG,GAAG,WAAW,CAAC,sBAAsB,CAAC,yBAAyB,EAAE,GAAG,EAAE,CAAC,QAAQ,CAAC,CAAA;AACzF,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,EAClB,UAAU,EACV,mBAAmB,GACpB,GAAG,GAAG,CAAA;AACP,OAAO,EACL,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,UAAU,EACV,mBAAmB,EACnB,eAAe,EACf,kBAAkB,GACnB,CAAA;AACD,oDAAoD;AACpD,eAAe,GAAG,CAAA"}
+{"version":3,"file":"gpt-4o-realtime-preview.js","sourceRoot":"","sources":["../../src/model/gpt-4o-realtime-preview.ts"],"names":[],"mappings":"AAAA,sCAAsC;AACtC,OAAO,QAAQ,MAAM,2BAA2B,CAAA;AAChD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAA;AAE/C,cAAc,iBAAiB,CAAA;AAC/B,cAAc,qBAAqB,CAAA;AACnC,kBAAkB;AAClB,MAAM,GAAG,GAAG,WAAW,CAAC,sBAAsB,CAAC,yBAAyB,EAAE,GAAG,EAAE,CAAC,QAAQ,CAAC,CAAA;AACzF,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,EAClB,UAAU,EACV,mBAAmB,EACnB,cAAc,GACf,GAAG,GAAG,CAAA;AACP,OAAO,EACL,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,UAAU,EACV,mBAAmB,EACnB,eAAe,EACf,kBAAkB,EAClB,cAAc,GACf,CAAA;AACD,oDAAoD;AACpD,eAAe,GAAG,CAAA"}

package/esm/model/gpt-4o.d.ts

@@ -1,6 +1,7 @@
 import { GptEncoding } from '../GptEncoding.js';
+export * from '../constants.js';
 export * from '../specialTokens.js';
 declare const api: GptEncoding;
-declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, void>, encode: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, tokenLimit: number) => false | number, encodeChat: (chat: readonly import("../GptEncoding.js").ChatMessage[], model?: "gpt-4o" | "gpt-3.5-turbo" | "gpt-3.5-turbo-instruct" | "babbage-002" | "davinci-002" | "text-embedding-3-small" | "o1-preview" | "o1-preview-2024-09-12" | "o1-mini" | "o1-mini-2024-09-12" | "chatgpt-4o-latest" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o-mini" | "gpt-4o-mini-2024-07-18" | "gpt-4o-realtime-preview" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-2024-08-06-finetune" | "gpt-4o-mini-2024-07-18-finetune" | "gpt-4o-mini-training" | "gpt-4o-mini-training-2024-07-18" | "gpt-4-turbo" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo-preview" | "gpt-4-0125-preview" | "gpt-4-1106-preview" | "gpt-4" | "gpt-4-0613" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo-finetune" | "gpt-3.5-turbo-16k" | "gpt-4-32k" | "gpt-4-32k-0613" | "gpt-4-vision-preview" | "gpt-4-1106-vision-preview" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo-0613" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-0301" | "text-embedding-3-large" | "text-embedding-ada-002" | "gpt-3.5-turbo-instruct-0914" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-002" | "code-davinci-001" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-002" | "code-cushman-001" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | undefined) => number[], encodeChatGenerator: (chat: Iterable<import("../GptEncoding.js").ChatMessage>, model?: "gpt-4o" | "gpt-3.5-turbo" | "gpt-3.5-turbo-instruct" | "babbage-002" | "davinci-002" | "text-embedding-3-small" | "o1-preview" | "o1-preview-2024-09-12" | "o1-mini" | "o1-mini-2024-09-12" | "chatgpt-4o-latest" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o-mini" | "gpt-4o-mini-2024-07-18" | "gpt-4o-realtime-preview" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-2024-08-06-finetune" | "gpt-4o-mini-2024-07-18-finetune" | "gpt-4o-mini-training" | "gpt-4o-mini-training-2024-07-18" | "gpt-4-turbo" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo-preview" | "gpt-4-0125-preview" | "gpt-4-1106-preview" | "gpt-4" | "gpt-4-0613" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo-finetune" | "gpt-3.5-turbo-16k" | "gpt-4-32k" | "gpt-4-32k-0613" | "gpt-4-vision-preview" | "gpt-4-1106-vision-preview" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo-0613" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-0301" | "text-embedding-3-large" | "text-embedding-ada-002" | "gpt-3.5-turbo-instruct-0914" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-002" | "code-davinci-001" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-002" | "code-cushman-001" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | undefined) => Generator<number[], void, undefined>;
-export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, };
+declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, void>, encode: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, tokenLimit: number) => false | number, encodeChat: (chat: readonly import("../GptEncoding.js").ChatMessage[], model?: "gpt-4o" | "gpt-3.5-turbo" | "gpt-3.5-turbo-instruct" | "babbage-002" | "davinci-002" | "text-embedding-3-small" | "o1-preview" | "o1-preview-2024-09-12" | "o1-mini" | "o1-mini-2024-09-12" | "chatgpt-4o-latest" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o-mini" | "gpt-4o-mini-2024-07-18" | "gpt-4o-realtime-preview" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-2024-08-06-finetune" | "gpt-4o-mini-2024-07-18-finetune" | "gpt-4o-mini-training" | "gpt-4o-mini-training-2024-07-18" | "gpt-4-turbo" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo-preview" | "gpt-4-0125-preview" | "gpt-4-1106-preview" | "gpt-4" | "gpt-4-0613" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo-finetune" | "gpt-3.5-turbo-16k" | "gpt-4-32k" | "gpt-4-32k-0613" | "gpt-4-vision-preview" | "gpt-4-1106-vision-preview" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo-0613" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-0301" | "text-embedding-3-large" | "text-embedding-ada-002" | "gpt-3.5-turbo-instruct-0914" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-002" | "code-davinci-001" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-002" | "code-cushman-001" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | undefined) => number[], encodeChatGenerator: (chat: Iterable<import("../GptEncoding.js").ChatMessage>, model?: "gpt-4o" | "gpt-3.5-turbo" | "gpt-3.5-turbo-instruct" | "babbage-002" | "davinci-002" | "text-embedding-3-small" | "o1-preview" | "o1-preview-2024-09-12" | "o1-mini" | "o1-mini-2024-09-12" | "chatgpt-4o-latest" | "gpt-4o-2024-08-06" | "gpt-4o-2024-05-13" | "gpt-4o-mini" | "gpt-4o-mini-2024-07-18" | "gpt-4o-realtime-preview" | "gpt-4o-realtime-preview-2024-10-01" | "gpt-4o-audio-preview" | "gpt-4o-audio-preview-2024-10-01" | "gpt-4o-2024-08-06-finetune" | "gpt-4o-mini-2024-07-18-finetune" | "gpt-4o-mini-training" | "gpt-4o-mini-training-2024-07-18" | "gpt-4-turbo" | "gpt-4-turbo-2024-04-09" | "gpt-4-turbo-preview" | "gpt-4-0125-preview" | "gpt-4-1106-preview" | "gpt-4" | "gpt-4-0613" | "gpt-3.5-turbo-0125" | "gpt-3.5-turbo-1106" | "gpt-3.5-turbo-finetune" | "gpt-3.5-turbo-16k" | "gpt-4-32k" | "gpt-4-32k-0613" | "gpt-4-vision-preview" | "gpt-4-1106-vision-preview" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo-0613" | "gpt-3.5-turbo-16k-0613" | "gpt-3.5-turbo-0301" | "text-embedding-3-large" | "text-embedding-ada-002" | "gpt-3.5-turbo-instruct-0914" | "text-ada-001" | "text-babbage-001" | "text-curie-001" | "text-davinci-001" | "text-davinci-002" | "text-davinci-003" | "ada" | "babbage" | "curie" | "davinci" | "code-davinci-002" | "code-davinci-001" | "davinci-codex" | "code-davinci-edit-001" | "code-cushman-002" | "code-cushman-001" | "cushman-codex" | "code-search-ada-code-001" | "code-search-ada-text-001" | "text-davinci-edit-001" | "text-similarity-ada-001" | "text-search-ada-doc-001" | "text-search-ada-query-001" | "text-similarity-babbage-001" | "text-search-babbage-doc-001" | "text-search-babbage-query-001" | "code-search-babbage-code-001" | "code-search-babbage-text-001" | "text-similarity-curie-001" | "text-search-curie-doc-001" | "text-search-curie-query-001" | "text-similarity-davinci-001" | "text-search-davinci-doc-001" | "text-search-davinci-query-001" | undefined) => Generator<number[], void, undefined>, vocabularySize: number;
+export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, vocabularySize, };
 export default api;

package/esm/model/gpt-4o.js

@@ -1,11 +1,12 @@
 /* eslint-disable import/extensions */
 import bpeRanks from '../bpeRanks/o200k_base.js';
 import { GptEncoding } from '../GptEncoding.js';
+export * from '../constants.js';
 export * from '../specialTokens.js';
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-4o', () => bpeRanks);
-const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, encodeChat, encodeChatGenerator, } = api;
-export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, };
+const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, encodeChat, encodeChatGenerator, vocabularySize, } = api;
+export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, vocabularySize, };
 // eslint-disable-next-line import/no-default-export
 export default api;
 //# sourceMappingURL=gpt-4o.js.map

package/esm/model/gpt-4o.js.map

@@ -1 +1 @@
-{"version":3,"file":"gpt-4o.js","sourceRoot":"","sources":["../../src/model/gpt-4o.ts"],"names":[],"mappings":"AAAA,sCAAsC;AACtC,OAAO,QAAQ,MAAM,2BAA2B,CAAA;AAChD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAA;AAE/C,cAAc,qBAAqB,CAAA;AACnC,kBAAkB;AAClB,MAAM,GAAG,GAAG,WAAW,CAAC,sBAAsB,CAAC,QAAQ,EAAE,GAAG,EAAE,CAAC,QAAQ,CAAC,CAAA;AACxE,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,EAClB,UAAU,EACV,mBAAmB,GACpB,GAAG,GAAG,CAAA;AACP,OAAO,EACL,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,UAAU,EACV,mBAAmB,EACnB,eAAe,EACf,kBAAkB,GACnB,CAAA;AACD,oDAAoD;AACpD,eAAe,GAAG,CAAA"}
+{"version":3,"file":"gpt-4o.js","sourceRoot":"","sources":["../../src/model/gpt-4o.ts"],"names":[],"mappings":"AAAA,sCAAsC;AACtC,OAAO,QAAQ,MAAM,2BAA2B,CAAA;AAChD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAA;AAE/C,cAAc,iBAAiB,CAAA;AAC/B,cAAc,qBAAqB,CAAA;AACnC,kBAAkB;AAClB,MAAM,GAAG,GAAG,WAAW,CAAC,sBAAsB,CAAC,QAAQ,EAAE,GAAG,EAAE,CAAC,QAAQ,CAAC,CAAA;AACxE,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,EAClB,UAAU,EACV,mBAAmB,EACnB,cAAc,GACf,GAAG,GAAG,CAAA;AACP,OAAO,EACL,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,UAAU,EACV,mBAAmB,EACnB,eAAe,EACf,kBAAkB,EAClB,cAAc,GACf,CAAA;AACD,oDAAoD;AACpD,eAAe,GAAG,CAAA"}

package/esm/modelParams.d.ts

@@ -13,10 +13,8 @@ export interface EncodingParams extends BytePairEncodingConfig {
      * It's complex due to its need to deal with a wide variety of cases in text processing.
      */
     tokenSplitRegex: RegExp;
-    specialTokenMapping: Map<string, number>;
+    specialTokensEncoder: Map<string, number>;
     modelName?: ModelName;
-    /** increases memory consumption, but speeds up subsequent decoding */
-    enableCache?: boolean;
 }
 export declare const tokenSplitRegex: RegExp;
 export type GetMergeableRanksFn = (encodingName: EncodingName) => RawBytePairRanks;

package/esm/modelParams.js.map

@@ -1 +1 @@
-{"version":3,"file":"modelParams.js","sourceRoot":"","sources":["../src/modelParams.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,UAAU,EAAE,MAAM,gCAAgC,CAAA;AAC3D,OAAO,EAAE,SAAS,EAAE,MAAM,+BAA+B,CAAA;AACzD,OAAO,EAAE,QAAQ,EAAE,MAAM,8BAA8B,CAAA;AACvD,OAAO,EAAE,QAAQ,EAAE,MAAM,8BAA8B,CAAA;AACvD,OAAO,EAAE,QAAQ,EAAE,MAAM,8BAA8B,CAAA;AAuBvD,MAAM,CAAC,MAAM,eAAe,GAC1B,8EAA8E,CAAA;AAShF,MAAM,UAAU,iBAAiB,CAC/B,YAA0B,EAC1B,iBAAsC;IAEtC,MAAM,sBAAsB,GAAG,iBAAiB,CAAC,YAAY,CAAC,CAAA;IAC9D,QAAQ,YAAY,CAAC,WAAW,EAAE,EAAE,CAAC;QACnC,KAAK,WAAW;YACd,OAAO,QAAQ,CAAC,sBAAsB,CAAC,CAAA;QAEzC,KAAK,WAAW;YACd,OAAO,QAAQ,CAAC,sBAAsB,CAAC,CAAA;QAEzC,KAAK,WAAW;YACd,OAAO,QAAQ,CAAC,sBAAsB,CAAC,CAAA;QAEzC,KAAK,aAAa;YAChB,OAAO,UAAU,CAAC,sBAAsB,CAAC,CAAA;QAE3C,KAAK,YAAY;YACf,OAAO,SAAS,CAAC,sBAAsB,CAAC,CAAA;QAE1C;YACE,MAAM,IAAI,KAAK,CAAC,0BAA0B,YAAY,EAAE,CAAC,CAAA;IAC7D,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,YAA0B,EAC1B,iBAA2C;IAE3C,MAAM,sBAAsB,GAAG,MAAM,iBAAiB,CAAC,YAAY,CAAC,CAAA;IACpE,OAAO,iBAAiB,CAAC,YAAY,EAAE,GAAG,EAAE,CAAC,sBAAsB,CAAC,CAAA;AACtE,CAAC"}
+{"version":3,"file":"modelParams.js","sourceRoot":"","sources":["../src/modelParams.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,UAAU,EAAE,MAAM,gCAAgC,CAAA;AAC3D,OAAO,EAAE,SAAS,EAAE,MAAM,+BAA+B,CAAA;AACzD,OAAO,EAAE,QAAQ,EAAE,MAAM,8BAA8B,CAAA;AACvD,OAAO,EAAE,QAAQ,EAAE,MAAM,8BAA8B,CAAA;AACvD,OAAO,EAAE,QAAQ,EAAE,MAAM,8BAA8B,CAAA;AAoBvD,MAAM,CAAC,MAAM,eAAe,GAC1B,8EAA8E,CAAA;AAShF,MAAM,UAAU,iBAAiB,CAC/B,YAA0B,EAC1B,iBAAsC;IAEtC,MAAM,sBAAsB,GAAG,iBAAiB,CAAC,YAAY,CAAC,CAAA;IAC9D,QAAQ,YAAY,CAAC,WAAW,EAAE,EAAE,CAAC;QACnC,KAAK,WAAW;YACd,OAAO,QAAQ,CAAC,sBAAsB,CAAC,CAAA;QAEzC,KAAK,WAAW;YACd,OAAO,QAAQ,CAAC,sBAAsB,CAAC,CAAA;QAEzC,KAAK,WAAW;YACd,OAAO,QAAQ,CAAC,sBAAsB,CAAC,CAAA;QAEzC,KAAK,aAAa;YAChB,OAAO,UAAU,CAAC,sBAAsB,CAAC,CAAA;QAE3C,KAAK,YAAY;YACf,OAAO,SAAS,CAAC,sBAAsB,CAAC,CAAA;QAE1C;YACE,MAAM,IAAI,KAAK,CAAC,0BAA0B,YAAY,EAAE,CAAC,CAAA;IAC7D,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,YAA0B,EAC1B,iBAA2C;IAE3C,MAAM,sBAAsB,GAAG,MAAM,iBAAiB,CAAC,YAAY,CAAC,CAAA;IACpE,OAAO,iBAAiB,CAAC,YAAY,EAAE,GAAG,EAAE,CAAC,sBAAsB,CAAC,CAAA;AACtE,CAAC"}

package/esm/models.d.ts

@@ -1,4 +1,4 @@
-import type { EncodingName } from './mapping';
+import type { EncodingName } from './mapping.js';
 export interface Model {
     humanName: string;
     encoding: EncodingName;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "gpt-tokenizer",
-  "version": "2.6.0",
+  "version": "2.6.2",
   "description": "A pure JavaScript implementation of a BPE tokenizer (Encoder/Decoder) for GPT-2 / GPT-3 / GPT-4 and other OpenAI models",
   "keywords": [
     "BPE",
@@ -77,7 +77,7 @@
     "codegen:bpe": "rm -rf src/bpeRanks && yarn tsx src/codegen/generateJsBpe.ts",
     "build": "yarn build:cjs && yarn build:esm && yarn build:umd",
     "build:cjs": "yarn rrun tsc --outDir cjs --module commonjs --target es2022 --project tsconfig-cjs.json",
-    "build:esm": "mkdir -p esm && echo '{\"name\": \"gpt-tokenizer\", \"type\": \"module\"}' > ./esm/package.json && yarn rrun tsc --outDir esm --module esnext --target es2022",
+    "build:esm": "mkdir -p esm && echo '{\"name\": \"gpt-tokenizer\", \"type\": \"module\"}' > ./esm/package.json && yarn rrun tsc --outDir esm --target es2022",
     "build:umd": "yarn build:umd:cl100k_base && yarn build:umd:p50k_base && yarn build:umd:p50k_edit && yarn build:umd:r50k_base && yarn build:umd:o200k_base",
     "build:umd:cl100k_base": "beemo webpack --entry='./src/main.ts' --env 'outDir=dist' --env 'moduleTarget=umd' --env 'engineTarget=web' --env 'codeTarget=es2022' --env 'name=GPTTokenizer_cl100k_base' --env 'filename=cl100k_base.js'",
     "build:umd:p50k_base": "beemo webpack --entry='./src/encoding/p50k_base.ts' --env 'outDir=dist' --env 'moduleTarget=umd' --env 'engineTarget=web' --env 'codeTarget=es2022' --env 'name=GPTTokenizer_p50k_base' --env 'filename=p50k_base.js'",
@@ -116,9 +116,9 @@
     "tagFormat": "${version}"
   },
   "devDependencies": {
-    "@niieani/scaffold": "^1.7.32",
-    "tsx": "^4.19.1",
-    "typescript": "^5.6.2"
+    "@niieani/scaffold": "^1.7.37",
+    "tsx": "^4.19.2",
+    "typescript": "^5.6.3"
   },
   "resolutions": {
     "typescript": "5.6.2"

package/src/BytePairEncodingCore.ts

@@ -6,8 +6,8 @@ import { escapeRegExp } from './util.js'
 export type RawBytePairRanks = readonly (string | readonly number[])[]
 
 export interface BytePairEncodingConfig {
-  mergeableBytePairRanks: RawBytePairRanks
-  specialTokenMapping?: Map<string, number>
+  bytePairRankDecoder: RawBytePairRanks
+  specialTokensEncoder?: Map<string, number>
   tokenSplitRegex: RegExp
 }
 
@@ -38,18 +38,18 @@ export class BytePairEncodingCore {
   private textEncoder = new TextEncoder()
 
   constructor({
-    mergeableBytePairRanks: bytePairEncoder,
-    specialTokenMapping: specialTokenEncoder,
+    bytePairRankDecoder,
+    specialTokensEncoder,
     tokenSplitRegex,
   }: BytePairEncodingConfig) {
-    this.bytePairRankDecoder = bytePairEncoder
+    this.bytePairRankDecoder = bytePairRankDecoder
     this.bytePairStringRankEncoder = new Map<string, number>()
 
     // size without array holes (which may be present in the encoder)
-    this.mergeableBytePairRankCount = Object.keys(bytePairEncoder).length
+    this.mergeableBytePairRankCount = Object.keys(bytePairRankDecoder).length
     const binaryLookup: [Uint8Array, number][] = []
     // forEach skips array holes:
-    bytePairEncoder.forEach((value, rank) => {
+    bytePairRankDecoder.forEach((value, rank) => {
       if (typeof value === 'string') {
         this.bytePairStringRankEncoder.set(value, rank)
         return
@@ -61,9 +61,10 @@ export class BytePairEncodingCore {
     this.bytePairNonUtfSortedEncoder = binaryLookup.sort((a, b) =>
       compareUint8Arrays(a[0], b[0]),
     )
-    this.specialTokensEncoder = specialTokenEncoder ?? new Map<string, number>()
-    this.specialTokensDecoder = specialTokenEncoder
-      ? new Map([...specialTokenEncoder].map(([key, value]) => [value, key]))
+    this.specialTokensEncoder =
+      specialTokensEncoder ?? new Map<string, number>()
+    this.specialTokensDecoder = specialTokensEncoder
+      ? new Map([...specialTokensEncoder].map(([key, value]) => [value, key]))
       : new Map<number, string>()
     this.tokenSplitRegex = tokenSplitRegex
 
@@ -86,20 +87,18 @@ export class BytePairEncodingCore {
     let lastTokenLength = 0
 
     while (true) {
-      const nextSpecialStartIndex = this.findNextSpecialStartIndex(
+      const nextSpecialMatch = this.findNextSpecialToken(
         text,
         allowedSpecial,
         startIndex,
       )
+      const nextSpecialStartIndex = nextSpecialMatch?.[0]
 
-      const endIndex =
-        nextSpecialStartIndex !== undefined
-          ? nextSpecialStartIndex
-          : text.length
+      const endIndex = nextSpecialStartIndex ?? text.length
 
-      const textSegment = text.slice(startIndex, endIndex - startIndex)
+      const textBeforeSpecial = text.slice(startIndex, endIndex)
 
-      for (const [match] of textSegment.matchAll(this.tokenSplitRegex)) {
+      for (const [match] of textBeforeSpecial.matchAll(this.tokenSplitRegex)) {
         const token = this.getBpeRankFromString(match)
         if (token !== undefined) {
           lastTokenLength = 1
@@ -114,7 +113,7 @@ export class BytePairEncodingCore {
       }
 
       if (nextSpecialStartIndex !== undefined) {
-        const specialToken = text.slice(Math.max(0, nextSpecialStartIndex))
+        const specialToken = nextSpecialMatch![1]
         const specialTokenValue = this.specialTokensEncoder.get(specialToken)
         if (specialTokenValue === undefined) {
           throw new Error(
@@ -123,7 +122,7 @@ export class BytePairEncodingCore {
         }
         yield [specialTokenValue]
         startIndex = nextSpecialStartIndex + specialToken.length
-        lastTokenLength = 0
+        lastTokenLength = 1
       } else {
         break
       }
@@ -138,20 +137,18 @@ export class BytePairEncodingCore {
 
     // eslint-disable-next-line no-constant-condition
     while (true) {
-      const nextSpecialStartIndex = this.findNextSpecialStartIndex(
+      const nextSpecialMatch = this.findNextSpecialToken(
         text,
         allowedSpecial,
         startIndex,
       )
+      const nextSpecialStartIndex = nextSpecialMatch?.[0]
 
-      const endIndex =
-        nextSpecialStartIndex !== undefined
-          ? nextSpecialStartIndex
-          : text.length
+      const endIndex = nextSpecialStartIndex ?? text.length
 
-      const textSegment = text.slice(startIndex, endIndex - startIndex)
+      const textBeforeSpecial = text.slice(startIndex, endIndex)
 
-      for (const [match] of textSegment.matchAll(this.tokenSplitRegex)) {
+      for (const [match] of textBeforeSpecial.matchAll(this.tokenSplitRegex)) {
         const token = this.getBpeRankFromString(match)
         if (token !== undefined) {
           tokensArray.push(token)
@@ -164,7 +161,7 @@ export class BytePairEncodingCore {
       }
 
       if (nextSpecialStartIndex !== undefined) {
-        const specialToken = text.slice(Math.max(0, nextSpecialStartIndex))
+        const specialToken = nextSpecialMatch![1]
         const specialTokenValue = this.specialTokensEncoder.get(specialToken)
         if (specialTokenValue === undefined) {
           throw new Error(
@@ -302,11 +299,11 @@ export class BytePairEncodingCore {
     return -1
   }
 
-  private findNextSpecialStartIndex(
+  private findNextSpecialToken(
     text: string,
     allowedSpecial: Set<string> | undefined,
     startIndex: number,
-  ): number | undefined {
+  ): [startIndex: number, token: string] | undefined {
     let searchIndex = startIndex
 
     // eslint-disable-next-line no-constant-condition
@@ -322,7 +319,8 @@ export class BytePairEncodingCore {
       const specialToken = nextSpecialMatch[0]
 
       if (allowedSpecial?.has(specialToken)) {
-        return nextSpecialMatch.index + searchIndex
+        const specialTokenStartIndex = nextSpecialMatch.index + searchIndex
+        return [specialTokenStartIndex, specialToken]
       }
 
       searchIndex = nextSpecialMatch.index + searchIndex + 1

package/src/GptEncoding.test.ts

@@ -1,5 +1,6 @@
 import fs from 'fs'
 import path from 'path'
+import { ALL_SPECIAL_TOKENS } from './constants.js'
 import { type ChatMessage, GptEncoding } from './GptEncoding.js'
 import {
   type ChatModelName,
@@ -61,6 +62,25 @@ const results = {
   r50k_base: sharedResults,
 } satisfies Record<EncodingName, unknown>
 
+const offsetPrompts = [
+  // Basic prompt with "hello world"
+  'hello world',
+
+  // Basic prompt with special token "<|endoftext|>"
+  'hello world<|endoftext|> green cow',
+
+  // Chinese text: "我非常渴望与人工智能一起工作"
+  '我非常渴望与人工智能一起工作',
+
+  // Contains the interesting tokens b'\xe0\xae\xbf\xe0\xae' and b'\xe0\xaf\x8d\xe0\xae'
+  // in which \xe0 is the start of a 3-byte UTF-8 character
+  'நடிகர் சூர்யா',
+
+  // Contains the interesting token b'\xa0\xe9\x99\xa4'
+  // in which \xe9 is the start of a 3-byte UTF-8 character and \xa0 is a continuation byte
+  ' Ġ除',
+]
+
 // eslint-disable-next-line @typescript-eslint/no-use-before-define
 const testPlans = loadTestPlans()
 
@@ -75,9 +95,16 @@ describe.each(encodingNames)('%s', (encodingName: EncodingName) => {
     isWithinTokenLimit,
   } = encoding
 
-  const result = results[encodingName]
+  describe('encode and decode', () => {
+    it.each(offsetPrompts)('offset prompt: %s', (str) => {
+      expect(
+        decode(encode(str, { allowedSpecial: ALL_SPECIAL_TOKENS })),
+      ).toEqual(str)
+    })
+  })
 
   describe('basic functionality', () => {
+    const result = results[encodingName]
     it('empty string', () => {
       const str = ''
       expect(encode(str)).toEqual([])
@@ -131,6 +158,14 @@ describe.each(encodingNames)('%s', (encodingName: EncodingName) => {
       })
     })
 
+    it('encodes and decodes special tokens', () => {
+      const str = 'hello <|endoftext|> world'
+      const encoded = encode(str, {
+        allowedSpecial: ALL_SPECIAL_TOKENS,
+      })
+      expect(decode(encoded)).toEqual(str)
+    })
+
     async function* getHelloWorldTokensAsync() {
       const str = 'hello 👋 world 🌍'
       for (const token of result[str]) {

package/src/GptEncoding.ts

@@ -1,6 +1,7 @@
 /* eslint-disable @typescript-eslint/member-ordering */
 /* eslint-disable no-param-reassign */
 import { BytePairEncodingCore, decoder } from './BytePairEncodingCore.js'
+import { ALL_SPECIAL_TOKENS } from './constants.js'
 import {
   type ChatModelName,
   type ChatParameters,
@@ -29,11 +30,19 @@ import {
 import { endsWithIncompleteUtfPairSurrogate } from './utfUtil.js'
 import { getMaxValueFromMap, getSpecialTokenRegex } from './util.js'
 
-export const ALL_SPECIAL_TOKENS = 'all'
-
 export interface EncodeOptions {
-  allowedSpecial?: Set<string>
-  disallowedSpecial?: Set<string>
+  /**
+   * A list of special tokens that are allowed in the input.
+   * If set to 'all', all special tokens are allowed except those in disallowedSpecial.
+   * @default undefined
+   */
+  allowedSpecial?: Set<string> | typeof ALL_SPECIAL_TOKENS
+  /**
+   * A list of special tokens that are disallowed in the input.
+   * If set to 'all', all special tokens are disallowed except those in allowedSpecial.
+   * @default 'all'
+   */
+  disallowedSpecial?: Set<string> | typeof ALL_SPECIAL_TOKENS
 }
 
 export interface ChatMessage {
@@ -60,41 +69,44 @@ export class GptEncoding {
 
   modelName?: ModelName
   private bytePairEncodingCoreProcessor: BytePairEncodingCore
-  private specialTokenMapping: Map<string, number>
+  private specialTokensEncoder: Map<string, number>
   private specialTokensSet: Set<string>
   private allSpecialTokenRegex: RegExp
   private defaultSpecialTokenConfig: SpecialTokenConfig
 
+  readonly vocabularySize: number
+
   private constructor({
-    mergeableBytePairRanks,
-    specialTokenMapping,
+    bytePairRankDecoder: mergeableBytePairRanks,
+    specialTokensEncoder,
     expectedVocabularySize,
     modelName,
     ...rest
   }: EncodingParams) {
-    this.specialTokenMapping = specialTokenMapping
-    this.specialTokensSet = new Set<string>(this.specialTokenMapping.keys())
+    this.specialTokensEncoder = specialTokensEncoder
+    this.specialTokensSet = new Set<string>(this.specialTokensEncoder.keys())
     this.allSpecialTokenRegex = getSpecialTokenRegex(this.specialTokensSet)
 
     this.bytePairEncodingCoreProcessor = new BytePairEncodingCore({
-      mergeableBytePairRanks,
-      specialTokenMapping,
+      bytePairRankDecoder: mergeableBytePairRanks,
+      specialTokensEncoder,
       ...rest,
     })
     this.defaultSpecialTokenConfig = this.processSpecialTokens()
 
     const maxTokenValue = Math.max(
       mergeableBytePairRanks.length - 1,
-      getMaxValueFromMap(specialTokenMapping),
+      getMaxValueFromMap(specialTokensEncoder),
     )
+
+    this.vocabularySize =
+      this.bytePairEncodingCoreProcessor.mergeableBytePairRankCount +
+      specialTokensEncoder.size
+
     if (expectedVocabularySize !== undefined) {
-      if (
-        this.bytePairEncodingCoreProcessor.mergeableBytePairRankCount +
-          specialTokenMapping.size !==
-        expectedVocabularySize
-      ) {
+      if (this.vocabularySize !== expectedVocabularySize) {
         throw new Error(
-          'The number of mergeable tokens and special tokens must be equal to explicit_n_vocab.',
+          'The number of mergeable tokens and special tokens must be equal to expectedVocabularySize.',
         )
       }
 
@@ -165,15 +177,37 @@ export class GptEncoding {
   }: EncodeOptions = {}): SpecialTokenConfig {
     let regexPattern: RegExp | undefined
 
-    if (allowedSpecial?.has(ALL_SPECIAL_TOKENS)) {
+    if (
+      allowedSpecial === ALL_SPECIAL_TOKENS ||
+      allowedSpecial?.has(ALL_SPECIAL_TOKENS)
+    ) {
       allowedSpecial = new Set(this.specialTokensSet)
+      const allowedSpecialSet = allowedSpecial
+      if (disallowedSpecial === ALL_SPECIAL_TOKENS) {
+        throw new Error(
+          'allowedSpecial and disallowedSpecial cannot both be set to "all".',
+        )
+      }
+      if (typeof disallowedSpecial === 'object') {
+        // remove any special tokens that are disallowed
+        disallowedSpecial.forEach((val) => allowedSpecialSet.delete(val))
+      } else {
+        // all special tokens are allowed, and no 'disallowedSpecial' is provided
+        disallowedSpecial = new Set()
+      }
     }
 
-    if (!disallowedSpecial || disallowedSpecial.has(ALL_SPECIAL_TOKENS)) {
+    if (
+      !disallowedSpecial ||
+      disallowedSpecial === ALL_SPECIAL_TOKENS ||
+      disallowedSpecial.has(ALL_SPECIAL_TOKENS)
+    ) {
       // by default, all special tokens are disallowed
       disallowedSpecial = new Set(this.specialTokensSet)
+      const disallowedSpecialSet = disallowedSpecial
       if (allowedSpecial?.size) {
-        allowedSpecial.forEach((val) => disallowedSpecial!.delete(val))
+        allowedSpecial.forEach((val) => disallowedSpecialSet.delete(val))
+        // disallowed takes precedence over allowed
         disallowedSpecial.forEach((val) => allowedSpecial.delete(val))
         regexPattern = getSpecialTokenRegex(disallowedSpecial)
       } else {
@@ -242,8 +276,8 @@ export class GptEncoding {
 
     const params: ChatParameters | undefined =
       chatModelParams[model as ChatModelName]
-    const chatStartToken = this.specialTokenMapping.get(ImStart)
-    const chatEndToken = this.specialTokenMapping.get(ImEnd)
+    const chatStartToken = this.specialTokensEncoder.get(ImStart)
+    const chatEndToken = this.specialTokensEncoder.get(ImEnd)
 
     if (!params || chatStartToken === undefined || chatEndToken === undefined) {
       throw new Error(`Model '${model}' does not support chat.`)

package/src/codegen/generateJsBpe.ts

@@ -119,8 +119,8 @@ await processFilesInDirectory(
     await fs.writeFile(
       path.join(__dirname, `../bpeRanks/${modelName}.js`),
       `/* eslint-disable */\n// @ts-nocheck\n// prettier-ignore
-/** @type {(string | number[])[]} */
 ${jsCodeConstsForEachChunk.join('\n')}
+/** @type {(string | number[])[]} */
 const bpe = ${jsCodeBpeArray};
 ${jsCodeToResetHelperArrays.join('\n')}
 export default bpe;`,

package/src/constants.ts

@@ -0,0 +1 @@
+export const ALL_SPECIAL_TOKENS = 'all'

package/src/encoding/cl100k_base.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/cl100k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 
 const api = GptEncoding.getEncodingApi('cl100k_base', () => bpeRanks)
@@ -14,6 +15,7 @@ const {
   isWithinTokenLimit,
   encodeChat,
   encodeChatGenerator,
+  vocabularySize,
 } = api
 export {
   decode,
@@ -24,6 +26,7 @@ export {
   encodeChatGenerator,
   encodeGenerator,
   isWithinTokenLimit,
+  vocabularySize,
 }
 // eslint-disable-next-line import/no-default-export
 export default api