gpt-tokenizer 2.6.0 → 2.6.1

package/esm/model/gpt-4o.js

@@ -4,8 +4,8 @@ import { GptEncoding } from '../GptEncoding.js';
 export * from '../specialTokens.js';
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-4o', () => bpeRanks);
-const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, encodeChat, encodeChatGenerator, } = api;
-export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, };
+const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, encodeChat, encodeChatGenerator, vocabularySize, } = api;
+export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, vocabularySize, };
 // eslint-disable-next-line import/no-default-export
 export default api;
 //# sourceMappingURL=gpt-4o.js.map

package/esm/model/gpt-4o.js.map

@@ -1 +1 @@
-{"version":3,"file":"gpt-4o.js","sourceRoot":"","sources":["../../src/model/gpt-4o.ts"],"names":[],"mappings":"AAAA,sCAAsC;AACtC,OAAO,QAAQ,MAAM,2BAA2B,CAAA;AAChD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAA;AAE/C,cAAc,qBAAqB,CAAA;AACnC,kBAAkB;AAClB,MAAM,GAAG,GAAG,WAAW,CAAC,sBAAsB,CAAC,QAAQ,EAAE,GAAG,EAAE,CAAC,QAAQ,CAAC,CAAA;AACxE,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,EAClB,UAAU,EACV,mBAAmB,GACpB,GAAG,GAAG,CAAA;AACP,OAAO,EACL,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,UAAU,EACV,mBAAmB,EACnB,eAAe,EACf,kBAAkB,GACnB,CAAA;AACD,oDAAoD;AACpD,eAAe,GAAG,CAAA"}
+{"version":3,"file":"gpt-4o.js","sourceRoot":"","sources":["../../src/model/gpt-4o.ts"],"names":[],"mappings":"AAAA,sCAAsC;AACtC,OAAO,QAAQ,MAAM,2BAA2B,CAAA;AAChD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAA;AAE/C,cAAc,qBAAqB,CAAA;AACnC,kBAAkB;AAClB,MAAM,GAAG,GAAG,WAAW,CAAC,sBAAsB,CAAC,QAAQ,EAAE,GAAG,EAAE,CAAC,QAAQ,CAAC,CAAA;AACxE,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,EAClB,UAAU,EACV,mBAAmB,EACnB,cAAc,GACf,GAAG,GAAG,CAAA;AACP,OAAO,EACL,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,UAAU,EACV,mBAAmB,EACnB,eAAe,EACf,kBAAkB,EAClB,cAAc,GACf,CAAA;AACD,oDAAoD;AACpD,eAAe,GAAG,CAAA"}

package/esm/modelParams.d.ts

@@ -15,8 +15,6 @@ export interface EncodingParams extends BytePairEncodingConfig {
     tokenSplitRegex: RegExp;
     specialTokenMapping: Map<string, number>;
     modelName?: ModelName;
-    /** increases memory consumption, but speeds up subsequent decoding */
-    enableCache?: boolean;
 }
 export declare const tokenSplitRegex: RegExp;
 export type GetMergeableRanksFn = (encodingName: EncodingName) => RawBytePairRanks;

package/esm/modelParams.js.map

@@ -1 +1 @@
-{"version":3,"file":"modelParams.js","sourceRoot":"","sources":["../src/modelParams.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,UAAU,EAAE,MAAM,gCAAgC,CAAA;AAC3D,OAAO,EAAE,SAAS,EAAE,MAAM,+BAA+B,CAAA;AACzD,OAAO,EAAE,QAAQ,EAAE,MAAM,8BAA8B,CAAA;AACvD,OAAO,EAAE,QAAQ,EAAE,MAAM,8BAA8B,CAAA;AACvD,OAAO,EAAE,QAAQ,EAAE,MAAM,8BAA8B,CAAA;AAuBvD,MAAM,CAAC,MAAM,eAAe,GAC1B,8EAA8E,CAAA;AAShF,MAAM,UAAU,iBAAiB,CAC/B,YAA0B,EAC1B,iBAAsC;IAEtC,MAAM,sBAAsB,GAAG,iBAAiB,CAAC,YAAY,CAAC,CAAA;IAC9D,QAAQ,YAAY,CAAC,WAAW,EAAE,EAAE,CAAC;QACnC,KAAK,WAAW;YACd,OAAO,QAAQ,CAAC,sBAAsB,CAAC,CAAA;QAEzC,KAAK,WAAW;YACd,OAAO,QAAQ,CAAC,sBAAsB,CAAC,CAAA;QAEzC,KAAK,WAAW;YACd,OAAO,QAAQ,CAAC,sBAAsB,CAAC,CAAA;QAEzC,KAAK,aAAa;YAChB,OAAO,UAAU,CAAC,sBAAsB,CAAC,CAAA;QAE3C,KAAK,YAAY;YACf,OAAO,SAAS,CAAC,sBAAsB,CAAC,CAAA;QAE1C;YACE,MAAM,IAAI,KAAK,CAAC,0BAA0B,YAAY,EAAE,CAAC,CAAA;IAC7D,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,YAA0B,EAC1B,iBAA2C;IAE3C,MAAM,sBAAsB,GAAG,MAAM,iBAAiB,CAAC,YAAY,CAAC,CAAA;IACpE,OAAO,iBAAiB,CAAC,YAAY,EAAE,GAAG,EAAE,CAAC,sBAAsB,CAAC,CAAA;AACtE,CAAC"}
+{"version":3,"file":"modelParams.js","sourceRoot":"","sources":["../src/modelParams.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,UAAU,EAAE,MAAM,gCAAgC,CAAA;AAC3D,OAAO,EAAE,SAAS,EAAE,MAAM,+BAA+B,CAAA;AACzD,OAAO,EAAE,QAAQ,EAAE,MAAM,8BAA8B,CAAA;AACvD,OAAO,EAAE,QAAQ,EAAE,MAAM,8BAA8B,CAAA;AACvD,OAAO,EAAE,QAAQ,EAAE,MAAM,8BAA8B,CAAA;AAoBvD,MAAM,CAAC,MAAM,eAAe,GAC1B,8EAA8E,CAAA;AAShF,MAAM,UAAU,iBAAiB,CAC/B,YAA0B,EAC1B,iBAAsC;IAEtC,MAAM,sBAAsB,GAAG,iBAAiB,CAAC,YAAY,CAAC,CAAA;IAC9D,QAAQ,YAAY,CAAC,WAAW,EAAE,EAAE,CAAC;QACnC,KAAK,WAAW;YACd,OAAO,QAAQ,CAAC,sBAAsB,CAAC,CAAA;QAEzC,KAAK,WAAW;YACd,OAAO,QAAQ,CAAC,sBAAsB,CAAC,CAAA;QAEzC,KAAK,WAAW;YACd,OAAO,QAAQ,CAAC,sBAAsB,CAAC,CAAA;QAEzC,KAAK,aAAa;YAChB,OAAO,UAAU,CAAC,sBAAsB,CAAC,CAAA;QAE3C,KAAK,YAAY;YACf,OAAO,SAAS,CAAC,sBAAsB,CAAC,CAAA;QAE1C;YACE,MAAM,IAAI,KAAK,CAAC,0BAA0B,YAAY,EAAE,CAAC,CAAA;IAC7D,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,YAA0B,EAC1B,iBAA2C;IAE3C,MAAM,sBAAsB,GAAG,MAAM,iBAAiB,CAAC,YAAY,CAAC,CAAA;IACpE,OAAO,iBAAiB,CAAC,YAAY,EAAE,GAAG,EAAE,CAAC,sBAAsB,CAAC,CAAA;AACtE,CAAC"}

package/esm/models.d.ts

@@ -1,4 +1,4 @@
-import type { EncodingName } from './mapping';
+import type { EncodingName } from './mapping.js';
 export interface Model {
     humanName: string;
     encoding: EncodingName;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "gpt-tokenizer",
-  "version": "2.6.0",
+  "version": "2.6.1",
   "description": "A pure JavaScript implementation of a BPE tokenizer (Encoder/Decoder) for GPT-2 / GPT-3 / GPT-4 and other OpenAI models",
   "keywords": [
     "BPE",
@@ -77,7 +77,7 @@
     "codegen:bpe": "rm -rf src/bpeRanks && yarn tsx src/codegen/generateJsBpe.ts",
     "build": "yarn build:cjs && yarn build:esm && yarn build:umd",
     "build:cjs": "yarn rrun tsc --outDir cjs --module commonjs --target es2022 --project tsconfig-cjs.json",
-    "build:esm": "mkdir -p esm && echo '{\"name\": \"gpt-tokenizer\", \"type\": \"module\"}' > ./esm/package.json && yarn rrun tsc --outDir esm --module esnext --target es2022",
+    "build:esm": "mkdir -p esm && echo '{\"name\": \"gpt-tokenizer\", \"type\": \"module\"}' > ./esm/package.json && yarn rrun tsc --outDir esm --target es2022",
     "build:umd": "yarn build:umd:cl100k_base && yarn build:umd:p50k_base && yarn build:umd:p50k_edit && yarn build:umd:r50k_base && yarn build:umd:o200k_base",
     "build:umd:cl100k_base": "beemo webpack --entry='./src/main.ts' --env 'outDir=dist' --env 'moduleTarget=umd' --env 'engineTarget=web' --env 'codeTarget=es2022' --env 'name=GPTTokenizer_cl100k_base' --env 'filename=cl100k_base.js'",
     "build:umd:p50k_base": "beemo webpack --entry='./src/encoding/p50k_base.ts' --env 'outDir=dist' --env 'moduleTarget=umd' --env 'engineTarget=web' --env 'codeTarget=es2022' --env 'name=GPTTokenizer_p50k_base' --env 'filename=p50k_base.js'",
@@ -116,9 +116,9 @@
     "tagFormat": "${version}"
   },
   "devDependencies": {
-    "@niieani/scaffold": "^1.7.32",
-    "tsx": "^4.19.1",
-    "typescript": "^5.6.2"
+    "@niieani/scaffold": "^1.7.37",
+    "tsx": "^4.19.2",
+    "typescript": "^5.6.3"
   },
   "resolutions": {
     "typescript": "5.6.2"

package/src/GptEncoding.ts

@@ -65,6 +65,8 @@ export class GptEncoding {
   private allSpecialTokenRegex: RegExp
   private defaultSpecialTokenConfig: SpecialTokenConfig
 
+  readonly vocabularySize: number
+
   private constructor({
     mergeableBytePairRanks,
     specialTokenMapping,
@@ -87,14 +89,15 @@ export class GptEncoding {
       mergeableBytePairRanks.length - 1,
       getMaxValueFromMap(specialTokenMapping),
     )
+
+    this.vocabularySize =
+      this.bytePairEncodingCoreProcessor.mergeableBytePairRankCount +
+      specialTokenMapping.size
+
     if (expectedVocabularySize !== undefined) {
-      if (
-        this.bytePairEncodingCoreProcessor.mergeableBytePairRankCount +
-          specialTokenMapping.size !==
-        expectedVocabularySize
-      ) {
+      if (this.vocabularySize !== expectedVocabularySize) {
         throw new Error(
-          'The number of mergeable tokens and special tokens must be equal to explicit_n_vocab.',
+          'The number of mergeable tokens and special tokens must be equal to expectedVocabularySize.',
         )
       }
 

package/src/codegen/generateJsBpe.ts

@@ -119,8 +119,8 @@ await processFilesInDirectory(
     await fs.writeFile(
       path.join(__dirname, `../bpeRanks/${modelName}.js`),
       `/* eslint-disable */\n// @ts-nocheck\n// prettier-ignore
-/** @type {(string | number[])[]} */
 ${jsCodeConstsForEachChunk.join('\n')}
+/** @type {(string | number[])[]} */
 const bpe = ${jsCodeBpeArray};
 ${jsCodeToResetHelperArrays.join('\n')}
 export default bpe;`,

package/src/encoding/cl100k_base.ts

@@ -14,6 +14,7 @@ const {
   isWithinTokenLimit,
   encodeChat,
   encodeChatGenerator,
+  vocabularySize,
 } = api
 export {
   decode,
@@ -24,6 +25,7 @@ export {
   encodeChatGenerator,
   encodeGenerator,
   isWithinTokenLimit,
+  vocabularySize,
 }
 // eslint-disable-next-line import/no-default-export
 export default api

package/src/encoding/o200k_base.ts

@@ -14,6 +14,7 @@ const {
   isWithinTokenLimit,
   encodeChat,
   encodeChatGenerator,
+  vocabularySize,
 } = api
 export {
   decode,
@@ -24,6 +25,7 @@ export {
   encodeChatGenerator,
   encodeGenerator,
   isWithinTokenLimit,
+  vocabularySize,
 }
 // eslint-disable-next-line import/no-default-export
 export default api

package/src/encoding/p50k_base.ts

@@ -12,6 +12,7 @@ const {
   encode,
   encodeGenerator,
   isWithinTokenLimit,
+  vocabularySize,
 } = api
 export {
   decode,
@@ -20,6 +21,7 @@ export {
   encode,
   encodeGenerator,
   isWithinTokenLimit,
+  vocabularySize,
 }
 // eslint-disable-next-line import/no-default-export
 export default api

package/src/encoding/p50k_edit.ts

@@ -12,6 +12,7 @@ const {
   encode,
   encodeGenerator,
   isWithinTokenLimit,
+  vocabularySize,
 } = api
 export {
   decode,
@@ -20,6 +21,7 @@ export {
   encode,
   encodeGenerator,
   isWithinTokenLimit,
+  vocabularySize,
 }
 // eslint-disable-next-line import/no-default-export
 export default api

package/src/encoding/r50k_base.ts

@@ -12,6 +12,7 @@ const {
   encode,
   encodeGenerator,
   isWithinTokenLimit,
+  vocabularySize,
 } = api
 export {
   decode,
@@ -20,6 +21,7 @@ export {
   encode,
   encodeGenerator,
   isWithinTokenLimit,
+  vocabularySize,
 }
 // eslint-disable-next-line import/no-default-export
 export default api

package/src/model/gpt-3.5-turbo-0125.ts

@@ -14,6 +14,7 @@ const {
   isWithinTokenLimit,
   encodeChat,
   encodeChatGenerator,
+  vocabularySize,
 } = api
 export {
   decode,
@@ -24,6 +25,7 @@ export {
   encodeChatGenerator,
   encodeGenerator,
   isWithinTokenLimit,
+  vocabularySize,
 }
 // eslint-disable-next-line import/no-default-export
 export default api

package/src/model/gpt-3.5-turbo-0301.ts

@@ -14,6 +14,7 @@ const {
   isWithinTokenLimit,
   encodeChat,
   encodeChatGenerator,
+  vocabularySize,
 } = api
 export {
   decode,
@@ -24,6 +25,7 @@ export {
   encodeChatGenerator,
   encodeGenerator,
   isWithinTokenLimit,
+  vocabularySize,
 }
 // eslint-disable-next-line import/no-default-export
 export default api

package/src/model/gpt-3.5-turbo-0613.ts

@@ -14,6 +14,7 @@ const {
   isWithinTokenLimit,
   encodeChat,
   encodeChatGenerator,
+  vocabularySize,
 } = api
 export {
   decode,
@@ -24,6 +25,7 @@ export {
   encodeChatGenerator,
   encodeGenerator,
   isWithinTokenLimit,
+  vocabularySize,
 }
 // eslint-disable-next-line import/no-default-export
 export default api

package/src/model/gpt-3.5-turbo-1106.ts

@@ -14,6 +14,7 @@ const {
   isWithinTokenLimit,
   encodeChat,
   encodeChatGenerator,
+  vocabularySize,
 } = api
 export {
   decode,
@@ -24,6 +25,7 @@ export {
   encodeChatGenerator,
   encodeGenerator,
   isWithinTokenLimit,
+  vocabularySize,
 }
 // eslint-disable-next-line import/no-default-export
 export default api

package/src/model/gpt-3.5-turbo-16k-0613.ts

@@ -14,6 +14,7 @@ const {
   isWithinTokenLimit,
   encodeChat,
   encodeChatGenerator,
+  vocabularySize,
 } = api
 export {
   decode,
@@ -24,6 +25,7 @@ export {
   encodeChatGenerator,
   encodeGenerator,
   isWithinTokenLimit,
+  vocabularySize,
 }
 // eslint-disable-next-line import/no-default-export
 export default api

package/src/model/gpt-3.5-turbo-16k.ts

@@ -14,6 +14,7 @@ const {
   isWithinTokenLimit,
   encodeChat,
   encodeChatGenerator,
+  vocabularySize,
 } = api
 export {
   decode,
@@ -24,6 +25,7 @@ export {
   encodeChatGenerator,
   encodeGenerator,
   isWithinTokenLimit,
+  vocabularySize,
 }
 // eslint-disable-next-line import/no-default-export
 export default api

package/src/model/gpt-3.5-turbo-finetune.ts

@@ -14,6 +14,7 @@ const {
   isWithinTokenLimit,
   encodeChat,
   encodeChatGenerator,
+  vocabularySize,
 } = api
 export {
   decode,
@@ -24,6 +25,7 @@ export {
   encodeChatGenerator,
   encodeGenerator,
   isWithinTokenLimit,
+  vocabularySize,
 }
 // eslint-disable-next-line import/no-default-export
 export default api

package/src/model/gpt-3.5-turbo.ts

@@ -14,6 +14,7 @@ const {
   isWithinTokenLimit,
   encodeChat,
   encodeChatGenerator,
+  vocabularySize,
 } = api
 export {
   decode,
@@ -24,6 +25,7 @@ export {
   encodeChatGenerator,
   encodeGenerator,
   isWithinTokenLimit,
+  vocabularySize,
 }
 // eslint-disable-next-line import/no-default-export
 export default api

package/src/model/gpt-4-0125-preview.ts

@@ -14,6 +14,7 @@ const {
   isWithinTokenLimit,
   encodeChat,
   encodeChatGenerator,
+  vocabularySize,
 } = api
 export {
   decode,
@@ -24,6 +25,7 @@ export {
   encodeChatGenerator,
   encodeGenerator,
   isWithinTokenLimit,
+  vocabularySize,
 }
 // eslint-disable-next-line import/no-default-export
 export default api

package/src/model/gpt-4-0314.ts

@@ -14,6 +14,7 @@ const {
   isWithinTokenLimit,
   encodeChat,
   encodeChatGenerator,
+  vocabularySize,
 } = api
 export {
   decode,
@@ -24,6 +25,7 @@ export {
   encodeChatGenerator,
   encodeGenerator,
   isWithinTokenLimit,
+  vocabularySize,
 }
 // eslint-disable-next-line import/no-default-export
 export default api

package/src/model/gpt-4-0613.ts

@@ -14,6 +14,7 @@ const {
   isWithinTokenLimit,
   encodeChat,
   encodeChatGenerator,
+  vocabularySize,
 } = api
 export {
   decode,
@@ -24,6 +25,7 @@ export {
   encodeChatGenerator,
   encodeGenerator,
   isWithinTokenLimit,
+  vocabularySize,
 }
 // eslint-disable-next-line import/no-default-export
 export default api

package/src/model/gpt-4-1106-preview.ts

@@ -14,6 +14,7 @@ const {
   isWithinTokenLimit,
   encodeChat,
   encodeChatGenerator,
+  vocabularySize,
 } = api
 export {
   decode,
@@ -24,6 +25,7 @@ export {
   encodeChatGenerator,
   encodeGenerator,
   isWithinTokenLimit,
+  vocabularySize,
 }
 // eslint-disable-next-line import/no-default-export
 export default api

package/src/model/gpt-4-1106-vision-preview.ts

@@ -14,6 +14,7 @@ const {
   isWithinTokenLimit,
   encodeChat,
   encodeChatGenerator,
+  vocabularySize,
 } = api
 export {
   decode,
@@ -24,6 +25,7 @@ export {
   encodeChatGenerator,
   encodeGenerator,
   isWithinTokenLimit,
+  vocabularySize,
 }
 // eslint-disable-next-line import/no-default-export
 export default api

package/src/model/gpt-4-32k-0314.ts

@@ -14,6 +14,7 @@ const {
   isWithinTokenLimit,
   encodeChat,
   encodeChatGenerator,
+  vocabularySize,
 } = api
 export {
   decode,
@@ -24,6 +25,7 @@ export {
   encodeChatGenerator,
   encodeGenerator,
   isWithinTokenLimit,
+  vocabularySize,
 }
 // eslint-disable-next-line import/no-default-export
 export default api

package/src/model/gpt-4-32k-0613.ts

@@ -14,6 +14,7 @@ const {
   isWithinTokenLimit,
   encodeChat,
   encodeChatGenerator,
+  vocabularySize,
 } = api
 export {
   decode,
@@ -24,6 +25,7 @@ export {
   encodeChatGenerator,
   encodeGenerator,
   isWithinTokenLimit,
+  vocabularySize,
 }
 // eslint-disable-next-line import/no-default-export
 export default api

package/src/model/gpt-4-32k.ts

@@ -14,6 +14,7 @@ const {
   isWithinTokenLimit,
   encodeChat,
   encodeChatGenerator,
+  vocabularySize,
 } = api
 export {
   decode,
@@ -24,6 +25,7 @@ export {
   encodeChatGenerator,
   encodeGenerator,
   isWithinTokenLimit,
+  vocabularySize,
 }
 // eslint-disable-next-line import/no-default-export
 export default api

package/src/model/gpt-4-turbo-2024-04-09.ts

@@ -14,6 +14,7 @@ const {
   isWithinTokenLimit,
   encodeChat,
   encodeChatGenerator,
+  vocabularySize,
 } = api
 export {
   decode,
@@ -24,6 +25,7 @@ export {
   encodeChatGenerator,
   encodeGenerator,
   isWithinTokenLimit,
+  vocabularySize,
 }
 // eslint-disable-next-line import/no-default-export
 export default api

package/src/model/gpt-4-turbo-preview.ts

@@ -14,6 +14,7 @@ const {
   isWithinTokenLimit,
   encodeChat,
   encodeChatGenerator,
+  vocabularySize,
 } = api
 export {
   decode,
@@ -24,6 +25,7 @@ export {
   encodeChatGenerator,
   encodeGenerator,
   isWithinTokenLimit,
+  vocabularySize,
 }
 // eslint-disable-next-line import/no-default-export
 export default api

package/src/model/gpt-4-turbo.ts

@@ -14,6 +14,7 @@ const {
   isWithinTokenLimit,
   encodeChat,
   encodeChatGenerator,
+  vocabularySize,
 } = api
 export {
   decode,
@@ -24,6 +25,7 @@ export {
   encodeChatGenerator,
   encodeGenerator,
   isWithinTokenLimit,
+  vocabularySize,
 }
 // eslint-disable-next-line import/no-default-export
 export default api

package/src/model/gpt-4-vision-preview.ts

@@ -14,6 +14,7 @@ const {
   isWithinTokenLimit,
   encodeChat,
   encodeChatGenerator,
+  vocabularySize,
 } = api
 export {
   decode,
@@ -24,6 +25,7 @@ export {
   encodeChatGenerator,
   encodeGenerator,
   isWithinTokenLimit,
+  vocabularySize,
 }
 // eslint-disable-next-line import/no-default-export
 export default api

package/src/model/gpt-4.ts

@@ -14,6 +14,7 @@ const {
   isWithinTokenLimit,
   encodeChat,
   encodeChatGenerator,
+  vocabularySize,
 } = api
 export {
   decode,
@@ -24,6 +25,7 @@ export {
   encodeChatGenerator,
   encodeGenerator,
   isWithinTokenLimit,
+  vocabularySize,
 }
 // eslint-disable-next-line import/no-default-export
 export default api

package/src/model/gpt-4o-2024-05-13.ts

@@ -14,6 +14,7 @@ const {
   isWithinTokenLimit,
   encodeChat,
   encodeChatGenerator,
+  vocabularySize,
 } = api
 export {
   decode,
@@ -24,6 +25,7 @@ export {
   encodeChatGenerator,
   encodeGenerator,
   isWithinTokenLimit,
+  vocabularySize,
 }
 // eslint-disable-next-line import/no-default-export
 export default api

package/src/model/gpt-4o-2024-08-06-finetune.ts

@@ -14,6 +14,7 @@ const {
   isWithinTokenLimit,
   encodeChat,
   encodeChatGenerator,
+  vocabularySize,
 } = api
 export {
   decode,
@@ -24,6 +25,7 @@ export {
   encodeChatGenerator,
   encodeGenerator,
   isWithinTokenLimit,
+  vocabularySize,
 }
 // eslint-disable-next-line import/no-default-export
 export default api

package/src/model/gpt-4o-2024-08-06.ts

@@ -14,6 +14,7 @@ const {
   isWithinTokenLimit,
   encodeChat,
   encodeChatGenerator,
+  vocabularySize,
 } = api
 export {
   decode,
@@ -24,6 +25,7 @@ export {
   encodeChatGenerator,
   encodeGenerator,
   isWithinTokenLimit,
+  vocabularySize,
 }
 // eslint-disable-next-line import/no-default-export
 export default api

package/src/model/gpt-4o-audio-preview-2024-10-01.ts

@@ -14,6 +14,7 @@ const {
   isWithinTokenLimit,
   encodeChat,
   encodeChatGenerator,
+  vocabularySize,
 } = api
 export {
   decode,
@@ -24,6 +25,7 @@ export {
   encodeChatGenerator,
   encodeGenerator,
   isWithinTokenLimit,
+  vocabularySize,
 }
 // eslint-disable-next-line import/no-default-export
 export default api

package/src/model/gpt-4o-audio-preview.ts

@@ -14,6 +14,7 @@ const {
   isWithinTokenLimit,
   encodeChat,
   encodeChatGenerator,
+  vocabularySize,
 } = api
 export {
   decode,
@@ -24,6 +25,7 @@ export {
   encodeChatGenerator,
   encodeGenerator,
   isWithinTokenLimit,
+  vocabularySize,
 }
 // eslint-disable-next-line import/no-default-export
 export default api

package/src/model/gpt-4o-mini-2024-07-18-finetune.ts

@@ -14,6 +14,7 @@ const {
   isWithinTokenLimit,
   encodeChat,
   encodeChatGenerator,
+  vocabularySize,
 } = api
 export {
   decode,
@@ -24,6 +25,7 @@ export {
   encodeChatGenerator,
   encodeGenerator,
   isWithinTokenLimit,
+  vocabularySize,
 }
 // eslint-disable-next-line import/no-default-export
 export default api

package/src/model/gpt-4o-mini-2024-07-18.ts

@@ -14,6 +14,7 @@ const {
   isWithinTokenLimit,
   encodeChat,
   encodeChatGenerator,
+  vocabularySize,
 } = api
 export {
   decode,
@@ -24,6 +25,7 @@ export {
   encodeChatGenerator,
   encodeGenerator,
   isWithinTokenLimit,
+  vocabularySize,
 }
 // eslint-disable-next-line import/no-default-export
 export default api