gpt-tokenizer 2.6.0 → 2.6.2

package/src/model/gpt-4.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/cl100k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-4', () => bpeRanks)
@@ -14,6 +15,7 @@ const {
   isWithinTokenLimit,
   encodeChat,
   encodeChatGenerator,
+  vocabularySize,
 } = api
 export {
   decode,
@@ -24,6 +26,7 @@ export {
   encodeChatGenerator,
   encodeGenerator,
   isWithinTokenLimit,
+  vocabularySize,
 }
 // eslint-disable-next-line import/no-default-export
 export default api

package/src/model/gpt-4o-2024-05-13.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/o200k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-4o-2024-05-13', () => bpeRanks)
@@ -14,6 +15,7 @@ const {
   isWithinTokenLimit,
   encodeChat,
   encodeChatGenerator,
+  vocabularySize,
 } = api
 export {
   decode,
@@ -24,6 +26,7 @@ export {
   encodeChatGenerator,
   encodeGenerator,
   isWithinTokenLimit,
+  vocabularySize,
 }
 // eslint-disable-next-line import/no-default-export
 export default api

package/src/model/gpt-4o-2024-08-06-finetune.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/o200k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-4o-2024-08-06-finetune', () => bpeRanks)
@@ -14,6 +15,7 @@ const {
   isWithinTokenLimit,
   encodeChat,
   encodeChatGenerator,
+  vocabularySize,
 } = api
 export {
   decode,
@@ -24,6 +26,7 @@ export {
   encodeChatGenerator,
   encodeGenerator,
   isWithinTokenLimit,
+  vocabularySize,
 }
 // eslint-disable-next-line import/no-default-export
 export default api

package/src/model/gpt-4o-2024-08-06.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/o200k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-4o-2024-08-06', () => bpeRanks)
@@ -14,6 +15,7 @@ const {
   isWithinTokenLimit,
   encodeChat,
   encodeChatGenerator,
+  vocabularySize,
 } = api
 export {
   decode,
@@ -24,6 +26,7 @@ export {
   encodeChatGenerator,
   encodeGenerator,
   isWithinTokenLimit,
+  vocabularySize,
 }
 // eslint-disable-next-line import/no-default-export
 export default api

package/src/model/gpt-4o-audio-preview-2024-10-01.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/o200k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-4o-audio-preview-2024-10-01', () => bpeRanks)
@@ -14,6 +15,7 @@ const {
   isWithinTokenLimit,
   encodeChat,
   encodeChatGenerator,
+  vocabularySize,
 } = api
 export {
   decode,
@@ -24,6 +26,7 @@ export {
   encodeChatGenerator,
   encodeGenerator,
   isWithinTokenLimit,
+  vocabularySize,
 }
 // eslint-disable-next-line import/no-default-export
 export default api

package/src/model/gpt-4o-audio-preview.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/o200k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-4o-audio-preview', () => bpeRanks)
@@ -14,6 +15,7 @@ const {
   isWithinTokenLimit,
   encodeChat,
   encodeChatGenerator,
+  vocabularySize,
 } = api
 export {
   decode,
@@ -24,6 +26,7 @@ export {
   encodeChatGenerator,
   encodeGenerator,
   isWithinTokenLimit,
+  vocabularySize,
 }
 // eslint-disable-next-line import/no-default-export
 export default api

package/src/model/gpt-4o-mini-2024-07-18-finetune.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/o200k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-4o-mini-2024-07-18-finetune', () => bpeRanks)
@@ -14,6 +15,7 @@ const {
   isWithinTokenLimit,
   encodeChat,
   encodeChatGenerator,
+  vocabularySize,
 } = api
 export {
   decode,
@@ -24,6 +26,7 @@ export {
   encodeChatGenerator,
   encodeGenerator,
   isWithinTokenLimit,
+  vocabularySize,
 }
 // eslint-disable-next-line import/no-default-export
 export default api

package/src/model/gpt-4o-mini-2024-07-18.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/o200k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-4o-mini-2024-07-18', () => bpeRanks)
@@ -14,6 +15,7 @@ const {
   isWithinTokenLimit,
   encodeChat,
   encodeChatGenerator,
+  vocabularySize,
 } = api
 export {
   decode,
@@ -24,6 +26,7 @@ export {
   encodeChatGenerator,
   encodeGenerator,
   isWithinTokenLimit,
+  vocabularySize,
 }
 // eslint-disable-next-line import/no-default-export
 export default api

package/src/model/gpt-4o-mini-training-2024-07-18.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/o200k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-4o-mini-training-2024-07-18', () => bpeRanks)
@@ -14,6 +15,7 @@ const {
   isWithinTokenLimit,
   encodeChat,
   encodeChatGenerator,
+  vocabularySize,
 } = api
 export {
   decode,
@@ -24,6 +26,7 @@ export {
   encodeChatGenerator,
   encodeGenerator,
   isWithinTokenLimit,
+  vocabularySize,
 }
 // eslint-disable-next-line import/no-default-export
 export default api

package/src/model/gpt-4o-mini-training.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/o200k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-4o-mini-training', () => bpeRanks)
@@ -14,6 +15,7 @@ const {
   isWithinTokenLimit,
   encodeChat,
   encodeChatGenerator,
+  vocabularySize,
 } = api
 export {
   decode,
@@ -24,6 +26,7 @@ export {
   encodeChatGenerator,
   encodeGenerator,
   isWithinTokenLimit,
+  vocabularySize,
 }
 // eslint-disable-next-line import/no-default-export
 export default api

package/src/model/gpt-4o-mini.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/o200k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-4o-mini', () => bpeRanks)
@@ -14,6 +15,7 @@ const {
   isWithinTokenLimit,
   encodeChat,
   encodeChatGenerator,
+  vocabularySize,
 } = api
 export {
   decode,
@@ -24,6 +26,7 @@ export {
   encodeChatGenerator,
   encodeGenerator,
   isWithinTokenLimit,
+  vocabularySize,
 }
 // eslint-disable-next-line import/no-default-export
 export default api

package/src/model/gpt-4o-realtime-preview-2024-10-01.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/o200k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-4o-realtime-preview-2024-10-01', () => bpeRanks)
@@ -14,6 +15,7 @@ const {
   isWithinTokenLimit,
   encodeChat,
   encodeChatGenerator,
+  vocabularySize,
 } = api
 export {
   decode,
@@ -24,6 +26,7 @@ export {
   encodeChatGenerator,
   encodeGenerator,
   isWithinTokenLimit,
+  vocabularySize,
 }
 // eslint-disable-next-line import/no-default-export
 export default api

package/src/model/gpt-4o-realtime-preview.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/o200k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-4o-realtime-preview', () => bpeRanks)
@@ -14,6 +15,7 @@ const {
   isWithinTokenLimit,
   encodeChat,
   encodeChatGenerator,
+  vocabularySize,
 } = api
 export {
   decode,
@@ -24,6 +26,7 @@ export {
   encodeChatGenerator,
   encodeGenerator,
   isWithinTokenLimit,
+  vocabularySize,
 }
 // eslint-disable-next-line import/no-default-export
 export default api

package/src/model/gpt-4o.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/o200k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-4o', () => bpeRanks)
@@ -14,6 +15,7 @@ const {
   isWithinTokenLimit,
   encodeChat,
   encodeChatGenerator,
+  vocabularySize,
 } = api
 export {
   decode,
@@ -24,6 +26,7 @@ export {
   encodeChatGenerator,
   encodeGenerator,
   isWithinTokenLimit,
+  vocabularySize,
 }
 // eslint-disable-next-line import/no-default-export
 export default api

package/src/modelParams.ts

@@ -22,11 +22,8 @@ export interface EncodingParams extends BytePairEncodingConfig {
    * It's complex due to its need to deal with a wide variety of cases in text processing.
    */
   tokenSplitRegex: RegExp
-  specialTokenMapping: Map<string, number>
+  specialTokensEncoder: Map<string, number>
   modelName?: ModelName
-
-  /** increases memory consumption, but speeds up subsequent decoding */
-  enableCache?: boolean
 }
 
 export const tokenSplitRegex =

package/src/models.ts

@@ -1,4 +1,4 @@
-import type { EncodingName } from './mapping'
+import type { EncodingName } from './mapping.js'
 
 // this data is compiled from:
 // - https://platform.openai.com/docs/models/