gpt-tokenizer 2.6.0 → 2.6.2

package/src/encoding/o200k_base.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/o200k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 
 const api = GptEncoding.getEncodingApi('o200k_base', () => bpeRanks)
@@ -14,6 +15,7 @@ const {
   isWithinTokenLimit,
   encodeChat,
   encodeChatGenerator,
+  vocabularySize,
 } = api
 export {
   decode,
@@ -24,6 +26,7 @@ export {
   encodeChatGenerator,
   encodeGenerator,
   isWithinTokenLimit,
+  vocabularySize,
 }
 // eslint-disable-next-line import/no-default-export
 export default api

package/src/encoding/p50k_base.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/p50k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 
 const api = GptEncoding.getEncodingApi('p50k_base', () => bpeRanks)
@@ -12,6 +13,7 @@ const {
   encode,
   encodeGenerator,
   isWithinTokenLimit,
+  vocabularySize,
 } = api
 export {
   decode,
@@ -20,6 +22,7 @@ export {
   encode,
   encodeGenerator,
   isWithinTokenLimit,
+  vocabularySize,
 }
 // eslint-disable-next-line import/no-default-export
 export default api

package/src/encoding/p50k_edit.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/p50k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 
 const api = GptEncoding.getEncodingApi('p50k_edit', () => bpeRanks)
@@ -12,6 +13,7 @@ const {
   encode,
   encodeGenerator,
   isWithinTokenLimit,
+  vocabularySize,
 } = api
 export {
   decode,
@@ -20,6 +22,7 @@ export {
   encode,
   encodeGenerator,
   isWithinTokenLimit,
+  vocabularySize,
 }
 // eslint-disable-next-line import/no-default-export
 export default api

package/src/encoding/r50k_base.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/r50k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 
 const api = GptEncoding.getEncodingApi('r50k_base', () => bpeRanks)
@@ -12,6 +13,7 @@ const {
   encode,
   encodeGenerator,
   isWithinTokenLimit,
+  vocabularySize,
 } = api
 export {
   decode,
@@ -20,6 +22,7 @@ export {
   encode,
   encodeGenerator,
   isWithinTokenLimit,
+  vocabularySize,
 }
 // eslint-disable-next-line import/no-default-export
 export default api

package/src/encodingParams/Cl100KBase.ts

@@ -13,7 +13,7 @@ import {
 } from '../specialTokens.js'
 
 export function Cl100KBase(
-  mergeableBytePairRanks: RawBytePairRanks,
+  bytePairRankDecoder: RawBytePairRanks,
 ): EncodingParams {
   const specialTokenMapping = new Map<string, number>([
     [EndOfText, 100_257],
@@ -29,7 +29,7 @@ export function Cl100KBase(
   return {
     tokenSplitRegex:
       /(?:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}{1,3}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+/giu,
-    mergeableBytePairRanks,
-    specialTokenMapping,
+    bytePairRankDecoder,
+    specialTokensEncoder: specialTokenMapping,
   }
 }

package/src/encodingParams/O200KBase.ts

@@ -13,7 +13,7 @@ import {
 } from '../specialTokens.js'
 
 export function O200KBase(
-  mergeableBytePairRanks: RawBytePairRanks,
+  bytePairRankDecoder: RawBytePairRanks,
 ): EncodingParams {
   const specialTokenMapping = new Map<string, number>([
     [EndOfText, 199_999],
@@ -29,7 +29,7 @@ export function O200KBase(
   return {
     tokenSplitRegex:
       /(?:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}{1,3}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+/giu,
-    mergeableBytePairRanks,
-    specialTokenMapping,
+    bytePairRankDecoder,
+    specialTokensEncoder: specialTokenMapping,
   }
 }

package/src/encodingParams/P50KBase.ts

@@ -4,12 +4,12 @@ import { type EncodingParams, tokenSplitRegex } from '../modelParams.js'
 import { EndOfText } from '../specialTokens.js'
 
 export function P50KBase(
-  mergeableBytePairRanks: RawBytePairRanks,
+  bytePairRankDecoder: RawBytePairRanks,
 ): EncodingParams {
   return {
     expectedVocabularySize: 50_281,
     tokenSplitRegex,
-    mergeableBytePairRanks,
-    specialTokenMapping: new Map<string, number>([[EndOfText, 50_256]]),
+    bytePairRankDecoder,
+    specialTokensEncoder: new Map<string, number>([[EndOfText, 50_256]]),
   }
 }

package/src/encodingParams/P50KEdit.ts

@@ -4,7 +4,7 @@ import { type EncodingParams, tokenSplitRegex } from '../modelParams.js'
 import { EndOfText, FimMiddle, FimPrefix, FimSuffix } from '../specialTokens.js'
 
 export function P50KEdit(
-  mergeableBytePairRanks: RawBytePairRanks,
+  bytePairRankDecoder: RawBytePairRanks,
 ): EncodingParams {
   const specialTokenMapping = new Map<string, number>([
     [EndOfText, 50_256],
@@ -15,7 +15,7 @@ export function P50KEdit(
 
   return {
     tokenSplitRegex,
-    mergeableBytePairRanks,
-    specialTokenMapping,
+    bytePairRankDecoder,
+    specialTokensEncoder: specialTokenMapping,
   }
 }

package/src/encodingParams/R50KBase.ts

@@ -4,12 +4,12 @@ import { type EncodingParams, tokenSplitRegex } from '../modelParams.js'
 import { EndOfText } from '../specialTokens.js'
 
 export function R50KBase(
-  mergeableBytePairRanks: RawBytePairRanks,
+  bytePairRankDecoder: RawBytePairRanks,
 ): EncodingParams {
   return {
     expectedVocabularySize: 50_257,
     tokenSplitRegex,
-    mergeableBytePairRanks,
-    specialTokenMapping: new Map<string, number>([[EndOfText, 50_256]]),
+    bytePairRankDecoder,
+    specialTokensEncoder: new Map<string, number>([[EndOfText, 50_256]]),
   }
 }

package/src/model/gpt-3.5-turbo-0125.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/cl100k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-3.5-turbo-0125', () => bpeRanks)
@@ -14,6 +15,7 @@ const {
   isWithinTokenLimit,
   encodeChat,
   encodeChatGenerator,
+  vocabularySize,
 } = api
 export {
   decode,
@@ -24,6 +26,7 @@ export {
   encodeChatGenerator,
   encodeGenerator,
   isWithinTokenLimit,
+  vocabularySize,
 }
 // eslint-disable-next-line import/no-default-export
 export default api

package/src/model/gpt-3.5-turbo-0301.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/cl100k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-3.5-turbo-0301', () => bpeRanks)
@@ -14,6 +15,7 @@ const {
   isWithinTokenLimit,
   encodeChat,
   encodeChatGenerator,
+  vocabularySize,
 } = api
 export {
   decode,
@@ -24,6 +26,7 @@ export {
   encodeChatGenerator,
   encodeGenerator,
   isWithinTokenLimit,
+  vocabularySize,
 }
 // eslint-disable-next-line import/no-default-export
 export default api

package/src/model/gpt-3.5-turbo-0613.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/cl100k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-3.5-turbo-0613', () => bpeRanks)
@@ -14,6 +15,7 @@ const {
   isWithinTokenLimit,
   encodeChat,
   encodeChatGenerator,
+  vocabularySize,
 } = api
 export {
   decode,
@@ -24,6 +26,7 @@ export {
   encodeChatGenerator,
   encodeGenerator,
   isWithinTokenLimit,
+  vocabularySize,
 }
 // eslint-disable-next-line import/no-default-export
 export default api

package/src/model/gpt-3.5-turbo-1106.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/cl100k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-3.5-turbo-1106', () => bpeRanks)
@@ -14,6 +15,7 @@ const {
   isWithinTokenLimit,
   encodeChat,
   encodeChatGenerator,
+  vocabularySize,
 } = api
 export {
   decode,
@@ -24,6 +26,7 @@ export {
   encodeChatGenerator,
   encodeGenerator,
   isWithinTokenLimit,
+  vocabularySize,
 }
 // eslint-disable-next-line import/no-default-export
 export default api

package/src/model/gpt-3.5-turbo-16k-0613.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/cl100k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-3.5-turbo-16k-0613', () => bpeRanks)
@@ -14,6 +15,7 @@ const {
   isWithinTokenLimit,
   encodeChat,
   encodeChatGenerator,
+  vocabularySize,
 } = api
 export {
   decode,
@@ -24,6 +26,7 @@ export {
   encodeChatGenerator,
   encodeGenerator,
   isWithinTokenLimit,
+  vocabularySize,
 }
 // eslint-disable-next-line import/no-default-export
 export default api

package/src/model/gpt-3.5-turbo-16k.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/cl100k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-3.5-turbo-16k', () => bpeRanks)
@@ -14,6 +15,7 @@ const {
   isWithinTokenLimit,
   encodeChat,
   encodeChatGenerator,
+  vocabularySize,
 } = api
 export {
   decode,
@@ -24,6 +26,7 @@ export {
   encodeChatGenerator,
   encodeGenerator,
   isWithinTokenLimit,
+  vocabularySize,
 }
 // eslint-disable-next-line import/no-default-export
 export default api

package/src/model/gpt-3.5-turbo-finetune.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/cl100k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-3.5-turbo-finetune', () => bpeRanks)
@@ -14,6 +15,7 @@ const {
   isWithinTokenLimit,
   encodeChat,
   encodeChatGenerator,
+  vocabularySize,
 } = api
 export {
   decode,
@@ -24,6 +26,7 @@ export {
   encodeChatGenerator,
   encodeGenerator,
   isWithinTokenLimit,
+  vocabularySize,
 }
 // eslint-disable-next-line import/no-default-export
 export default api

package/src/model/gpt-3.5-turbo.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/cl100k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-3.5-turbo', () => bpeRanks)
@@ -14,6 +15,7 @@ const {
   isWithinTokenLimit,
   encodeChat,
   encodeChatGenerator,
+  vocabularySize,
 } = api
 export {
   decode,
@@ -24,6 +26,7 @@ export {
   encodeChatGenerator,
   encodeGenerator,
   isWithinTokenLimit,
+  vocabularySize,
 }
 // eslint-disable-next-line import/no-default-export
 export default api

package/src/model/gpt-4-0125-preview.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/cl100k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-4-0125-preview', () => bpeRanks)
@@ -14,6 +15,7 @@ const {
   isWithinTokenLimit,
   encodeChat,
   encodeChatGenerator,
+  vocabularySize,
 } = api
 export {
   decode,
@@ -24,6 +26,7 @@ export {
   encodeChatGenerator,
   encodeGenerator,
   isWithinTokenLimit,
+  vocabularySize,
 }
 // eslint-disable-next-line import/no-default-export
 export default api

package/src/model/gpt-4-0314.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/cl100k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-4-0314', () => bpeRanks)
@@ -14,6 +15,7 @@ const {
   isWithinTokenLimit,
   encodeChat,
   encodeChatGenerator,
+  vocabularySize,
 } = api
 export {
   decode,
@@ -24,6 +26,7 @@ export {
   encodeChatGenerator,
   encodeGenerator,
   isWithinTokenLimit,
+  vocabularySize,
 }
 // eslint-disable-next-line import/no-default-export
 export default api

package/src/model/gpt-4-0613.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/cl100k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-4-0613', () => bpeRanks)
@@ -14,6 +15,7 @@ const {
   isWithinTokenLimit,
   encodeChat,
   encodeChatGenerator,
+  vocabularySize,
 } = api
 export {
   decode,
@@ -24,6 +26,7 @@ export {
   encodeChatGenerator,
   encodeGenerator,
   isWithinTokenLimit,
+  vocabularySize,
 }
 // eslint-disable-next-line import/no-default-export
 export default api

package/src/model/gpt-4-1106-preview.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/cl100k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-4-1106-preview', () => bpeRanks)
@@ -14,6 +15,7 @@ const {
   isWithinTokenLimit,
   encodeChat,
   encodeChatGenerator,
+  vocabularySize,
 } = api
 export {
   decode,
@@ -24,6 +26,7 @@ export {
   encodeChatGenerator,
   encodeGenerator,
   isWithinTokenLimit,
+  vocabularySize,
 }
 // eslint-disable-next-line import/no-default-export
 export default api

package/src/model/gpt-4-1106-vision-preview.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/cl100k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-4-1106-vision-preview', () => bpeRanks)
@@ -14,6 +15,7 @@ const {
   isWithinTokenLimit,
   encodeChat,
   encodeChatGenerator,
+  vocabularySize,
 } = api
 export {
   decode,
@@ -24,6 +26,7 @@ export {
   encodeChatGenerator,
   encodeGenerator,
   isWithinTokenLimit,
+  vocabularySize,
 }
 // eslint-disable-next-line import/no-default-export
 export default api

package/src/model/gpt-4-32k-0314.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/cl100k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-4-32k-0314', () => bpeRanks)
@@ -14,6 +15,7 @@ const {
   isWithinTokenLimit,
   encodeChat,
   encodeChatGenerator,
+  vocabularySize,
 } = api
 export {
   decode,
@@ -24,6 +26,7 @@ export {
   encodeChatGenerator,
   encodeGenerator,
   isWithinTokenLimit,
+  vocabularySize,
 }
 // eslint-disable-next-line import/no-default-export
 export default api

package/src/model/gpt-4-32k-0613.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/cl100k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-4-32k-0613', () => bpeRanks)
@@ -14,6 +15,7 @@ const {
   isWithinTokenLimit,
   encodeChat,
   encodeChatGenerator,
+  vocabularySize,
 } = api
 export {
   decode,
@@ -24,6 +26,7 @@ export {
   encodeChatGenerator,
   encodeGenerator,
   isWithinTokenLimit,
+  vocabularySize,
 }
 // eslint-disable-next-line import/no-default-export
 export default api

package/src/model/gpt-4-32k.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/cl100k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-4-32k', () => bpeRanks)
@@ -14,6 +15,7 @@ const {
   isWithinTokenLimit,
   encodeChat,
   encodeChatGenerator,
+  vocabularySize,
 } = api
 export {
   decode,
@@ -24,6 +26,7 @@ export {
   encodeChatGenerator,
   encodeGenerator,
   isWithinTokenLimit,
+  vocabularySize,
 }
 // eslint-disable-next-line import/no-default-export
 export default api

package/src/model/gpt-4-turbo-2024-04-09.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/cl100k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-4-turbo-2024-04-09', () => bpeRanks)
@@ -14,6 +15,7 @@ const {
   isWithinTokenLimit,
   encodeChat,
   encodeChatGenerator,
+  vocabularySize,
 } = api
 export {
   decode,
@@ -24,6 +26,7 @@ export {
   encodeChatGenerator,
   encodeGenerator,
   isWithinTokenLimit,
+  vocabularySize,
 }
 // eslint-disable-next-line import/no-default-export
 export default api

package/src/model/gpt-4-turbo-preview.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/cl100k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-4-turbo-preview', () => bpeRanks)
@@ -14,6 +15,7 @@ const {
   isWithinTokenLimit,
   encodeChat,
   encodeChatGenerator,
+  vocabularySize,
 } = api
 export {
   decode,
@@ -24,6 +26,7 @@ export {
   encodeChatGenerator,
   encodeGenerator,
   isWithinTokenLimit,
+  vocabularySize,
 }
 // eslint-disable-next-line import/no-default-export
 export default api

package/src/model/gpt-4-turbo.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/cl100k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-4-turbo', () => bpeRanks)
@@ -14,6 +15,7 @@ const {
   isWithinTokenLimit,
   encodeChat,
   encodeChatGenerator,
+  vocabularySize,
 } = api
 export {
   decode,
@@ -24,6 +26,7 @@ export {
   encodeChatGenerator,
   encodeGenerator,
   isWithinTokenLimit,
+  vocabularySize,
 }
 // eslint-disable-next-line import/no-default-export
 export default api

package/src/model/gpt-4-vision-preview.ts

@@ -2,6 +2,7 @@
 import bpeRanks from '../bpeRanks/cl100k_base.js'
 import { GptEncoding } from '../GptEncoding.js'
 
+export * from '../constants.js'
 export * from '../specialTokens.js'
 // prettier-ignore
 const api = GptEncoding.getEncodingApiForModel('gpt-4-vision-preview', () => bpeRanks)
@@ -14,6 +15,7 @@ const {
   isWithinTokenLimit,
   encodeChat,
   encodeChatGenerator,
+  vocabularySize,
 } = api
 export {
   decode,
@@ -24,6 +26,7 @@ export {
   encodeChatGenerator,
   encodeGenerator,
   isWithinTokenLimit,
+  vocabularySize,
 }
 // eslint-disable-next-line import/no-default-export
 export default api