gpt-tokenizer 2.0.0 → 2.1.1

package/esm/model/text-embedding-ada-002.js

@@ -1,2 +1,4 @@
+// eslint-disable-next-line no-restricted-exports, import/no-default-export
+export { default } from '../encoding/cl100k_base.js';
 export * from '../encoding/cl100k_base.js';
 //# sourceMappingURL=text-embedding-ada-002.js.map

package/esm/model/text-embedding-ada-002.js.map

@@ -1 +1 @@
-{"version":3,"file":"text-embedding-ada-002.js","sourceRoot":"","sources":["../../src/model/text-embedding-ada-002.ts"],"names":[],"mappings":"AAAA,cAAc,4BAA4B,CAAA"}
+{"version":3,"file":"text-embedding-ada-002.js","sourceRoot":"","sources":["../../src/model/text-embedding-ada-002.ts"],"names":[],"mappings":"AAAA,2EAA2E;AAC3E,OAAO,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAA;AACpD,cAAc,4BAA4B,CAAA"}

package/esm/model/text-search-ada-doc-001.d.ts

@@ -1 +1,2 @@
+export { default } from '../encoding/r50k_base.js';
 export * from '../encoding/r50k_base.js';

package/esm/model/text-search-ada-doc-001.js

@@ -1,2 +1,4 @@
+// eslint-disable-next-line no-restricted-exports, import/no-default-export
+export { default } from '../encoding/r50k_base.js';
 export * from '../encoding/r50k_base.js';
 //# sourceMappingURL=text-search-ada-doc-001.js.map

package/esm/model/text-search-ada-doc-001.js.map

@@ -1 +1 @@
-{"version":3,"file":"text-search-ada-doc-001.js","sourceRoot":"","sources":["../../src/model/text-search-ada-doc-001.ts"],"names":[],"mappings":"AAAA,cAAc,0BAA0B,CAAA"}
+{"version":3,"file":"text-search-ada-doc-001.js","sourceRoot":"","sources":["../../src/model/text-search-ada-doc-001.ts"],"names":[],"mappings":"AAAA,2EAA2E;AAC3E,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAA;AAClD,cAAc,0BAA0B,CAAA"}

package/esm/model/text-search-babbage-doc-001.d.ts

@@ -1 +1,2 @@
+export { default } from '../encoding/r50k_base.js';
 export * from '../encoding/r50k_base.js';

package/esm/model/text-search-babbage-doc-001.js

@@ -1,2 +1,4 @@
+// eslint-disable-next-line no-restricted-exports, import/no-default-export
+export { default } from '../encoding/r50k_base.js';
 export * from '../encoding/r50k_base.js';
 //# sourceMappingURL=text-search-babbage-doc-001.js.map

package/esm/model/text-search-babbage-doc-001.js.map

@@ -1 +1 @@
-{"version":3,"file":"text-search-babbage-doc-001.js","sourceRoot":"","sources":["../../src/model/text-search-babbage-doc-001.ts"],"names":[],"mappings":"AAAA,cAAc,0BAA0B,CAAA"}
+{"version":3,"file":"text-search-babbage-doc-001.js","sourceRoot":"","sources":["../../src/model/text-search-babbage-doc-001.ts"],"names":[],"mappings":"AAAA,2EAA2E;AAC3E,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAA;AAClD,cAAc,0BAA0B,CAAA"}

package/esm/model/text-search-curie-doc-001.d.ts

@@ -1 +1,2 @@
+export { default } from '../encoding/r50k_base.js';
 export * from '../encoding/r50k_base.js';

package/esm/model/text-search-curie-doc-001.js

@@ -1,2 +1,4 @@
+// eslint-disable-next-line no-restricted-exports, import/no-default-export
+export { default } from '../encoding/r50k_base.js';
 export * from '../encoding/r50k_base.js';
 //# sourceMappingURL=text-search-curie-doc-001.js.map

package/esm/model/text-search-curie-doc-001.js.map

@@ -1 +1 @@
-{"version":3,"file":"text-search-curie-doc-001.js","sourceRoot":"","sources":["../../src/model/text-search-curie-doc-001.ts"],"names":[],"mappings":"AAAA,cAAc,0BAA0B,CAAA"}
+{"version":3,"file":"text-search-curie-doc-001.js","sourceRoot":"","sources":["../../src/model/text-search-curie-doc-001.ts"],"names":[],"mappings":"AAAA,2EAA2E;AAC3E,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAA;AAClD,cAAc,0BAA0B,CAAA"}

package/esm/model/text-search-davinci-doc-001.d.ts

@@ -1 +1,2 @@
+export { default } from '../encoding/r50k_base.js';
 export * from '../encoding/r50k_base.js';

package/esm/model/text-search-davinci-doc-001.js

@@ -1,2 +1,4 @@
+// eslint-disable-next-line no-restricted-exports, import/no-default-export
+export { default } from '../encoding/r50k_base.js';
 export * from '../encoding/r50k_base.js';
 //# sourceMappingURL=text-search-davinci-doc-001.js.map

package/esm/model/text-search-davinci-doc-001.js.map

@@ -1 +1 @@
-{"version":3,"file":"text-search-davinci-doc-001.js","sourceRoot":"","sources":["../../src/model/text-search-davinci-doc-001.ts"],"names":[],"mappings":"AAAA,cAAc,0BAA0B,CAAA"}
+{"version":3,"file":"text-search-davinci-doc-001.js","sourceRoot":"","sources":["../../src/model/text-search-davinci-doc-001.ts"],"names":[],"mappings":"AAAA,2EAA2E;AAC3E,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAA;AAClD,cAAc,0BAA0B,CAAA"}

package/esm/model/text-similarity-ada-001.d.ts

@@ -1 +1,2 @@
+export { default } from '../encoding/r50k_base.js';
 export * from '../encoding/r50k_base.js';

package/esm/model/text-similarity-ada-001.js

@@ -1,2 +1,4 @@
+// eslint-disable-next-line no-restricted-exports, import/no-default-export
+export { default } from '../encoding/r50k_base.js';
 export * from '../encoding/r50k_base.js';
 //# sourceMappingURL=text-similarity-ada-001.js.map

package/esm/model/text-similarity-ada-001.js.map

@@ -1 +1 @@
-{"version":3,"file":"text-similarity-ada-001.js","sourceRoot":"","sources":["../../src/model/text-similarity-ada-001.ts"],"names":[],"mappings":"AAAA,cAAc,0BAA0B,CAAA"}
+{"version":3,"file":"text-similarity-ada-001.js","sourceRoot":"","sources":["../../src/model/text-similarity-ada-001.ts"],"names":[],"mappings":"AAAA,2EAA2E;AAC3E,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAA;AAClD,cAAc,0BAA0B,CAAA"}

package/esm/model/text-similarity-babbage-001.d.ts

@@ -1 +1,2 @@
+export { default } from '../encoding/r50k_base.js';
 export * from '../encoding/r50k_base.js';

package/esm/model/text-similarity-babbage-001.js

@@ -1,2 +1,4 @@
+// eslint-disable-next-line no-restricted-exports, import/no-default-export
+export { default } from '../encoding/r50k_base.js';
 export * from '../encoding/r50k_base.js';
 //# sourceMappingURL=text-similarity-babbage-001.js.map

package/esm/model/text-similarity-babbage-001.js.map

@@ -1 +1 @@
-{"version":3,"file":"text-similarity-babbage-001.js","sourceRoot":"","sources":["../../src/model/text-similarity-babbage-001.ts"],"names":[],"mappings":"AAAA,cAAc,0BAA0B,CAAA"}
+{"version":3,"file":"text-similarity-babbage-001.js","sourceRoot":"","sources":["../../src/model/text-similarity-babbage-001.ts"],"names":[],"mappings":"AAAA,2EAA2E;AAC3E,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAA;AAClD,cAAc,0BAA0B,CAAA"}

package/esm/model/text-similarity-curie-001.d.ts

@@ -1 +1,2 @@
+export { default } from '../encoding/r50k_base.js';
 export * from '../encoding/r50k_base.js';

package/esm/model/text-similarity-curie-001.js

@@ -1,2 +1,4 @@
+// eslint-disable-next-line no-restricted-exports, import/no-default-export
+export { default } from '../encoding/r50k_base.js';
 export * from '../encoding/r50k_base.js';
 //# sourceMappingURL=text-similarity-curie-001.js.map

package/esm/model/text-similarity-curie-001.js.map

@@ -1 +1 @@
-{"version":3,"file":"text-similarity-curie-001.js","sourceRoot":"","sources":["../../src/model/text-similarity-curie-001.ts"],"names":[],"mappings":"AAAA,cAAc,0BAA0B,CAAA"}
+{"version":3,"file":"text-similarity-curie-001.js","sourceRoot":"","sources":["../../src/model/text-similarity-curie-001.ts"],"names":[],"mappings":"AAAA,2EAA2E;AAC3E,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAA;AAClD,cAAc,0BAA0B,CAAA"}

package/esm/model/text-similarity-davinci-001.d.ts

@@ -1 +1,2 @@
+export { default } from '../encoding/r50k_base.js';
 export * from '../encoding/r50k_base.js';

package/esm/model/text-similarity-davinci-001.js

@@ -1,2 +1,4 @@
+// eslint-disable-next-line no-restricted-exports, import/no-default-export
+export { default } from '../encoding/r50k_base.js';
 export * from '../encoding/r50k_base.js';
 //# sourceMappingURL=text-similarity-davinci-001.js.map

package/esm/model/text-similarity-davinci-001.js.map

@@ -1 +1 @@
-{"version":3,"file":"text-similarity-davinci-001.js","sourceRoot":"","sources":["../../src/model/text-similarity-davinci-001.ts"],"names":[],"mappings":"AAAA,cAAc,0BAA0B,CAAA"}
+{"version":3,"file":"text-similarity-davinci-001.js","sourceRoot":"","sources":["../../src/model/text-similarity-davinci-001.ts"],"names":[],"mappings":"AAAA,2EAA2E;AAC3E,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAA;AAClD,cAAc,0BAA0B,CAAA"}

package/esm/modelParams.d.ts

@@ -1,6 +1,6 @@
 import { EncoderMap } from './EncoderMap.js';
-import type { EncodingName } from './mapping.js';
-export interface ModelParams {
+import type { EncodingName, ModelName } from './mapping.js';
+export interface EncodingParams {
     /**
      * The expected total number of tokens in the vocabulary, including both regular and special tokens.
      * This parameter is used to ensure that the combined number of regular and special tokens matches the expected total.
@@ -15,8 +15,9 @@ export interface ModelParams {
     tokenSplitRegex: RegExp;
     mergeableBytePairRanks: EncoderMap;
     specialTokenMapping: Map<string, number>;
+    modelName?: ModelName;
 }
 export type GetMergeableRanksFn = (encodingName: EncodingName) => EncoderMap;
 export type GetMergeableRanksAsyncFn = (encodingName: EncodingName) => Promise<EncoderMap>;
-export declare function getModelParams(encodingName: EncodingName, getMergeableRanks: GetMergeableRanksFn): ModelParams;
-export declare function getModelParamsAsync(encodingName: EncodingName, getMergeableRanks: GetMergeableRanksAsyncFn): Promise<ModelParams>;
+export declare function getEncodingParams(encodingName: EncodingName, getMergeableRanks: GetMergeableRanksFn): EncodingParams;
+export declare function getModelParamsAsync(encodingName: EncodingName, getMergeableRanks: GetMergeableRanksAsyncFn): Promise<EncodingParams>;

package/esm/modelParams.js

@@ -1,6 +1,6 @@
 /* eslint-disable no-magic-numbers */
 import { EncoderMap } from './EncoderMap.js';
-import { EndOfPrompt, EndOfText, FimMiddle, FimPrefix, FimSuffix, } from './specialTokens.js';
+import { EndOfPrompt, EndOfText, FimMiddle, FimPrefix, FimSuffix, ImEnd, ImSep, ImStart, } from './specialTokens.js';
 const tokenSplitRegex = /'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+/gu;
 function R50KBase(mergeableBytePairRanks) {
     return {
@@ -37,6 +37,9 @@ function Cl100KBase(mergeableBytePairRanks) {
         [FimPrefix, 100258],
         [FimMiddle, 100259],
         [FimSuffix, 100260],
+        [ImStart, 100264],
+        [ImEnd, 100265],
+        [ImSep, 100266],
         [EndOfPrompt, 100276],
     ]);
     return {
@@ -45,7 +48,7 @@ function Cl100KBase(mergeableBytePairRanks) {
         specialTokenMapping,
     };
 }
-export function getModelParams(encodingName, getMergeableRanks) {
+export function getEncodingParams(encodingName, getMergeableRanks) {
     const mergeableBytePairRanks = getMergeableRanks(encodingName);
     switch (encodingName.toLowerCase()) {
         case 'r50k_base':
@@ -62,6 +65,6 @@ export function getModelParams(encodingName, getMergeableRanks) {
 }
 export async function getModelParamsAsync(encodingName, getMergeableRanks) {
     const mergeableBytePairRanks = await getMergeableRanks(encodingName);
-    return getModelParams(encodingName, () => mergeableBytePairRanks);
+    return getEncodingParams(encodingName, () => mergeableBytePairRanks);
 }
 //# sourceMappingURL=modelParams.js.map

package/esm/modelParams.js.map

@@ -1 +1 @@
-{"version":3,"file":"modelParams.js","sourceRoot":"","sources":["../src/modelParams.ts"],"names":[],"mappings":"AAAA,qCAAqC;AACrC,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAA;AAE5C,OAAO,EACL,WAAW,EACX,SAAS,EACT,SAAS,EACT,SAAS,EACT,SAAS,GACV,MAAM,oBAAoB,CAAA;AAmB3B,MAAM,eAAe,GACnB,8EAA8E,CAAA;AAEhF,SAAS,QAAQ,CAAC,sBAAkC;IAClD,OAAO;QACL,sBAAsB,EAAE,KAAM;QAC9B,eAAe;QACf,sBAAsB;QACtB,mBAAmB,EAAE,IAAI,GAAG,CAAiB,CAAC,CAAC,SAAS,EAAE,KAAM,CAAC,CAAC,CAAC;KACpE,CAAA;AACH,CAAC;AAED,SAAS,QAAQ,CAAC,sBAAkC;IAClD,OAAO;QACL,sBAAsB,EAAE,KAAM;QAC9B,eAAe;QACf,sBAAsB;QACtB,mBAAmB,EAAE,IAAI,GAAG,CAAiB,CAAC,CAAC,SAAS,EAAE,KAAM,CAAC,CAAC,CAAC;KACpE,CAAA;AACH,CAAC;AAED,SAAS,QAAQ,CAAC,sBAAkC;IAClD,MAAM,mBAAmB,GAAG,IAAI,GAAG,CAAiB;QAClD,CAAC,SAAS,EAAE,KAAM,CAAC;QACnB,CAAC,SAAS,EAAE,KAAM,CAAC;QACnB,CAAC,SAAS,EAAE,KAAM,CAAC;QACnB,CAAC,SAAS,EAAE,KAAM,CAAC;KACpB,CAAC,CAAA;IAEF,OAAO;QACL,eAAe;QACf,sBAAsB;QACtB,mBAAmB;KACpB,CAAA;AACH,CAAC;AAED,SAAS,UAAU,CAAC,sBAAkC;IACpD,MAAM,mBAAmB,GAAG,IAAI,GAAG,CAAiB;QAClD,CAAC,SAAS,EAAE,MAAO,CAAC;QACpB,CAAC,SAAS,EAAE,MAAO,CAAC;QACpB,CAAC,SAAS,EAAE,MAAO,CAAC;QACpB,CAAC,SAAS,EAAE,MAAO,CAAC;QACpB,CAAC,WAAW,EAAE,MAAO,CAAC;KACvB,CAAC,CAAA;IAEF,OAAO;QACL,eAAe,EACb,uHAAuH;QACzH,sBAAsB;QACtB,mBAAmB;KACpB,CAAA;AACH,CAAC;AAOD,MAAM,UAAU,cAAc,CAC5B,YAA0B,EAC1B,iBAAsC;IAEtC,MAAM,sBAAsB,GAAG,iBAAiB,CAAC,YAAY,CAAC,CAAA;IAC9D,QAAQ,YAAY,CAAC,WAAW,EAAE,EAAE;QAClC,KAAK,WAAW;YACd,OAAO,QAAQ,CAAC,sBAAsB,CAAC,CAAA;QAEzC,KAAK,WAAW;YACd,OAAO,QAAQ,CAAC,sBAAsB,CAAC,CAAA;QAEzC,KAAK,WAAW;YACd,OAAO,QAAQ,CAAC,sBAAsB,CAAC,CAAA;QAEzC,KAAK,aAAa;YAChB,OAAO,UAAU,CAAC,sBAAsB,CAAC,CAAA;QAE3C;YACE,MAAM,IAAI,KAAK,CAAC,0BAA0B,YAAY,EAAE,CAAC,CAAA;KAC5D;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,YAA0B,EAC1B,iBAA2C;IAE3C,MAAM,sBAAsB,GAAG,MAAM,iBAAiB,CAAC,YAAY,CAAC,CAAA;IACpE,OAAO,cAAc,CAAC,YAAY,EAAE,GAAG,EAAE,CAAC,sBAAsB,CAAC,CAAA;AACnE,CAAC"}
+{"version":3,"file":"modelParams.js","sourceRoot":"","sources":["../src/modelParams.ts"],"names":[],"mappings":"AAAA,qCAAqC;AACrC,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAA;AAE5C,OAAO,EACL,WAAW,EACX,SAAS,EACT,SAAS,EACT,SAAS,EACT,SAAS,EACT,KAAK,EACL,KAAK,EACL,OAAO,GACR,MAAM,oBAAoB,CAAA;AAoB3B,MAAM,eAAe,GACnB,8EAA8E,CAAA;AAEhF,SAAS,QAAQ,CAAC,sBAAkC;IAClD,OAAO;QACL,sBAAsB,EAAE,KAAM;QAC9B,eAAe;QACf,sBAAsB;QACtB,mBAAmB,EAAE,IAAI,GAAG,CAAiB,CAAC,CAAC,SAAS,EAAE,KAAM,CAAC,CAAC,CAAC;KACpE,CAAA;AACH,CAAC;AAED,SAAS,QAAQ,CAAC,sBAAkC;IAClD,OAAO;QACL,sBAAsB,EAAE,KAAM;QAC9B,eAAe;QACf,sBAAsB;QACtB,mBAAmB,EAAE,IAAI,GAAG,CAAiB,CAAC,CAAC,SAAS,EAAE,KAAM,CAAC,CAAC,CAAC;KACpE,CAAA;AACH,CAAC;AAED,SAAS,QAAQ,CAAC,sBAAkC;IAClD,MAAM,mBAAmB,GAAG,IAAI,GAAG,CAAiB;QAClD,CAAC,SAAS,EAAE,KAAM,CAAC;QACnB,CAAC,SAAS,EAAE,KAAM,CAAC;QACnB,CAAC,SAAS,EAAE,KAAM,CAAC;QACnB,CAAC,SAAS,EAAE,KAAM,CAAC;KACpB,CAAC,CAAA;IAEF,OAAO;QACL,eAAe;QACf,sBAAsB;QACtB,mBAAmB;KACpB,CAAA;AACH,CAAC;AAED,SAAS,UAAU,CAAC,sBAAkC;IACpD,MAAM,mBAAmB,GAAG,IAAI,GAAG,CAAiB;QAClD,CAAC,SAAS,EAAE,MAAO,CAAC;QACpB,CAAC,SAAS,EAAE,MAAO,CAAC;QACpB,CAAC,SAAS,EAAE,MAAO,CAAC;QACpB,CAAC,SAAS,EAAE,MAAO,CAAC;QACpB,CAAC,OAAO,EAAE,MAAO,CAAC;QAClB,CAAC,KAAK,EAAE,MAAO,CAAC;QAChB,CAAC,KAAK,EAAE,MAAO,CAAC;QAChB,CAAC,WAAW,EAAE,MAAO,CAAC;KACvB,CAAC,CAAA;IAEF,OAAO;QACL,eAAe,EACb,uHAAuH;QACzH,sBAAsB;QACtB,mBAAmB;KACpB,CAAA;AACH,CAAC;AAOD,MAAM,UAAU,iBAAiB,CAC/B,YAA0B,EAC1B,iBAAsC;IAEtC,MAAM,sBAAsB,GAAG,iBAAiB,CAAC,YAAY,CAAC,CAAA;IAC9D,QAAQ,YAAY,CAAC,WAAW,EAAE,EAAE;QAClC,KAAK,WAAW;YACd,OAAO,QAAQ,CAAC,sBAAsB,CAAC,CAAA;QAEzC,KAAK,WAAW;YACd,OAAO,QAAQ,CAAC,sBAAsB,CAAC,CAAA;QAEzC,KAAK,WAAW;YACd,OAAO,QAAQ,CAAC,sBAAsB,CAAC,CAAA;QAEzC,KAAK,aAAa;YAChB,OAAO,UAAU,CAAC,sBAAsB,CAAC,CAAA;QAE3C;YACE,MAAM,IAAI,KAAK,CAAC,0BAA0B,YAAY,EAAE,CAAC,CAAA;KAC5D;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,YAA0B,EAC1B,iBAA2C;IAE3C,MAAM,sBAAsB,GAAG,MAAM,iBAAiB,CAAC,YAAY,CAAC,CAAA;IACpE,OAAO,iBAAiB,CAAC,YAAY,EAAE,GAAG,EAAE,CAAC,sBAAsB,CAAC,CAAA;AACtE,CAAC"}

package/esm/specialTokens.d.ts

@@ -2,4 +2,7 @@ export declare const EndOfText = "<|endoftext|>";
 export declare const FimPrefix = "<|fim_prefix|>";
 export declare const FimMiddle = "<|fim_middle|>";
 export declare const FimSuffix = "<|fim_suffix|>";
+export declare const ImStart = "<|im_start|>";
+export declare const ImEnd = "<|im_end|>";
+export declare const ImSep = "<|im_sep|>";
 export declare const EndOfPrompt = "<|endofprompt|>";

package/esm/specialTokens.js

@@ -2,5 +2,8 @@ export const EndOfText = '<|endoftext|>';
 export const FimPrefix = '<|fim_prefix|>';
 export const FimMiddle = '<|fim_middle|>';
 export const FimSuffix = '<|fim_suffix|>';
+export const ImStart = '<|im_start|>'; // 100264
+export const ImEnd = '<|im_end|>'; // 100265
+export const ImSep = '<|im_sep|>'; // 100266
 export const EndOfPrompt = '<|endofprompt|>';
 //# sourceMappingURL=specialTokens.js.map

package/esm/specialTokens.js.map

@@ -1 +1 @@
-{"version":3,"file":"specialTokens.js","sourceRoot":"","sources":["../src/specialTokens.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,SAAS,GAAG,eAAe,CAAA;AACxC,MAAM,CAAC,MAAM,SAAS,GAAG,gBAAgB,CAAA;AACzC,MAAM,CAAC,MAAM,SAAS,GAAG,gBAAgB,CAAA;AACzC,MAAM,CAAC,MAAM,SAAS,GAAG,gBAAgB,CAAA;AACzC,MAAM,CAAC,MAAM,WAAW,GAAG,iBAAiB,CAAA"}
+{"version":3,"file":"specialTokens.js","sourceRoot":"","sources":["../src/specialTokens.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,SAAS,GAAG,eAAe,CAAA;AACxC,MAAM,CAAC,MAAM,SAAS,GAAG,gBAAgB,CAAA;AACzC,MAAM,CAAC,MAAM,SAAS,GAAG,gBAAgB,CAAA;AACzC,MAAM,CAAC,MAAM,SAAS,GAAG,gBAAgB,CAAA;AACzC,MAAM,CAAC,MAAM,OAAO,GAAG,cAAc,CAAA,CAAC,SAAS;AAC/C,MAAM,CAAC,MAAM,KAAK,GAAG,YAAY,CAAA,CAAC,SAAS;AAC3C,MAAM,CAAC,MAAM,KAAK,GAAG,YAAY,CAAA,CAAC,SAAS;AAC3C,MAAM,CAAC,MAAM,WAAW,GAAG,iBAAiB,CAAA"}

package/esm/util.d.ts

@@ -0,0 +1,2 @@
+export declare function getMaxValueFromMap(map: Map<unknown, number>): number;
+export declare function getSpecialTokenRegex(tokens: Set<string>): RegExp;

package/esm/util.js

@@ -0,0 +1,14 @@
+import { escapeRegExp } from './escapeRegExp.js';
+export function getMaxValueFromMap(map) {
+    let max = 0;
+    map.forEach((val) => {
+        max = Math.max(max, val);
+    });
+    return max;
+}
+export function getSpecialTokenRegex(tokens) {
+    const escapedTokens = [...tokens].map(escapeRegExp);
+    const inner = escapedTokens.join('|');
+    return new RegExp(`(${inner})`);
+}
+//# sourceMappingURL=util.js.map

package/esm/util.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"util.js","sourceRoot":"","sources":["../src/util.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAA;AAEhD,MAAM,UAAU,kBAAkB,CAAC,GAAyB;IAC1D,IAAI,GAAG,GAAG,CAAC,CAAA;IACX,GAAG,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE;QAClB,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,CAAA;IAC1B,CAAC,CAAC,CAAA;IACF,OAAO,GAAG,CAAA;AACZ,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,MAAmB;IACtD,MAAM,aAAa,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,YAAY,CAAC,CAAA;IACnD,MAAM,KAAK,GAAG,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;IACrC,OAAO,IAAI,MAAM,CAAC,IAAI,KAAK,GAAG,CAAC,CAAA;AACjC,CAAC"}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "gpt-tokenizer",
-  "version": "2.0.0",
-  "description": "BPE Encoder Decoder for GPT-2 / GPT-3",
+  "version": "2.1.1",
+  "description": "A pure JavaScript implementation of a BPE tokenizer (Encoder/Decoder) for GPT-2 / GPT-3 / GPT-4 and other OpenAI models",
   "keywords": [
     "BPE",
     "encoder",
@@ -72,14 +72,16 @@
     "dist"
   ],
   "scripts": {
+    "codegen:models": "yarn tsx src/codegen/generateByModel.ts",
+    "codegen:encodings": "yarn tsx src/codegen/generateJsEncodings.ts",
     "build": "yarn build:cjs && yarn build:esm && yarn build:umd",
     "build:cjs": "yarn rrun tsc --outDir cjs --module commonjs --target es2022 --project tsconfig-cjs.json",
     "build:esm": "yarn rrun tsc --outDir esm --module esnext --target es2022 && echo '{\"name\": \"gpt-tokenizer\", \"type\": \"module\"}' > ./esm/package.json",
     "build:umd": "yarn build:umd:cl100k_base && yarn build:umd:p50k_base && yarn build:umd:p50k_edit && yarn build:umd:r50k_base",
-    "build:umd:cl100k_base": "beemo webpack --entry='./src/main.ts' --env 'outDir=dist' --env 'moduleTarget=umd' --env 'engineTarget=web' --env 'codeTarget=es2022' --env 'name=GPTTokenizer' --env 'export=default' --env 'filename=cl100k_base.js'",
-    "build:umd:p50k_base": "beemo webpack --entry='./src/encoding/p50k_base.ts' --env 'outDir=dist' --env 'moduleTarget=umd' --env 'engineTarget=web' --env 'codeTarget=es2022' --env 'name=GPTTokenizer' --env 'export=default' --env 'filename=p50k_base.js'",
-    "build:umd:p50k_edit": "beemo webpack --entry='./src/encoding/p50k_edit.ts' --env 'outDir=dist' --env 'moduleTarget=umd' --env 'engineTarget=web' --env 'codeTarget=es2022' --env 'name=GPTTokenizer' --env 'export=default' --env 'filename=p50k_edit.js'",
-    "build:umd:r50k_base": "beemo webpack --entry='./src/encoding/r50k_base.ts' --env 'outDir=dist' --env 'moduleTarget=umd' --env 'engineTarget=web' --env 'codeTarget=es2022' --env 'name=GPTTokenizer' --env 'export=default' --env 'filename=r50k_base.js'",
+    "build:umd:cl100k_base": "beemo webpack --entry='./src/main.ts' --env 'outDir=dist' --env 'moduleTarget=umd' --env 'engineTarget=web' --env 'codeTarget=es2022' --env 'name=GPTTokenizer_cl100k_base' --env 'filename=cl100k_base.js'",
+    "build:umd:p50k_base": "beemo webpack --entry='./src/encoding/p50k_base.ts' --env 'outDir=dist' --env 'moduleTarget=umd' --env 'engineTarget=web' --env 'codeTarget=es2022' --env 'name=GPTTokenizer_p50k_base' --env 'filename=p50k_base.js'",
+    "build:umd:p50k_edit": "beemo webpack --entry='./src/encoding/p50k_edit.ts' --env 'outDir=dist' --env 'moduleTarget=umd' --env 'engineTarget=web' --env 'codeTarget=es2022' --env 'name=GPTTokenizer_p50k_edit' --env 'filename=p50k_edit.js'",
+    "build:umd:r50k_base": "beemo webpack --entry='./src/encoding/r50k_base.ts' --env 'outDir=dist' --env 'moduleTarget=umd' --env 'engineTarget=web' --env 'codeTarget=es2022' --env 'name=GPTTokenizer_r50k_base' --env 'filename=r50k_base.js'",
     "clean": "git clean -dfX --exclude=node_modules src && beemo typescript:sync-project-refs",
     "format": "yarn rrun prettier --write \"./{src,tests,.config}/**/!(*.d).{.js,jsx,ts,tsx,json,md}\"",
     "postinstallDev": "yarn prepare",

package/src/GptEncoding.test.ts

@@ -1,7 +1,12 @@
 import fs from 'fs'
 import path from 'path'
 import { GptEncoding } from './GptEncoding.js'
-import { type EncodingName, encodingNames } from './mapping.js'
+import {
+  type ChatModelName,
+  type EncodingName,
+  chatModelParams,
+  encodingNames,
+} from './mapping.js'
 import { resolveEncoding } from './resolveEncoding.js'
 
 const sharedResults = {
@@ -155,6 +160,78 @@ describe.each(encodingNames)('%s', (encodingName: EncodingName) => {
   })
 })
 
+const chatModelNames = Object.keys(chatModelParams) as readonly ChatModelName[]
+
+const exampleMessages = [
+  {
+    role: 'system',
+    content:
+      'You are a helpful, pattern-following assistant that translates corporate jargon into plain English.',
+  },
+  {
+    role: 'system',
+    name: 'example_user',
+    content: 'New synergies will help drive top-line growth.',
+  },
+  {
+    role: 'system',
+    name: 'example_assistant',
+    content: 'Things working well together will increase revenue.',
+  },
+  {
+    role: 'system',
+    name: 'example_user',
+    content:
+      "Let's circle back when we have more bandwidth to touch base on opportunities for increased leverage.",
+  },
+  {
+    role: 'system',
+    name: 'example_assistant',
+    content: "Let's talk later when we're less busy about how to do better.",
+  },
+  {
+    role: 'user',
+    content:
+      "This late pivot means we don't have time to boil the ocean for the client deliverable.",
+  },
+] as const
+
+describe.each(chatModelNames)('%s', (modelName) => {
+  const encoding = GptEncoding.getEncodingApiForModel(
+    modelName,
+    resolveEncoding,
+  )
+  const expectedEncodedLength = modelName.startsWith('gpt-3.5-turbo')
+    ? 127
+    : 121
+
+  describe('chat functionality', () => {
+    it('encodes a chat correctly', () => {
+      const encoded = encoding.encodeChat(exampleMessages)
+      expect(encoded).toMatchSnapshot()
+      expect(encoded).toHaveLength(expectedEncodedLength)
+
+      const decoded = encoding.decode(encoded)
+      expect(decoded).toMatchSnapshot()
+    })
+
+    it('isWithinTokenLimit: false', () => {
+      const isWithinTokenLimit = encoding.isWithinTokenLimit(
+        exampleMessages,
+        50,
+      )
+      expect(isWithinTokenLimit).toBe(false)
+    })
+    it('isWithinTokenLimit: true (number)', () => {
+      const isWithinTokenLimit = encoding.isWithinTokenLimit(
+        exampleMessages,
+        150,
+      )
+      expect(isWithinTokenLimit).toBe(expectedEncodedLength)
+    })
+  })
+})
+
 function loadTestPlans() {
   const testPlanPath = path.join(__dirname, '../data/TestPlans.txt')
   const testPlanData = fs.readFileSync(testPlanPath, 'utf8')

package/src/GptEncoding.ts

@@ -1,16 +1,16 @@
 /* eslint-disable no-param-reassign */
 import { BytePairEncodingCore } from './BytePairEncodingCore.js'
-import { escapeRegExp } from './escapeRegExp.js'
 import {
   type EncodingName,
   type ModelName,
+  chatModelParams,
   modelToEncodingMap,
 } from './mapping.js'
 import {
+  type EncodingParams,
   type GetMergeableRanksAsyncFn,
   type GetMergeableRanksFn,
-  type ModelParams,
-  getModelParams,
+  getEncodingParams,
   getModelParamsAsync,
 } from './modelParams.js'
 import {
@@ -19,10 +19,29 @@ import {
   FimMiddle,
   FimPrefix,
   FimSuffix,
+  ImEnd,
+  ImSep,
+  ImStart,
 } from './specialTokens.js'
 import { endsWithIncompleteUtfPairSurrogate } from './utfUtil.js'
+import { getMaxValueFromMap, getSpecialTokenRegex } from './util.js'
 
-export const DISALLOW_ALL_SPECIAL_TOKENS = 'all'
+export const ALL_SPECIAL_TOKENS = 'all'
+
+export interface EncodeOptions {
+  allowedSpecial?: Set<string>
+  disallowedSpecial?: Set<string>
+}
+
+export interface ChatMessage {
+  role?: 'system' | 'user' | 'assistant'
+  name?: string
+  content: string
+}
+
+export interface EncodeChatOptions {
+  primeWithAssistantResponse?: string
+}
 
 export class GptEncoding {
   static EndOfPrompt = EndOfPrompt
@@ -32,6 +51,7 @@ export class GptEncoding {
   static FimSuffix = FimSuffix
 
   decoder = new TextDecoder('utf8')
+  modelName?: ModelName
   private bytePairEncodingCoreProcessor: BytePairEncodingCore
   private specialTokenMapping: Map<string, number>
 
@@ -40,10 +60,11 @@ export class GptEncoding {
     mergeableBytePairRanks,
     specialTokenMapping,
     expectedVocabularySize,
-  }: ModelParams) {
+    modelName,
+  }: EncodingParams) {
     const maxTokenValue = Math.max(
-      GptEncoding.getMaxValueFromMap(mergeableBytePairRanks),
-      GptEncoding.getMaxValueFromMap(specialTokenMapping),
+      getMaxValueFromMap(mergeableBytePairRanks),
+      getMaxValueFromMap(specialTokenMapping),
     )
     this.specialTokenMapping = specialTokenMapping
 
@@ -76,13 +97,14 @@ export class GptEncoding {
     this.decodeGenerator = this.decodeGenerator.bind(this)
     this.decodeAsyncGenerator = this.decodeAsyncGenerator.bind(this)
     this.isWithinTokenLimit = this.isWithinTokenLimit.bind(this)
+    this.modelName = modelName
   }
 
   static getEncodingApi(
     encodingName: EncodingName,
     getMergeableRanks: GetMergeableRanksFn,
   ): GptEncoding {
-    const modelParams = getModelParams(encodingName, getMergeableRanks)
+    const modelParams = getEncodingParams(encodingName, getMergeableRanks)
     return new GptEncoding(modelParams)
   }
 
@@ -91,7 +113,8 @@ export class GptEncoding {
     getMergeableRanks: GetMergeableRanksFn,
   ): GptEncoding {
     const encodingName = modelToEncodingMap[modelName]
-    return GptEncoding.getEncodingApi(encodingName, getMergeableRanks)
+    const modelParams = getEncodingParams(encodingName, getMergeableRanks)
+    return new GptEncoding({ ...modelParams, modelName })
   }
 
   static async getEncodingApiAsync(
@@ -110,43 +133,34 @@ export class GptEncoding {
     getMergeableRanks: GetMergeableRanksAsyncFn,
   ): Promise<GptEncoding> {
     const encodingName = modelToEncodingMap[modelName]
-    return GptEncoding.getEncodingApiAsync(encodingName, getMergeableRanks)
-  }
-
-  private static getMaxValueFromMap(map: Map<unknown, number>): number {
-    let max = 0
-    map.forEach((val) => {
-      max = Math.max(max, val)
-    })
-    return max
-  }
-
-  private static specialTokenRegex(tokens: Set<string>): RegExp {
-    const escapedTokens = [...tokens].map(escapeRegExp)
-    const inner = escapedTokens.join('|')
-    return new RegExp(`(${inner})`)
+    const modelParams = await getModelParamsAsync(
+      encodingName,
+      getMergeableRanks,
+    )
+    return new GptEncoding({ ...modelParams, modelName })
   }
 
   encodeGenerator(
     lineToEncode: string,
-    allowedSpecial = new Set<string>(),
-    disallowedSpecial: Set<string> = new Set<string>([
-      DISALLOW_ALL_SPECIAL_TOKENS,
-    ]),
-  ): Generator<number[], number> {
+    {
+      allowedSpecial = new Set<string>(),
+      disallowedSpecial = new Set<string>([ALL_SPECIAL_TOKENS]),
+    }: EncodeOptions = {},
+  ): Generator<number[], number, undefined> {
     const specialTokensSet = new Set<string>(this.specialTokenMapping.keys())
 
-    if (disallowedSpecial.has(DISALLOW_ALL_SPECIAL_TOKENS)) {
+    if (disallowedSpecial.has(ALL_SPECIAL_TOKENS)) {
       disallowedSpecial = new Set<string>(specialTokensSet)
+      allowedSpecial.forEach((val) => disallowedSpecial.delete(val))
       disallowedSpecial.forEach((val) => allowedSpecial.delete(val))
     }
 
-    if (allowedSpecial.has(DISALLOW_ALL_SPECIAL_TOKENS)) {
+    if (allowedSpecial.has(ALL_SPECIAL_TOKENS)) {
       allowedSpecial = specialTokensSet
     }
 
     if (disallowedSpecial.size > 0) {
-      const regexPattern = GptEncoding.specialTokenRegex(disallowedSpecial)
+      const regexPattern = getSpecialTokenRegex(disallowedSpecial)
       const match = lineToEncode.match(regexPattern)
       if (match !== null) {
         throw new Error(`Disallowed special token found: ${match[0]}`)
@@ -159,23 +173,90 @@ export class GptEncoding {
     )
   }
 
-  encode(
-    lineToEncode: string,
-    allowedSpecial = new Set<string>(),
-    disallowedSpecial: Set<string> = new Set<string>([
-      DISALLOW_ALL_SPECIAL_TOKENS,
-    ]),
-  ): number[] {
-    return [
-      ...this.encodeGenerator(lineToEncode, allowedSpecial, disallowedSpecial),
-    ].flat()
+  encode(lineToEncode: string, encodeOptions: EncodeOptions = {}): number[] {
+    return [...this.encodeGenerator(lineToEncode, encodeOptions)].flat()
+  }
+
+  /**
+   * Progressively tokenizes an OpenAI chat.
+   * Warning: gpt-3.5-turbo and gpt-4 chat format may change over time.
+   * Returns tokens assuming the 'gpt-3.5-turbo-0301' / 'gpt-4-0314' format.
+   * Based on OpenAI's guidelines: https://github.com/openai/openai-python/blob/main/chatml.md
+   * Also mentioned in section 6 of this document: https://github.com/openai/openai-cookbook/blob/main/examples/How_to_count_tokens_with_tiktoken.ipynb
+   */
+  *encodeChatGenerator(
+    chat: Iterable<ChatMessage>,
+    model = this.modelName,
+  ): Generator<number[], void, undefined> {
+    if (!model) {
+      throw new Error(
+        'Model name must be provided either during initialization or passed in to the method.',
+      )
+    }
+    const params = chatModelParams[model]
+    const chatStartToken = this.specialTokenMapping.get(ImStart)
+    const chatEndToken = this.specialTokenMapping.get(ImEnd)
+
+    if (!params || chatStartToken === undefined || chatEndToken === undefined) {
+      throw new Error(`Model '${model}' does not support chat.`)
+    }
+    const allowedSpecial = new Set([ImSep])
+    const { messageSeparator, roleSeparator } = params
+    const encodedMessageSeparator =
+      messageSeparator.length > 0 ? this.encode(messageSeparator) : []
+    const encodedRoleSeparator =
+      roleSeparator.length > 0
+        ? this.encode(roleSeparator, { allowedSpecial })
+        : []
+    const nameCache = new Map<string, number[]>()
+
+    for (const { role = 'system', name = role, content } of chat) {
+      if (content === undefined) {
+        throw new Error('Content must be defined for all messages.')
+      }
+
+      yield [chatStartToken]
+      const encodedName = nameCache.get(name) ?? this.encode(name)
+      nameCache.set(name, encodedName)
+      yield encodedName
+      if (encodedRoleSeparator.length > 0) {
+        yield encodedRoleSeparator
+      }
+      yield* this.encodeGenerator(content)
+      yield [chatEndToken]
+      yield encodedMessageSeparator
+    }
+
+    // every reply is primed with <|start|>assistant<|message|>
+    yield [chatStartToken]
+    yield* this.encodeGenerator('assistant')
+    if (encodedRoleSeparator.length > 0) {
+      yield encodedRoleSeparator
+    }
+  }
+
+  /**
+   * Encodes a chat into a single array of tokens.
+   * Warning: gpt-3.5-turbo and gpt-4 chat format may change over time.
+   * Returns tokens assuming the 'gpt-3.5-turbo-0301' / 'gpt-4-0314' format.
+   * Based on OpenAI's guidelines: https://github.com/openai/openai-python/blob/main/chatml.md
+   * Also mentioned in section 6 of this document: https://github.com/openai/openai-cookbook/blob/main/examples/How_to_count_tokens_with_tiktoken.ipynb
+   */
+  encodeChat(chat: readonly ChatMessage[], model = this.modelName): number[] {
+    return [...this.encodeChatGenerator(chat, model)].flat()
   }
 
   /**
    * @returns {false | number} false if token limit is exceeded, otherwise the number of tokens
    */
-  isWithinTokenLimit(text: string, tokenLimit: number): false | number {
-    const tokenGenerator = this.encodeGenerator(text)
+  isWithinTokenLimit(
+    input: string | Iterable<ChatMessage>,
+    tokenLimit: number,
+  ): false | number {
+    const tokenGenerator =
+      typeof input === 'string'
+        ? this.encodeGenerator(input)
+        : this.encodeChatGenerator(input)
     let count = 0
     for (const tokens of tokenGenerator) {
       count += tokens.length