gpt-tokenizer 2.0.0-beta.2 → 2.1.0

package/esm/mapping.d.ts

@@ -5,7 +5,11 @@ export declare const r50k_base = "r50k_base";
 export declare const encodingNames: readonly ["cl100k_base", "p50k_base", "r50k_base", "p50k_edit"];
 export declare const modelToEncodingMap: {
     readonly 'gpt-4': "cl100k_base";
+    readonly 'gpt-4-32k': "cl100k_base";
+    readonly 'gpt-4-0314': "cl100k_base";
+    readonly 'gpt-4-32k-0314': "cl100k_base";
     readonly 'gpt-3.5-turbo': "cl100k_base";
+    readonly 'gpt-3.5-turbo-0301': "cl100k_base";
     readonly 'text-davinci-003': "p50k_base";
     readonly 'text-davinci-002': "p50k_base";
     readonly 'text-davinci-001': "r50k_base";
@@ -36,5 +40,38 @@ export declare const modelToEncodingMap: {
     readonly 'code-search-babbage-code-001': "r50k_base";
     readonly 'code-search-ada-code-001': "r50k_base";
 };
+export interface ChatParameters {
+    messageSeparator: string;
+    roleSeparator: string;
+}
+declare const internalChatModelParams: {
+    'gpt-3.5-turbo': {
+        messageSeparator: string;
+        roleSeparator: string;
+    };
+    'gpt-3.5-turbo-0301': {
+        messageSeparator: string;
+        roleSeparator: string;
+    };
+    'gpt-4': {
+        messageSeparator: string;
+        roleSeparator: string;
+    };
+    'gpt-4-0314': {
+        messageSeparator: string;
+        roleSeparator: string;
+    };
+    'gpt-4-32k': {
+        messageSeparator: string;
+        roleSeparator: string;
+    };
+    'gpt-4-32k-0314': {
+        messageSeparator: string;
+        roleSeparator: string;
+    };
+};
+export declare const chatModelParams: Partial<Record<ModelName, ChatParameters>>;
 export type ModelName = keyof typeof modelToEncodingMap;
+export type ChatModelName = keyof typeof internalChatModelParams;
 export type EncodingName = (typeof modelToEncodingMap)[ModelName];
+export {};

package/esm/mapping.js

@@ -1,4 +1,5 @@
 /* eslint-disable camelcase */
+import { ImSep } from './specialTokens.js';
 export const cl100k_base = 'cl100k_base';
 export const p50k_base = 'p50k_base';
 export const p50k_edit = 'p50k_edit';
@@ -12,7 +13,11 @@ export const encodingNames = [
 export const modelToEncodingMap = {
     // chat
     'gpt-4': cl100k_base,
+    'gpt-4-32k': cl100k_base,
+    'gpt-4-0314': cl100k_base,
+    'gpt-4-32k-0314': cl100k_base,
     'gpt-3.5-turbo': cl100k_base,
+    'gpt-3.5-turbo-0301': cl100k_base,
     // text
     'text-davinci-003': p50k_base,
     'text-davinci-002': p50k_base,
@@ -48,4 +53,31 @@ export const modelToEncodingMap = {
     'code-search-babbage-code-001': r50k_base,
     'code-search-ada-code-001': r50k_base,
 };
+const internalChatModelParams = {
+    'gpt-3.5-turbo': {
+        messageSeparator: '\n',
+        roleSeparator: '\n',
+    },
+    'gpt-3.5-turbo-0301': {
+        messageSeparator: '\n',
+        roleSeparator: '\n',
+    },
+    'gpt-4': {
+        messageSeparator: '',
+        roleSeparator: ImSep,
+    },
+    'gpt-4-0314': {
+        messageSeparator: '',
+        roleSeparator: ImSep,
+    },
+    'gpt-4-32k': {
+        messageSeparator: '',
+        roleSeparator: ImSep,
+    },
+    'gpt-4-32k-0314': {
+        messageSeparator: '',
+        roleSeparator: ImSep,
+    },
+};
+export const chatModelParams = internalChatModelParams;
 //# sourceMappingURL=mapping.js.map

package/esm/mapping.js.map

@@ -1 +1 @@
-{"version":3,"file":"mapping.js","sourceRoot":"","sources":["../src/mapping.ts"],"names":[],"mappings":"AAAA,8BAA8B;AAC9B,MAAM,CAAC,MAAM,WAAW,GAAG,aAAa,CAAA;AACxC,MAAM,CAAC,MAAM,SAAS,GAAG,WAAW,CAAA;AACpC,MAAM,CAAC,MAAM,SAAS,GAAG,WAAW,CAAA;AACpC,MAAM,CAAC,MAAM,SAAS,GAAG,WAAW,CAAA;AAEpC,MAAM,CAAC,MAAM,aAAa,GAAG;IAC3B,WAAW;IACX,SAAS;IACT,SAAS;IACT,SAAS;CACD,CAAA;AAEV,MAAM,CAAC,MAAM,kBAAkB,GAAG;IAChC,OAAO;IACP,OAAO,EAAE,WAAW;IACpB,eAAe,EAAE,WAAW;IAC5B,OAAO;IACP,kBAAkB,EAAE,SAAS;IAC7B,kBAAkB,EAAE,SAAS;IAC7B,kBAAkB,EAAE,SAAS;IAC7B,gBAAgB,EAAE,SAAS;IAC3B,kBAAkB,EAAE,SAAS;IAC7B,cAAc,EAAE,SAAS;IACzB,OAAO,EAAE,SAAS;IAClB,KAAK,EAAE,SAAS;IAChB,OAAO,EAAE,SAAS;IAClB,GAAG,EAAE,SAAS;IACd,OAAO;IACP,kBAAkB,EAAE,SAAS;IAC7B,kBAAkB,EAAE,SAAS;IAC7B,kBAAkB,EAAE,SAAS;IAC7B,kBAAkB,EAAE,SAAS;IAC7B,eAAe,EAAE,SAAS;IAC1B,eAAe,EAAE,SAAS;IAC1B,OAAO;IACP,uBAAuB,EAAE,SAAS;IAClC,uBAAuB,EAAE,SAAS;IAClC,aAAa;IACb,wBAAwB,EAAE,WAAW;IACrC,iBAAiB;IACjB,6BAA6B,EAAE,SAAS;IACxC,2BAA2B,EAAE,SAAS;IACtC,6BAA6B,EAAE,SAAS;IACxC,yBAAyB,EAAE,SAAS;IACpC,6BAA6B,EAAE,SAAS;IACxC,2BAA2B,EAAE,SAAS;IACtC,6BAA6B,EAAE,SAAS;IACxC,yBAAyB,EAAE,SAAS;IACpC,8BAA8B,EAAE,SAAS;IACzC,0BAA0B,EAAE,SAAS;CAC7B,CAAA"}
+{"version":3,"file":"mapping.js","sourceRoot":"","sources":["../src/mapping.ts"],"names":[],"mappings":"AAAA,8BAA8B;AAE9B,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAA;AAE1C,MAAM,CAAC,MAAM,WAAW,GAAG,aAAa,CAAA;AACxC,MAAM,CAAC,MAAM,SAAS,GAAG,WAAW,CAAA;AACpC,MAAM,CAAC,MAAM,SAAS,GAAG,WAAW,CAAA;AACpC,MAAM,CAAC,MAAM,SAAS,GAAG,WAAW,CAAA;AAEpC,MAAM,CAAC,MAAM,aAAa,GAAG;IAC3B,WAAW;IACX,SAAS;IACT,SAAS;IACT,SAAS;CACD,CAAA;AAEV,MAAM,CAAC,MAAM,kBAAkB,GAAG;IAChC,OAAO;IACP,OAAO,EAAE,WAAW;IACpB,WAAW,EAAE,WAAW;IACxB,YAAY,EAAE,WAAW;IACzB,gBAAgB,EAAE,WAAW;IAC7B,eAAe,EAAE,WAAW;IAC5B,oBAAoB,EAAE,WAAW;IACjC,OAAO;IACP,kBAAkB,EAAE,SAAS;IAC7B,kBAAkB,EAAE,SAAS;IAC7B,kBAAkB,EAAE,SAAS;IAC7B,gBAAgB,EAAE,SAAS;IAC3B,kBAAkB,EAAE,SAAS;IAC7B,cAAc,EAAE,SAAS;IACzB,OAAO,EAAE,SAAS;IAClB,KAAK,EAAE,SAAS;IAChB,OAAO,EAAE,SAAS;IAClB,GAAG,EAAE,SAAS;IACd,OAAO;IACP,kBAAkB,EAAE,SAAS;IAC7B,kBAAkB,EAAE,SAAS;IAC7B,kBAAkB,EAAE,SAAS;IAC7B,kBAAkB,EAAE,SAAS;IAC7B,eAAe,EAAE,SAAS;IAC1B,eAAe,EAAE,SAAS;IAC1B,OAAO;IACP,uBAAuB,EAAE,SAAS;IAClC,uBAAuB,EAAE,SAAS;IAClC,aAAa;IACb,wBAAwB,EAAE,WAAW;IACrC,iBAAiB;IACjB,6BAA6B,EAAE,SAAS;IACxC,2BAA2B,EAAE,SAAS;IACtC,6BAA6B,EAAE,SAAS;IACxC,yBAAyB,EAAE,SAAS;IACpC,6BAA6B,EAAE,SAAS;IACxC,2BAA2B,EAAE,SAAS;IACtC,6BAA6B,EAAE,SAAS;IACxC,yBAAyB,EAAE,SAAS;IACpC,8BAA8B,EAAE,SAAS;IACzC,0BAA0B,EAAE,SAAS;CAC7B,CAAA;AAOV,MAAM,uBAAuB,GAAG;IAC9B,eAAe,EAAE;QACf,gBAAgB,EAAE,IAAI;QACtB,aAAa,EAAE,IAAI;KACpB;IACD,oBAAoB,EAAE;QACpB,gBAAgB,EAAE,IAAI;QACtB,aAAa,EAAE,IAAI;KACpB;IACD,OAAO,EAAE;QACP,gBAAgB,EAAE,EAAE;QACpB,aAAa,EAAE,KAAK;KACrB;IACD,YAAY,EAAE;QACZ,gBAAgB,EAAE,EAAE;QACpB,aAAa,EAAE,KAAK;KACrB;IACD,WAAW,EAAE;QACX,gBAAgB,EAAE,EAAE;QACpB,aAAa,EAAE,KAAK;KACrB;IACD,gBAAgB,EAAE;QAChB,gBAAgB,EAAE,EAAE;QACpB,aAAa,EAAE,KAAK;KACrB;CACF,CAAA;AAED,MAAM,CAAC,MAAM,eAAe,GAC1B,uBAAuB,CAAA"}

package/esm/model/gpt-3.5-turbo-0301.d.ts

@@ -0,0 +1,6 @@
+import { GptEncoding } from '../GptEncoding.js';
+export * from '../specialTokens.js';
+declare const api: GptEncoding;
+declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void, unknown>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, unknown>, encode: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, { allowedSpecial, disallowedSpecial, }?: import("../GptEncoding.js").EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, tokenLimit: number) => number | false, encodeChat: (chat: readonly import("../GptEncoding.js").ChatMessage[], model?: "gpt-4" | "gpt-4-32k" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo" | "gpt-3.5-turbo-0301" | "text-davinci-003" | "text-davinci-002" | "text-davinci-001" | "text-curie-001" | "text-babbage-001" | "text-ada-001" | "davinci" | "curie" | "babbage" | "ada" | "code-davinci-002" | "code-davinci-001" | "code-cushman-002" | "code-cushman-001" | "davinci-codex" | "cushman-codex" | "text-davinci-edit-001" | "code-davinci-edit-001" | "text-embedding-ada-002" | "text-similarity-davinci-001" | "text-similarity-curie-001" | "text-similarity-babbage-001" | "text-similarity-ada-001" | "text-search-davinci-doc-001" | "text-search-curie-doc-001" | "text-search-babbage-doc-001" | "text-search-ada-doc-001" | "code-search-babbage-code-001" | "code-search-ada-code-001" | undefined) => number[], encodeChatGenerator: (chat: Iterable<import("../GptEncoding.js").ChatMessage>, model?: "gpt-4" | "gpt-4-32k" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo" | "gpt-3.5-turbo-0301" | "text-davinci-003" | "text-davinci-002" | "text-davinci-001" | "text-curie-001" | "text-babbage-001" | "text-ada-001" | "davinci" | "curie" | "babbage" | "ada" | "code-davinci-002" | "code-davinci-001" | "code-cushman-002" | "code-cushman-001" | "davinci-codex" | "cushman-codex" | "text-davinci-edit-001" | "code-davinci-edit-001" | "text-embedding-ada-002" | "text-similarity-davinci-001" | "text-similarity-curie-001" | "text-similarity-babbage-001" | "text-similarity-ada-001" | "text-search-davinci-doc-001" | "text-search-curie-doc-001" | "text-search-babbage-doc-001" | "text-search-ada-doc-001" | "code-search-babbage-code-001" | "code-search-ada-code-001" | undefined) => Generator<number[], void, undefined>;
+export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, };
+export default api;

package/esm/model/gpt-3.5-turbo-0301.js

@@ -0,0 +1,11 @@
+/* eslint-disable import/extensions */
+import { convertTokenBytePairEncodingFromTuples } from '../convertTokenBytePairEncodingFromTuples.js';
+import encoder from '../encodings/cl100k_base.js';
+import { GptEncoding } from '../GptEncoding.js';
+export * from '../specialTokens.js';
+const api = GptEncoding.getEncodingApiForModel('gpt-3.5-turbo-0301', () => convertTokenBytePairEncodingFromTuples(encoder));
+const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, encodeChat, encodeChatGenerator, } = api;
+export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, };
+// eslint-disable-next-line import/no-default-export
+export default api;
+//# sourceMappingURL=gpt-3.5-turbo-0301.js.map

package/esm/model/gpt-3.5-turbo-0301.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"gpt-3.5-turbo-0301.js","sourceRoot":"","sources":["../../src/model/gpt-3.5-turbo-0301.ts"],"names":[],"mappings":"AAAA,sCAAsC;AACtC,OAAO,EAAE,sCAAsC,EAAE,MAAM,8CAA8C,CAAA;AACrG,OAAO,OAAO,MAAM,6BAA6B,CAAA;AACjD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAA;AAE/C,cAAc,qBAAqB,CAAA;AAEnC,MAAM,GAAG,GAAG,WAAW,CAAC,sBAAsB,CAAC,oBAAoB,EAAE,GAAG,EAAE,CACxE,sCAAsC,CAAC,OAAO,CAAC,CAChD,CAAA;AACD,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,EAClB,UAAU,EACV,mBAAmB,GACpB,GAAG,GAAG,CAAA;AACP,OAAO,EACL,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,UAAU,EACV,mBAAmB,EACnB,eAAe,EACf,kBAAkB,GACnB,CAAA;AACD,oDAAoD;AACpD,eAAe,GAAG,CAAA"}

package/esm/model/gpt-3.5-turbo.d.ts

@@ -1 +1,6 @@
-export * from '../encoding/cl100k_base.js';
+import { GptEncoding } from '../GptEncoding.js';
+export * from '../specialTokens.js';
+declare const api: GptEncoding;
+declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void, unknown>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, unknown>, encode: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, { allowedSpecial, disallowedSpecial, }?: import("../GptEncoding.js").EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, tokenLimit: number) => number | false, encodeChat: (chat: readonly import("../GptEncoding.js").ChatMessage[], model?: "gpt-4" | "gpt-4-32k" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo" | "gpt-3.5-turbo-0301" | "text-davinci-003" | "text-davinci-002" | "text-davinci-001" | "text-curie-001" | "text-babbage-001" | "text-ada-001" | "davinci" | "curie" | "babbage" | "ada" | "code-davinci-002" | "code-davinci-001" | "code-cushman-002" | "code-cushman-001" | "davinci-codex" | "cushman-codex" | "text-davinci-edit-001" | "code-davinci-edit-001" | "text-embedding-ada-002" | "text-similarity-davinci-001" | "text-similarity-curie-001" | "text-similarity-babbage-001" | "text-similarity-ada-001" | "text-search-davinci-doc-001" | "text-search-curie-doc-001" | "text-search-babbage-doc-001" | "text-search-ada-doc-001" | "code-search-babbage-code-001" | "code-search-ada-code-001" | undefined) => number[], encodeChatGenerator: (chat: Iterable<import("../GptEncoding.js").ChatMessage>, model?: "gpt-4" | "gpt-4-32k" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo" | "gpt-3.5-turbo-0301" | "text-davinci-003" | "text-davinci-002" | "text-davinci-001" | "text-curie-001" | "text-babbage-001" | "text-ada-001" | "davinci" | "curie" | "babbage" | "ada" | "code-davinci-002" | "code-davinci-001" | "code-cushman-002" | "code-cushman-001" | "davinci-codex" | "cushman-codex" | "text-davinci-edit-001" | "code-davinci-edit-001" | "text-embedding-ada-002" | "text-similarity-davinci-001" | "text-similarity-curie-001" | "text-similarity-babbage-001" | "text-similarity-ada-001" | "text-search-davinci-doc-001" | "text-search-curie-doc-001" | "text-search-babbage-doc-001" | "text-search-ada-doc-001" | "code-search-babbage-code-001" | "code-search-ada-code-001" | undefined) => Generator<number[], void, undefined>;
+export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, };
+export default api;

package/esm/model/gpt-3.5-turbo.js

@@ -1,2 +1,11 @@
-export * from '../encoding/cl100k_base.js';
+/* eslint-disable import/extensions */
+import { convertTokenBytePairEncodingFromTuples } from '../convertTokenBytePairEncodingFromTuples.js';
+import encoder from '../encodings/cl100k_base.js';
+import { GptEncoding } from '../GptEncoding.js';
+export * from '../specialTokens.js';
+const api = GptEncoding.getEncodingApiForModel('gpt-3.5-turbo', () => convertTokenBytePairEncodingFromTuples(encoder));
+const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, encodeChat, encodeChatGenerator, } = api;
+export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, };
+// eslint-disable-next-line import/no-default-export
+export default api;
 //# sourceMappingURL=gpt-3.5-turbo.js.map

package/esm/model/gpt-3.5-turbo.js.map

@@ -1 +1 @@
-{"version":3,"file":"gpt-3.5-turbo.js","sourceRoot":"","sources":["../../src/model/gpt-3.5-turbo.ts"],"names":[],"mappings":"AAAA,cAAc,4BAA4B,CAAA"}
+{"version":3,"file":"gpt-3.5-turbo.js","sourceRoot":"","sources":["../../src/model/gpt-3.5-turbo.ts"],"names":[],"mappings":"AAAA,sCAAsC;AACtC,OAAO,EAAE,sCAAsC,EAAE,MAAM,8CAA8C,CAAA;AACrG,OAAO,OAAO,MAAM,6BAA6B,CAAA;AACjD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAA;AAE/C,cAAc,qBAAqB,CAAA;AAEnC,MAAM,GAAG,GAAG,WAAW,CAAC,sBAAsB,CAAC,eAAe,EAAE,GAAG,EAAE,CACnE,sCAAsC,CAAC,OAAO,CAAC,CAChD,CAAA;AACD,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,EAClB,UAAU,EACV,mBAAmB,GACpB,GAAG,GAAG,CAAA;AACP,OAAO,EACL,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,UAAU,EACV,mBAAmB,EACnB,eAAe,EACf,kBAAkB,GACnB,CAAA;AACD,oDAAoD;AACpD,eAAe,GAAG,CAAA"}

package/esm/model/gpt-4-0314.d.ts

@@ -0,0 +1,6 @@
+import { GptEncoding } from '../GptEncoding.js';
+export * from '../specialTokens.js';
+declare const api: GptEncoding;
+declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void, unknown>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, unknown>, encode: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, { allowedSpecial, disallowedSpecial, }?: import("../GptEncoding.js").EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, tokenLimit: number) => number | false, encodeChat: (chat: readonly import("../GptEncoding.js").ChatMessage[], model?: "gpt-4" | "gpt-4-32k" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo" | "gpt-3.5-turbo-0301" | "text-davinci-003" | "text-davinci-002" | "text-davinci-001" | "text-curie-001" | "text-babbage-001" | "text-ada-001" | "davinci" | "curie" | "babbage" | "ada" | "code-davinci-002" | "code-davinci-001" | "code-cushman-002" | "code-cushman-001" | "davinci-codex" | "cushman-codex" | "text-davinci-edit-001" | "code-davinci-edit-001" | "text-embedding-ada-002" | "text-similarity-davinci-001" | "text-similarity-curie-001" | "text-similarity-babbage-001" | "text-similarity-ada-001" | "text-search-davinci-doc-001" | "text-search-curie-doc-001" | "text-search-babbage-doc-001" | "text-search-ada-doc-001" | "code-search-babbage-code-001" | "code-search-ada-code-001" | undefined) => number[], encodeChatGenerator: (chat: Iterable<import("../GptEncoding.js").ChatMessage>, model?: "gpt-4" | "gpt-4-32k" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo" | "gpt-3.5-turbo-0301" | "text-davinci-003" | "text-davinci-002" | "text-davinci-001" | "text-curie-001" | "text-babbage-001" | "text-ada-001" | "davinci" | "curie" | "babbage" | "ada" | "code-davinci-002" | "code-davinci-001" | "code-cushman-002" | "code-cushman-001" | "davinci-codex" | "cushman-codex" | "text-davinci-edit-001" | "code-davinci-edit-001" | "text-embedding-ada-002" | "text-similarity-davinci-001" | "text-similarity-curie-001" | "text-similarity-babbage-001" | "text-similarity-ada-001" | "text-search-davinci-doc-001" | "text-search-curie-doc-001" | "text-search-babbage-doc-001" | "text-search-ada-doc-001" | "code-search-babbage-code-001" | "code-search-ada-code-001" | undefined) => Generator<number[], void, undefined>;
+export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, };
+export default api;

package/esm/model/gpt-4-0314.js

@@ -0,0 +1,11 @@
+/* eslint-disable import/extensions */
+import { convertTokenBytePairEncodingFromTuples } from '../convertTokenBytePairEncodingFromTuples.js';
+import encoder from '../encodings/cl100k_base.js';
+import { GptEncoding } from '../GptEncoding.js';
+export * from '../specialTokens.js';
+const api = GptEncoding.getEncodingApiForModel('gpt-4-0314', () => convertTokenBytePairEncodingFromTuples(encoder));
+const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, encodeChat, encodeChatGenerator, } = api;
+export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, };
+// eslint-disable-next-line import/no-default-export
+export default api;
+//# sourceMappingURL=gpt-4-0314.js.map

package/esm/model/gpt-4-0314.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"gpt-4-0314.js","sourceRoot":"","sources":["../../src/model/gpt-4-0314.ts"],"names":[],"mappings":"AAAA,sCAAsC;AACtC,OAAO,EAAE,sCAAsC,EAAE,MAAM,8CAA8C,CAAA;AACrG,OAAO,OAAO,MAAM,6BAA6B,CAAA;AACjD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAA;AAE/C,cAAc,qBAAqB,CAAA;AAEnC,MAAM,GAAG,GAAG,WAAW,CAAC,sBAAsB,CAAC,YAAY,EAAE,GAAG,EAAE,CAChE,sCAAsC,CAAC,OAAO,CAAC,CAChD,CAAA;AACD,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,EAClB,UAAU,EACV,mBAAmB,GACpB,GAAG,GAAG,CAAA;AACP,OAAO,EACL,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,UAAU,EACV,mBAAmB,EACnB,eAAe,EACf,kBAAkB,GACnB,CAAA;AACD,oDAAoD;AACpD,eAAe,GAAG,CAAA"}

package/esm/model/gpt-4-32k-0314.d.ts

@@ -0,0 +1,6 @@
+import { GptEncoding } from '../GptEncoding.js';
+export * from '../specialTokens.js';
+declare const api: GptEncoding;
+declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void, unknown>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, unknown>, encode: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, { allowedSpecial, disallowedSpecial, }?: import("../GptEncoding.js").EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, tokenLimit: number) => number | false, encodeChat: (chat: readonly import("../GptEncoding.js").ChatMessage[], model?: "gpt-4" | "gpt-4-32k" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo" | "gpt-3.5-turbo-0301" | "text-davinci-003" | "text-davinci-002" | "text-davinci-001" | "text-curie-001" | "text-babbage-001" | "text-ada-001" | "davinci" | "curie" | "babbage" | "ada" | "code-davinci-002" | "code-davinci-001" | "code-cushman-002" | "code-cushman-001" | "davinci-codex" | "cushman-codex" | "text-davinci-edit-001" | "code-davinci-edit-001" | "text-embedding-ada-002" | "text-similarity-davinci-001" | "text-similarity-curie-001" | "text-similarity-babbage-001" | "text-similarity-ada-001" | "text-search-davinci-doc-001" | "text-search-curie-doc-001" | "text-search-babbage-doc-001" | "text-search-ada-doc-001" | "code-search-babbage-code-001" | "code-search-ada-code-001" | undefined) => number[], encodeChatGenerator: (chat: Iterable<import("../GptEncoding.js").ChatMessage>, model?: "gpt-4" | "gpt-4-32k" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo" | "gpt-3.5-turbo-0301" | "text-davinci-003" | "text-davinci-002" | "text-davinci-001" | "text-curie-001" | "text-babbage-001" | "text-ada-001" | "davinci" | "curie" | "babbage" | "ada" | "code-davinci-002" | "code-davinci-001" | "code-cushman-002" | "code-cushman-001" | "davinci-codex" | "cushman-codex" | "text-davinci-edit-001" | "code-davinci-edit-001" | "text-embedding-ada-002" | "text-similarity-davinci-001" | "text-similarity-curie-001" | "text-similarity-babbage-001" | "text-similarity-ada-001" | "text-search-davinci-doc-001" | "text-search-curie-doc-001" | "text-search-babbage-doc-001" | "text-search-ada-doc-001" | "code-search-babbage-code-001" | "code-search-ada-code-001" | undefined) => Generator<number[], void, undefined>;
+export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, };
+export default api;

package/esm/model/gpt-4-32k-0314.js

@@ -0,0 +1,11 @@
+/* eslint-disable import/extensions */
+import { convertTokenBytePairEncodingFromTuples } from '../convertTokenBytePairEncodingFromTuples.js';
+import encoder from '../encodings/cl100k_base.js';
+import { GptEncoding } from '../GptEncoding.js';
+export * from '../specialTokens.js';
+const api = GptEncoding.getEncodingApiForModel('gpt-4-32k-0314', () => convertTokenBytePairEncodingFromTuples(encoder));
+const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, encodeChat, encodeChatGenerator, } = api;
+export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, };
+// eslint-disable-next-line import/no-default-export
+export default api;
+//# sourceMappingURL=gpt-4-32k-0314.js.map

package/esm/model/gpt-4-32k-0314.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"gpt-4-32k-0314.js","sourceRoot":"","sources":["../../src/model/gpt-4-32k-0314.ts"],"names":[],"mappings":"AAAA,sCAAsC;AACtC,OAAO,EAAE,sCAAsC,EAAE,MAAM,8CAA8C,CAAA;AACrG,OAAO,OAAO,MAAM,6BAA6B,CAAA;AACjD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAA;AAE/C,cAAc,qBAAqB,CAAA;AAEnC,MAAM,GAAG,GAAG,WAAW,CAAC,sBAAsB,CAAC,gBAAgB,EAAE,GAAG,EAAE,CACpE,sCAAsC,CAAC,OAAO,CAAC,CAChD,CAAA;AACD,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,EAClB,UAAU,EACV,mBAAmB,GACpB,GAAG,GAAG,CAAA;AACP,OAAO,EACL,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,UAAU,EACV,mBAAmB,EACnB,eAAe,EACf,kBAAkB,GACnB,CAAA;AACD,oDAAoD;AACpD,eAAe,GAAG,CAAA"}

package/esm/model/gpt-4-32k.d.ts

@@ -0,0 +1,6 @@
+import { GptEncoding } from '../GptEncoding.js';
+export * from '../specialTokens.js';
+declare const api: GptEncoding;
+declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void, unknown>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, unknown>, encode: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, { allowedSpecial, disallowedSpecial, }?: import("../GptEncoding.js").EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, tokenLimit: number) => number | false, encodeChat: (chat: readonly import("../GptEncoding.js").ChatMessage[], model?: "gpt-4" | "gpt-4-32k" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo" | "gpt-3.5-turbo-0301" | "text-davinci-003" | "text-davinci-002" | "text-davinci-001" | "text-curie-001" | "text-babbage-001" | "text-ada-001" | "davinci" | "curie" | "babbage" | "ada" | "code-davinci-002" | "code-davinci-001" | "code-cushman-002" | "code-cushman-001" | "davinci-codex" | "cushman-codex" | "text-davinci-edit-001" | "code-davinci-edit-001" | "text-embedding-ada-002" | "text-similarity-davinci-001" | "text-similarity-curie-001" | "text-similarity-babbage-001" | "text-similarity-ada-001" | "text-search-davinci-doc-001" | "text-search-curie-doc-001" | "text-search-babbage-doc-001" | "text-search-ada-doc-001" | "code-search-babbage-code-001" | "code-search-ada-code-001" | undefined) => number[], encodeChatGenerator: (chat: Iterable<import("../GptEncoding.js").ChatMessage>, model?: "gpt-4" | "gpt-4-32k" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo" | "gpt-3.5-turbo-0301" | "text-davinci-003" | "text-davinci-002" | "text-davinci-001" | "text-curie-001" | "text-babbage-001" | "text-ada-001" | "davinci" | "curie" | "babbage" | "ada" | "code-davinci-002" | "code-davinci-001" | "code-cushman-002" | "code-cushman-001" | "davinci-codex" | "cushman-codex" | "text-davinci-edit-001" | "code-davinci-edit-001" | "text-embedding-ada-002" | "text-similarity-davinci-001" | "text-similarity-curie-001" | "text-similarity-babbage-001" | "text-similarity-ada-001" | "text-search-davinci-doc-001" | "text-search-curie-doc-001" | "text-search-babbage-doc-001" | "text-search-ada-doc-001" | "code-search-babbage-code-001" | "code-search-ada-code-001" | undefined) => Generator<number[], void, undefined>;
+export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, };
+export default api;

package/esm/model/gpt-4-32k.js

@@ -0,0 +1,11 @@
+/* eslint-disable import/extensions */
+import { convertTokenBytePairEncodingFromTuples } from '../convertTokenBytePairEncodingFromTuples.js';
+import encoder from '../encodings/cl100k_base.js';
+import { GptEncoding } from '../GptEncoding.js';
+export * from '../specialTokens.js';
+const api = GptEncoding.getEncodingApiForModel('gpt-4-32k', () => convertTokenBytePairEncodingFromTuples(encoder));
+const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, encodeChat, encodeChatGenerator, } = api;
+export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, };
+// eslint-disable-next-line import/no-default-export
+export default api;
+//# sourceMappingURL=gpt-4-32k.js.map

package/esm/model/gpt-4-32k.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"gpt-4-32k.js","sourceRoot":"","sources":["../../src/model/gpt-4-32k.ts"],"names":[],"mappings":"AAAA,sCAAsC;AACtC,OAAO,EAAE,sCAAsC,EAAE,MAAM,8CAA8C,CAAA;AACrG,OAAO,OAAO,MAAM,6BAA6B,CAAA;AACjD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAA;AAE/C,cAAc,qBAAqB,CAAA;AAEnC,MAAM,GAAG,GAAG,WAAW,CAAC,sBAAsB,CAAC,WAAW,EAAE,GAAG,EAAE,CAC/D,sCAAsC,CAAC,OAAO,CAAC,CAChD,CAAA;AACD,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,EAClB,UAAU,EACV,mBAAmB,GACpB,GAAG,GAAG,CAAA;AACP,OAAO,EACL,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,UAAU,EACV,mBAAmB,EACnB,eAAe,EACf,kBAAkB,GACnB,CAAA;AACD,oDAAoD;AACpD,eAAe,GAAG,CAAA"}

package/esm/model/gpt-4.d.ts

@@ -1 +1,6 @@
-export * from '../encoding/cl100k_base.js';
+import { GptEncoding } from '../GptEncoding.js';
+export * from '../specialTokens.js';
+declare const api: GptEncoding;
+declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void, unknown>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, unknown>, encode: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, { allowedSpecial, disallowedSpecial, }?: import("../GptEncoding.js").EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, tokenLimit: number) => number | false, encodeChat: (chat: readonly import("../GptEncoding.js").ChatMessage[], model?: "gpt-4" | "gpt-4-32k" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo" | "gpt-3.5-turbo-0301" | "text-davinci-003" | "text-davinci-002" | "text-davinci-001" | "text-curie-001" | "text-babbage-001" | "text-ada-001" | "davinci" | "curie" | "babbage" | "ada" | "code-davinci-002" | "code-davinci-001" | "code-cushman-002" | "code-cushman-001" | "davinci-codex" | "cushman-codex" | "text-davinci-edit-001" | "code-davinci-edit-001" | "text-embedding-ada-002" | "text-similarity-davinci-001" | "text-similarity-curie-001" | "text-similarity-babbage-001" | "text-similarity-ada-001" | "text-search-davinci-doc-001" | "text-search-curie-doc-001" | "text-search-babbage-doc-001" | "text-search-ada-doc-001" | "code-search-babbage-code-001" | "code-search-ada-code-001" | undefined) => number[], encodeChatGenerator: (chat: Iterable<import("../GptEncoding.js").ChatMessage>, model?: "gpt-4" | "gpt-4-32k" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo" | "gpt-3.5-turbo-0301" | "text-davinci-003" | "text-davinci-002" | "text-davinci-001" | "text-curie-001" | "text-babbage-001" | "text-ada-001" | "davinci" | "curie" | "babbage" | "ada" | "code-davinci-002" | "code-davinci-001" | "code-cushman-002" | "code-cushman-001" | "davinci-codex" | "cushman-codex" | "text-davinci-edit-001" | "code-davinci-edit-001" | "text-embedding-ada-002" | "text-similarity-davinci-001" | "text-similarity-curie-001" | "text-similarity-babbage-001" | "text-similarity-ada-001" | "text-search-davinci-doc-001" | "text-search-curie-doc-001" | "text-search-babbage-doc-001" | "text-search-ada-doc-001" | "code-search-babbage-code-001" | "code-search-ada-code-001" | undefined) => Generator<number[], void, undefined>;
+export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, };
+export default api;

package/esm/model/gpt-4.js

@@ -1,2 +1,11 @@
-export * from '../encoding/cl100k_base.js';
+/* eslint-disable import/extensions */
+import { convertTokenBytePairEncodingFromTuples } from '../convertTokenBytePairEncodingFromTuples.js';
+import encoder from '../encodings/cl100k_base.js';
+import { GptEncoding } from '../GptEncoding.js';
+export * from '../specialTokens.js';
+const api = GptEncoding.getEncodingApiForModel('gpt-4', () => convertTokenBytePairEncodingFromTuples(encoder));
+const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, encodeChat, encodeChatGenerator, } = api;
+export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, };
+// eslint-disable-next-line import/no-default-export
+export default api;
 //# sourceMappingURL=gpt-4.js.map

package/esm/model/gpt-4.js.map

@@ -1 +1 @@
-{"version":3,"file":"gpt-4.js","sourceRoot":"","sources":["../../src/model/gpt-4.ts"],"names":[],"mappings":"AAAA,cAAc,4BAA4B,CAAA"}
+{"version":3,"file":"gpt-4.js","sourceRoot":"","sources":["../../src/model/gpt-4.ts"],"names":[],"mappings":"AAAA,sCAAsC;AACtC,OAAO,EAAE,sCAAsC,EAAE,MAAM,8CAA8C,CAAA;AACrG,OAAO,OAAO,MAAM,6BAA6B,CAAA;AACjD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAA;AAE/C,cAAc,qBAAqB,CAAA;AAEnC,MAAM,GAAG,GAAG,WAAW,CAAC,sBAAsB,CAAC,OAAO,EAAE,GAAG,EAAE,CAC3D,sCAAsC,CAAC,OAAO,CAAC,CAChD,CAAA;AACD,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,EAClB,UAAU,EACV,mBAAmB,GACpB,GAAG,GAAG,CAAA;AACP,OAAO,EACL,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,UAAU,EACV,mBAAmB,EACnB,eAAe,EACf,kBAAkB,GACnB,CAAA;AACD,oDAAoD;AACpD,eAAe,GAAG,CAAA"}

package/esm/modelParams.d.ts

@@ -1,6 +1,6 @@
 import { EncoderMap } from './EncoderMap.js';
-import type { EncodingName } from './mapping.js';
-export interface ModelParams {
+import type { EncodingName, ModelName } from './mapping.js';
+export interface EncodingParams {
     /**
      * The expected total number of tokens in the vocabulary, including both regular and special tokens.
      * This parameter is used to ensure that the combined number of regular and special tokens matches the expected total.
@@ -15,8 +15,9 @@ export interface ModelParams {
     tokenSplitRegex: RegExp;
     mergeableBytePairRanks: EncoderMap;
     specialTokenMapping: Map<string, number>;
+    modelName?: ModelName;
 }
 export type GetMergeableRanksFn = (encodingName: EncodingName) => EncoderMap;
 export type GetMergeableRanksAsyncFn = (encodingName: EncodingName) => Promise<EncoderMap>;
-export declare function getModelParams(encodingName: EncodingName, getMergeableRanks: GetMergeableRanksFn): ModelParams;
-export declare function getModelParamsAsync(encodingName: EncodingName, getMergeableRanks: GetMergeableRanksAsyncFn): Promise<ModelParams>;
+export declare function getEncodingParams(encodingName: EncodingName, getMergeableRanks: GetMergeableRanksFn): EncodingParams;
+export declare function getModelParamsAsync(encodingName: EncodingName, getMergeableRanks: GetMergeableRanksAsyncFn): Promise<EncodingParams>;

package/esm/modelParams.js

@@ -1,6 +1,6 @@
 /* eslint-disable no-magic-numbers */
 import { EncoderMap } from './EncoderMap.js';
-import { EndOfPrompt, EndOfText, FimMiddle, FimPrefix, FimSuffix, } from './specialTokens.js';
+import { EndOfPrompt, EndOfText, FimMiddle, FimPrefix, FimSuffix, ImEnd, ImSep, ImStart, } from './specialTokens.js';
 const tokenSplitRegex = /'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+/gu;
 function R50KBase(mergeableBytePairRanks) {
     return {
@@ -37,6 +37,9 @@ function Cl100KBase(mergeableBytePairRanks) {
         [FimPrefix, 100258],
         [FimMiddle, 100259],
         [FimSuffix, 100260],
+        [ImStart, 100264],
+        [ImEnd, 100265],
+        [ImSep, 100266],
         [EndOfPrompt, 100276],
     ]);
     return {
@@ -45,7 +48,7 @@ function Cl100KBase(mergeableBytePairRanks) {
         specialTokenMapping,
     };
 }
-export function getModelParams(encodingName, getMergeableRanks) {
+export function getEncodingParams(encodingName, getMergeableRanks) {
     const mergeableBytePairRanks = getMergeableRanks(encodingName);
     switch (encodingName.toLowerCase()) {
         case 'r50k_base':
@@ -62,6 +65,6 @@ export function getModelParams(encodingName, getMergeableRanks) {
 }
 export async function getModelParamsAsync(encodingName, getMergeableRanks) {
     const mergeableBytePairRanks = await getMergeableRanks(encodingName);
-    return getModelParams(encodingName, () => mergeableBytePairRanks);
+    return getEncodingParams(encodingName, () => mergeableBytePairRanks);
 }
 //# sourceMappingURL=modelParams.js.map

package/esm/modelParams.js.map

@@ -1 +1 @@
-{"version":3,"file":"modelParams.js","sourceRoot":"","sources":["../src/modelParams.ts"],"names":[],"mappings":"AAAA,qCAAqC;AACrC,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAA;AAE5C,OAAO,EACL,WAAW,EACX,SAAS,EACT,SAAS,EACT,SAAS,EACT,SAAS,GACV,MAAM,oBAAoB,CAAA;AAmB3B,MAAM,eAAe,GACnB,8EAA8E,CAAA;AAEhF,SAAS,QAAQ,CAAC,sBAAkC;IAClD,OAAO;QACL,sBAAsB,EAAE,KAAM;QAC9B,eAAe;QACf,sBAAsB;QACtB,mBAAmB,EAAE,IAAI,GAAG,CAAiB,CAAC,CAAC,SAAS,EAAE,KAAM,CAAC,CAAC,CAAC;KACpE,CAAA;AACH,CAAC;AAED,SAAS,QAAQ,CAAC,sBAAkC;IAClD,OAAO;QACL,sBAAsB,EAAE,KAAM;QAC9B,eAAe;QACf,sBAAsB;QACtB,mBAAmB,EAAE,IAAI,GAAG,CAAiB,CAAC,CAAC,SAAS,EAAE,KAAM,CAAC,CAAC,CAAC;KACpE,CAAA;AACH,CAAC;AAED,SAAS,QAAQ,CAAC,sBAAkC;IAClD,MAAM,mBAAmB,GAAG,IAAI,GAAG,CAAiB;QAClD,CAAC,SAAS,EAAE,KAAM,CAAC;QACnB,CAAC,SAAS,EAAE,KAAM,CAAC;QACnB,CAAC,SAAS,EAAE,KAAM,CAAC;QACnB,CAAC,SAAS,EAAE,KAAM,CAAC;KACpB,CAAC,CAAA;IAEF,OAAO;QACL,eAAe;QACf,sBAAsB;QACtB,mBAAmB;KACpB,CAAA;AACH,CAAC;AAED,SAAS,UAAU,CAAC,sBAAkC;IACpD,MAAM,mBAAmB,GAAG,IAAI,GAAG,CAAiB;QAClD,CAAC,SAAS,EAAE,MAAO,CAAC;QACpB,CAAC,SAAS,EAAE,MAAO,CAAC;QACpB,CAAC,SAAS,EAAE,MAAO,CAAC;QACpB,CAAC,SAAS,EAAE,MAAO,CAAC;QACpB,CAAC,WAAW,EAAE,MAAO,CAAC;KACvB,CAAC,CAAA;IAEF,OAAO;QACL,eAAe,EACb,uHAAuH;QACzH,sBAAsB;QACtB,mBAAmB;KACpB,CAAA;AACH,CAAC;AAOD,MAAM,UAAU,cAAc,CAC5B,YAA0B,EAC1B,iBAAsC;IAEtC,MAAM,sBAAsB,GAAG,iBAAiB,CAAC,YAAY,CAAC,CAAA;IAC9D,QAAQ,YAAY,CAAC,WAAW,EAAE,EAAE;QAClC,KAAK,WAAW;YACd,OAAO,QAAQ,CAAC,sBAAsB,CAAC,CAAA;QAEzC,KAAK,WAAW;YACd,OAAO,QAAQ,CAAC,sBAAsB,CAAC,CAAA;QAEzC,KAAK,WAAW;YACd,OAAO,QAAQ,CAAC,sBAAsB,CAAC,CAAA;QAEzC,KAAK,aAAa;YAChB,OAAO,UAAU,CAAC,sBAAsB,CAAC,CAAA;QAE3C;YACE,MAAM,IAAI,KAAK,CAAC,0BAA0B,YAAY,EAAE,CAAC,CAAA;KAC5D;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,YAA0B,EAC1B,iBAA2C;IAE3C,MAAM,sBAAsB,GAAG,MAAM,iBAAiB,CAAC,YAAY,CAAC,CAAA;IACpE,OAAO,cAAc,CAAC,YAAY,EAAE,GAAG,EAAE,CAAC,sBAAsB,CAAC,CAAA;AACnE,CAAC"}
+{"version":3,"file":"modelParams.js","sourceRoot":"","sources":["../src/modelParams.ts"],"names":[],"mappings":"AAAA,qCAAqC;AACrC,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAA;AAE5C,OAAO,EACL,WAAW,EACX,SAAS,EACT,SAAS,EACT,SAAS,EACT,SAAS,EACT,KAAK,EACL,KAAK,EACL,OAAO,GACR,MAAM,oBAAoB,CAAA;AAoB3B,MAAM,eAAe,GACnB,8EAA8E,CAAA;AAEhF,SAAS,QAAQ,CAAC,sBAAkC;IAClD,OAAO;QACL,sBAAsB,EAAE,KAAM;QAC9B,eAAe;QACf,sBAAsB;QACtB,mBAAmB,EAAE,IAAI,GAAG,CAAiB,CAAC,CAAC,SAAS,EAAE,KAAM,CAAC,CAAC,CAAC;KACpE,CAAA;AACH,CAAC;AAED,SAAS,QAAQ,CAAC,sBAAkC;IAClD,OAAO;QACL,sBAAsB,EAAE,KAAM;QAC9B,eAAe;QACf,sBAAsB;QACtB,mBAAmB,EAAE,IAAI,GAAG,CAAiB,CAAC,CAAC,SAAS,EAAE,KAAM,CAAC,CAAC,CAAC;KACpE,CAAA;AACH,CAAC;AAED,SAAS,QAAQ,CAAC,sBAAkC;IAClD,MAAM,mBAAmB,GAAG,IAAI,GAAG,CAAiB;QAClD,CAAC,SAAS,EAAE,KAAM,CAAC;QACnB,CAAC,SAAS,EAAE,KAAM,CAAC;QACnB,CAAC,SAAS,EAAE,KAAM,CAAC;QACnB,CAAC,SAAS,EAAE,KAAM,CAAC;KACpB,CAAC,CAAA;IAEF,OAAO;QACL,eAAe;QACf,sBAAsB;QACtB,mBAAmB;KACpB,CAAA;AACH,CAAC;AAED,SAAS,UAAU,CAAC,sBAAkC;IACpD,MAAM,mBAAmB,GAAG,IAAI,GAAG,CAAiB;QAClD,CAAC,SAAS,EAAE,MAAO,CAAC;QACpB,CAAC,SAAS,EAAE,MAAO,CAAC;QACpB,CAAC,SAAS,EAAE,MAAO,CAAC;QACpB,CAAC,SAAS,EAAE,MAAO,CAAC;QACpB,CAAC,OAAO,EAAE,MAAO,CAAC;QAClB,CAAC,KAAK,EAAE,MAAO,CAAC;QAChB,CAAC,KAAK,EAAE,MAAO,CAAC;QAChB,CAAC,WAAW,EAAE,MAAO,CAAC;KACvB,CAAC,CAAA;IAEF,OAAO;QACL,eAAe,EACb,uHAAuH;QACzH,sBAAsB;QACtB,mBAAmB;KACpB,CAAA;AACH,CAAC;AAOD,MAAM,UAAU,iBAAiB,CAC/B,YAA0B,EAC1B,iBAAsC;IAEtC,MAAM,sBAAsB,GAAG,iBAAiB,CAAC,YAAY,CAAC,CAAA;IAC9D,QAAQ,YAAY,CAAC,WAAW,EAAE,EAAE;QAClC,KAAK,WAAW;YACd,OAAO,QAAQ,CAAC,sBAAsB,CAAC,CAAA;QAEzC,KAAK,WAAW;YACd,OAAO,QAAQ,CAAC,sBAAsB,CAAC,CAAA;QAEzC,KAAK,WAAW;YACd,OAAO,QAAQ,CAAC,sBAAsB,CAAC,CAAA;QAEzC,KAAK,aAAa;YAChB,OAAO,UAAU,CAAC,sBAAsB,CAAC,CAAA;QAE3C;YACE,MAAM,IAAI,KAAK,CAAC,0BAA0B,YAAY,EAAE,CAAC,CAAA;KAC5D;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,YAA0B,EAC1B,iBAA2C;IAE3C,MAAM,sBAAsB,GAAG,MAAM,iBAAiB,CAAC,YAAY,CAAC,CAAA;IACpE,OAAO,iBAAiB,CAAC,YAAY,EAAE,GAAG,EAAE,CAAC,sBAAsB,CAAC,CAAA;AACtE,CAAC"}

package/esm/specialTokens.d.ts

@@ -2,4 +2,7 @@ export declare const EndOfText = "<|endoftext|>";
 export declare const FimPrefix = "<|fim_prefix|>";
 export declare const FimMiddle = "<|fim_middle|>";
 export declare const FimSuffix = "<|fim_suffix|>";
+export declare const ImStart = "<|im_start|>";
+export declare const ImEnd = "<|im_end|>";
+export declare const ImSep = "<|im_sep|>";
 export declare const EndOfPrompt = "<|endofprompt|>";

package/esm/specialTokens.js

@@ -2,5 +2,8 @@ export const EndOfText = '<|endoftext|>';
 export const FimPrefix = '<|fim_prefix|>';
 export const FimMiddle = '<|fim_middle|>';
 export const FimSuffix = '<|fim_suffix|>';
+export const ImStart = '<|im_start|>'; // 100264
+export const ImEnd = '<|im_end|>'; // 100265
+export const ImSep = '<|im_sep|>'; // 100266
 export const EndOfPrompt = '<|endofprompt|>';
 //# sourceMappingURL=specialTokens.js.map

package/esm/specialTokens.js.map

@@ -1 +1 @@
-{"version":3,"file":"specialTokens.js","sourceRoot":"","sources":["../src/specialTokens.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,SAAS,GAAG,eAAe,CAAA;AACxC,MAAM,CAAC,MAAM,SAAS,GAAG,gBAAgB,CAAA;AACzC,MAAM,CAAC,MAAM,SAAS,GAAG,gBAAgB,CAAA;AACzC,MAAM,CAAC,MAAM,SAAS,GAAG,gBAAgB,CAAA;AACzC,MAAM,CAAC,MAAM,WAAW,GAAG,iBAAiB,CAAA"}
+{"version":3,"file":"specialTokens.js","sourceRoot":"","sources":["../src/specialTokens.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,SAAS,GAAG,eAAe,CAAA;AACxC,MAAM,CAAC,MAAM,SAAS,GAAG,gBAAgB,CAAA;AACzC,MAAM,CAAC,MAAM,SAAS,GAAG,gBAAgB,CAAA;AACzC,MAAM,CAAC,MAAM,SAAS,GAAG,gBAAgB,CAAA;AACzC,MAAM,CAAC,MAAM,OAAO,GAAG,cAAc,CAAA,CAAC,SAAS;AAC/C,MAAM,CAAC,MAAM,KAAK,GAAG,YAAY,CAAA,CAAC,SAAS;AAC3C,MAAM,CAAC,MAAM,KAAK,GAAG,YAAY,CAAA,CAAC,SAAS;AAC3C,MAAM,CAAC,MAAM,WAAW,GAAG,iBAAiB,CAAA"}

package/esm/util.d.ts

@@ -0,0 +1,2 @@
+export declare function getMaxValueFromMap(map: Map<unknown, number>): number;
+export declare function getSpecialTokenRegex(tokens: Set<string>): RegExp;

package/esm/util.js

@@ -0,0 +1,14 @@
+import { escapeRegExp } from './escapeRegExp.js';
+export function getMaxValueFromMap(map) {
+    let max = 0;
+    map.forEach((val) => {
+        max = Math.max(max, val);
+    });
+    return max;
+}
+export function getSpecialTokenRegex(tokens) {
+    const escapedTokens = [...tokens].map(escapeRegExp);
+    const inner = escapedTokens.join('|');
+    return new RegExp(`(${inner})`);
+}
+//# sourceMappingURL=util.js.map

package/esm/util.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"util.js","sourceRoot":"","sources":["../src/util.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAA;AAEhD,MAAM,UAAU,kBAAkB,CAAC,GAAyB;IAC1D,IAAI,GAAG,GAAG,CAAC,CAAA;IACX,GAAG,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE;QAClB,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,CAAA;IAC1B,CAAC,CAAC,CAAA;IACF,OAAO,GAAG,CAAA;AACZ,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,MAAmB;IACtD,MAAM,aAAa,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,YAAY,CAAC,CAAA;IACnD,MAAM,KAAK,GAAG,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;IACrC,OAAO,IAAI,MAAM,CAAC,IAAI,KAAK,GAAG,CAAC,CAAA;AACjC,CAAC"}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "gpt-tokenizer",
-  "version": "2.0.0-beta.2",
-  "description": "BPE Encoder Decoder for GPT-2 / GPT-3",
+  "version": "2.1.0",
+  "description": "A pure JavaScript implementation of a BPE tokenizer (Encoder/Decoder) for GPT-2 / GPT-3 / GPT-4 and other OpenAI models",
   "keywords": [
     "BPE",
     "encoder",
@@ -72,14 +72,16 @@
     "dist"
   ],
   "scripts": {
+    "codegen:models": "yarn tsx src/codegen/generateByModel.ts",
+    "codegen:encodings": "yarn tsx src/codegen/generateJsEncodings.ts",
     "build": "yarn build:cjs && yarn build:esm && yarn build:umd",
     "build:cjs": "yarn rrun tsc --outDir cjs --module commonjs --target es2022 --project tsconfig-cjs.json",
     "build:esm": "yarn rrun tsc --outDir esm --module esnext --target es2022 && echo '{\"name\": \"gpt-tokenizer\", \"type\": \"module\"}' > ./esm/package.json",
     "build:umd": "yarn build:umd:cl100k_base && yarn build:umd:p50k_base && yarn build:umd:p50k_edit && yarn build:umd:r50k_base",
-    "build:umd:cl100k_base": "beemo webpack --entry='./src/main.ts' --env 'outDir=dist' --env 'moduleTarget=umd' --env 'engineTarget=web' --env 'codeTarget=es2022' --env 'name=GPTTokenizer' --env 'export=default' --env 'filename=cl100k_base.js'",
-    "build:umd:p50k_base": "beemo webpack --entry='./src/encoding/p50k_base.ts' --env 'outDir=dist' --env 'moduleTarget=umd' --env 'engineTarget=web' --env 'codeTarget=es2022' --env 'name=GPTTokenizer' --env 'export=default' --env 'filename=p50k_base.js'",
-    "build:umd:p50k_edit": "beemo webpack --entry='./src/encoding/p50k_edit.ts' --env 'outDir=dist' --env 'moduleTarget=umd' --env 'engineTarget=web' --env 'codeTarget=es2022' --env 'name=GPTTokenizer' --env 'export=default' --env 'filename=p50k_edit.js'",
-    "build:umd:r50k_base": "beemo webpack --entry='./src/encoding/r50k_base.ts' --env 'outDir=dist' --env 'moduleTarget=umd' --env 'engineTarget=web' --env 'codeTarget=es2022' --env 'name=GPTTokenizer' --env 'export=default' --env 'filename=r50k_base.js'",
+    "build:umd:cl100k_base": "beemo webpack --entry='./src/main.ts' --env 'outDir=dist' --env 'moduleTarget=umd' --env 'engineTarget=web' --env 'codeTarget=es2022' --env 'name=GPTTokenizer_cl100k_base' --env 'export=api' --env 'filename=cl100k_base.js'",
+    "build:umd:p50k_base": "beemo webpack --entry='./src/encoding/p50k_base.ts' --env 'outDir=dist' --env 'moduleTarget=umd' --env 'engineTarget=web' --env 'codeTarget=es2022' --env 'name=GPTTokenizer_p50k_base' --env 'export=api' --env 'filename=p50k_base.js'",
+    "build:umd:p50k_edit": "beemo webpack --entry='./src/encoding/p50k_edit.ts' --env 'outDir=dist' --env 'moduleTarget=umd' --env 'engineTarget=web' --env 'codeTarget=es2022' --env 'name=GPTTokenizer_p50k_edit' --env 'export=api' --env 'filename=p50k_edit.js'",
+    "build:umd:r50k_base": "beemo webpack --entry='./src/encoding/r50k_base.ts' --env 'outDir=dist' --env 'moduleTarget=umd' --env 'engineTarget=web' --env 'codeTarget=es2022' --env 'name=GPTTokenizer_r50k_base' --env 'export=api' --env 'filename=r50k_base.js'",
     "clean": "git clean -dfX --exclude=node_modules src && beemo typescript:sync-project-refs",
     "format": "yarn rrun prettier --write \"./{src,tests,.config}/**/!(*.d).{.js,jsx,ts,tsx,json,md}\"",
     "postinstallDev": "yarn prepare",

package/src/GptEncoding.test.ts

@@ -1,7 +1,12 @@
 import fs from 'fs'
 import path from 'path'
 import { GptEncoding } from './GptEncoding.js'
-import { type EncodingName, encodingNames } from './mapping.js'
+import {
+  type ChatModelName,
+  type EncodingName,
+  chatModelParams,
+  encodingNames,
+} from './mapping.js'
 import { resolveEncoding } from './resolveEncoding.js'
 
 const sharedResults = {
@@ -155,6 +160,78 @@ describe.each(encodingNames)('%s', (encodingName: EncodingName) => {
   })
 })
 
+const chatModelNames = Object.keys(chatModelParams) as readonly ChatModelName[]
+
+const exampleMessages = [
+  {
+    role: 'system',
+    content:
+      'You are a helpful, pattern-following assistant that translates corporate jargon into plain English.',
+  },
+  {
+    role: 'system',
+    name: 'example_user',
+    content: 'New synergies will help drive top-line growth.',
+  },
+  {
+    role: 'system',
+    name: 'example_assistant',
+    content: 'Things working well together will increase revenue.',
+  },
+  {
+    role: 'system',
+    name: 'example_user',
+    content:
+      "Let's circle back when we have more bandwidth to touch base on opportunities for increased leverage.",
+  },
+  {
+    role: 'system',
+    name: 'example_assistant',
+    content: "Let's talk later when we're less busy about how to do better.",
+  },
+  {
+    role: 'user',
+    content:
+      "This late pivot means we don't have time to boil the ocean for the client deliverable.",
+  },
+] as const
+
+describe.each(chatModelNames)('%s', (modelName) => {
+  const encoding = GptEncoding.getEncodingApiForModel(
+    modelName,
+    resolveEncoding,
+  )
+  const expectedEncodedLength = modelName.startsWith('gpt-3.5-turbo')
+    ? 127
+    : 121
+
+  describe('chat functionality', () => {
+    it('encodes a chat correctly', () => {
+      const encoded = encoding.encodeChat(exampleMessages)
+      expect(encoded).toMatchSnapshot()
+      expect(encoded).toHaveLength(expectedEncodedLength)
+
+      const decoded = encoding.decode(encoded)
+      expect(decoded).toMatchSnapshot()
+    })
+
+    it('isWithinTokenLimit: false', () => {
+      const isWithinTokenLimit = encoding.isWithinTokenLimit(
+        exampleMessages,
+        50,
+      )
+      expect(isWithinTokenLimit).toBe(false)
+    })
+    it('isWithinTokenLimit: true (number)', () => {
+      const isWithinTokenLimit = encoding.isWithinTokenLimit(
+        exampleMessages,
+        150,
+      )
+      expect(isWithinTokenLimit).toBe(expectedEncodedLength)
+    })
+  })
+})
+
 function loadTestPlans() {
   const testPlanPath = path.join(__dirname, '../data/TestPlans.txt')
   const testPlanData = fs.readFileSync(testPlanPath, 'utf8')