gpt-tokenizer 2.0.0 → 2.1.0

package/cjs/encoding/p50k_edit.js

@@ -17,14 +17,14 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.isWithinTokenLimit = exports.encodeGenerator = exports.encode = exports.decodeGenerator = exports.decodeAsyncGenerator = exports.decode = void 0;
+exports.isWithinTokenLimit = exports.encodeGenerator = exports.encode = exports.decodeGenerator = exports.decodeAsyncGenerator = exports.decode = exports.api = void 0;
 /* eslint-disable import/extensions */
 const convertTokenBytePairEncodingFromTuples_js_1 = require("../convertTokenBytePairEncodingFromTuples.js");
 const p50k_base_js_1 = __importDefault(require("../encodings/p50k_base.js"));
 const GptEncoding_js_1 = require("../GptEncoding.js");
 __exportStar(require("../specialTokens.js"), exports);
-const api = GptEncoding_js_1.GptEncoding.getEncodingApi('p50k_edit', () => (0, convertTokenBytePairEncodingFromTuples_js_1.convertTokenBytePairEncodingFromTuples)(p50k_base_js_1.default));
-const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, } = api;
+exports.api = GptEncoding_js_1.GptEncoding.getEncodingApi('p50k_edit', () => (0, convertTokenBytePairEncodingFromTuples_js_1.convertTokenBytePairEncodingFromTuples)(p50k_base_js_1.default));
+const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, } = exports.api;
 exports.decode = decode;
 exports.decodeAsyncGenerator = decodeAsyncGenerator;
 exports.decodeGenerator = decodeGenerator;
@@ -32,5 +32,5 @@ exports.encode = encode;
 exports.encodeGenerator = encodeGenerator;
 exports.isWithinTokenLimit = isWithinTokenLimit;
 // eslint-disable-next-line import/no-default-export
-exports.default = api;
+exports.default = exports.api;
 //# sourceMappingURL=p50k_edit.js.map

package/cjs/encoding/p50k_edit.js.map

@@ -1 +1 @@
-{"version":3,"file":"p50k_edit.js","sourceRoot":"","sources":["../../src/encoding/p50k_edit.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA,sCAAsC;AACtC,4GAAqG;AACrG,6EAA+C;AAC/C,sDAA+C;AAE/C,sDAAmC;AAEnC,MAAM,GAAG,GAAG,4BAAW,CAAC,cAAc,CAAC,WAAW,EAAE,GAAG,EAAE,CACvD,IAAA,kFAAsC,EAAC,sBAAO,CAAC,CAChD,CAAA;AACD,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,GACnB,GAAG,GAAG,CAAA;AAEL,wBAAM;AACN,oDAAoB;AACpB,0CAAe;AACf,wBAAM;AACN,0CAAe;AACf,gDAAkB;AAEpB,oDAAoD;AACpD,kBAAe,GAAG,CAAA"}
+{"version":3,"file":"p50k_edit.js","sourceRoot":"","sources":["../../src/encoding/p50k_edit.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA,sCAAsC;AACtC,4GAAqG;AACrG,6EAA+C;AAC/C,sDAA+C;AAE/C,sDAAmC;AAEtB,QAAA,GAAG,GAAG,4BAAW,CAAC,cAAc,CAAC,WAAW,EAAE,GAAG,EAAE,CAC9D,IAAA,kFAAsC,EAAC,sBAAO,CAAC,CAChD,CAAA;AACD,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,GACnB,GAAG,WAAG,CAAA;AAEL,wBAAM;AACN,oDAAoB;AACpB,0CAAe;AACf,wBAAM;AACN,0CAAe;AACf,gDAAkB;AAEpB,oDAAoD;AACpD,kBAAe,WAAG,CAAA"}

package/cjs/encoding/r50k_base.d.ts

@@ -1,6 +1,6 @@
 import { GptEncoding } from '../GptEncoding.js';
 export * from '../specialTokens.js';
-declare const api: GptEncoding;
-declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void, unknown>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, unknown>, encode: (lineToEncode: string, allowedSpecial?: Set<string>, disallowedSpecial?: Set<string>) => number[], encodeGenerator: (lineToEncode: string, allowedSpecial?: Set<string>, disallowedSpecial?: Set<string>) => Generator<number[], number, unknown>, isWithinTokenLimit: (text: string, tokenLimit: number) => number | false;
+export declare const api: GptEncoding;
+declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void, unknown>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, unknown>, encode: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, { allowedSpecial, disallowedSpecial, }?: import("../GptEncoding.js").EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, tokenLimit: number) => number | false;
 export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, };
 export default api;

package/cjs/encoding/r50k_base.js

@@ -17,14 +17,14 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.isWithinTokenLimit = exports.encodeGenerator = exports.encode = exports.decodeGenerator = exports.decodeAsyncGenerator = exports.decode = void 0;
+exports.isWithinTokenLimit = exports.encodeGenerator = exports.encode = exports.decodeGenerator = exports.decodeAsyncGenerator = exports.decode = exports.api = void 0;
 /* eslint-disable import/extensions */
 const convertTokenBytePairEncodingFromTuples_js_1 = require("../convertTokenBytePairEncodingFromTuples.js");
 const r50k_base_js_1 = __importDefault(require("../encodings/r50k_base.js"));
 const GptEncoding_js_1 = require("../GptEncoding.js");
 __exportStar(require("../specialTokens.js"), exports);
-const api = GptEncoding_js_1.GptEncoding.getEncodingApi('r50k_base', () => (0, convertTokenBytePairEncodingFromTuples_js_1.convertTokenBytePairEncodingFromTuples)(r50k_base_js_1.default));
-const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, } = api;
+exports.api = GptEncoding_js_1.GptEncoding.getEncodingApi('r50k_base', () => (0, convertTokenBytePairEncodingFromTuples_js_1.convertTokenBytePairEncodingFromTuples)(r50k_base_js_1.default));
+const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, } = exports.api;
 exports.decode = decode;
 exports.decodeAsyncGenerator = decodeAsyncGenerator;
 exports.decodeGenerator = decodeGenerator;
@@ -32,5 +32,5 @@ exports.encode = encode;
 exports.encodeGenerator = encodeGenerator;
 exports.isWithinTokenLimit = isWithinTokenLimit;
 // eslint-disable-next-line import/no-default-export
-exports.default = api;
+exports.default = exports.api;
 //# sourceMappingURL=r50k_base.js.map

package/cjs/encoding/r50k_base.js.map

@@ -1 +1 @@
-{"version":3,"file":"r50k_base.js","sourceRoot":"","sources":["../../src/encoding/r50k_base.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA,sCAAsC;AACtC,4GAAqG;AACrG,6EAA+C;AAC/C,sDAA+C;AAE/C,sDAAmC;AAEnC,MAAM,GAAG,GAAG,4BAAW,CAAC,cAAc,CAAC,WAAW,EAAE,GAAG,EAAE,CACvD,IAAA,kFAAsC,EAAC,sBAAO,CAAC,CAChD,CAAA;AACD,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,GACnB,GAAG,GAAG,CAAA;AAEL,wBAAM;AACN,oDAAoB;AACpB,0CAAe;AACf,wBAAM;AACN,0CAAe;AACf,gDAAkB;AAEpB,oDAAoD;AACpD,kBAAe,GAAG,CAAA"}
+{"version":3,"file":"r50k_base.js","sourceRoot":"","sources":["../../src/encoding/r50k_base.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA,sCAAsC;AACtC,4GAAqG;AACrG,6EAA+C;AAC/C,sDAA+C;AAE/C,sDAAmC;AAEtB,QAAA,GAAG,GAAG,4BAAW,CAAC,cAAc,CAAC,WAAW,EAAE,GAAG,EAAE,CAC9D,IAAA,kFAAsC,EAAC,sBAAO,CAAC,CAChD,CAAA;AACD,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,GACnB,GAAG,WAAG,CAAA;AAEL,wBAAM;AACN,oDAAoB;AACpB,0CAAe;AACf,wBAAM;AACN,0CAAe;AACf,gDAAkB;AAEpB,oDAAoD;AACpD,kBAAe,WAAG,CAAA"}

package/cjs/mapping.d.ts

@@ -5,7 +5,11 @@ export declare const r50k_base = "r50k_base";
 export declare const encodingNames: readonly ["cl100k_base", "p50k_base", "r50k_base", "p50k_edit"];
 export declare const modelToEncodingMap: {
     readonly 'gpt-4': "cl100k_base";
+    readonly 'gpt-4-32k': "cl100k_base";
+    readonly 'gpt-4-0314': "cl100k_base";
+    readonly 'gpt-4-32k-0314': "cl100k_base";
     readonly 'gpt-3.5-turbo': "cl100k_base";
+    readonly 'gpt-3.5-turbo-0301': "cl100k_base";
     readonly 'text-davinci-003': "p50k_base";
     readonly 'text-davinci-002': "p50k_base";
     readonly 'text-davinci-001': "r50k_base";
@@ -36,5 +40,38 @@ export declare const modelToEncodingMap: {
     readonly 'code-search-babbage-code-001': "r50k_base";
     readonly 'code-search-ada-code-001': "r50k_base";
 };
+export interface ChatParameters {
+    messageSeparator: string;
+    roleSeparator: string;
+}
+declare const internalChatModelParams: {
+    'gpt-3.5-turbo': {
+        messageSeparator: string;
+        roleSeparator: string;
+    };
+    'gpt-3.5-turbo-0301': {
+        messageSeparator: string;
+        roleSeparator: string;
+    };
+    'gpt-4': {
+        messageSeparator: string;
+        roleSeparator: string;
+    };
+    'gpt-4-0314': {
+        messageSeparator: string;
+        roleSeparator: string;
+    };
+    'gpt-4-32k': {
+        messageSeparator: string;
+        roleSeparator: string;
+    };
+    'gpt-4-32k-0314': {
+        messageSeparator: string;
+        roleSeparator: string;
+    };
+};
+export declare const chatModelParams: Partial<Record<ModelName, ChatParameters>>;
 export type ModelName = keyof typeof modelToEncodingMap;
+export type ChatModelName = keyof typeof internalChatModelParams;
 export type EncodingName = (typeof modelToEncodingMap)[ModelName];
+export {};

package/cjs/mapping.js

@@ -1,7 +1,8 @@
 "use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.modelToEncodingMap = exports.encodingNames = exports.r50k_base = exports.p50k_edit = exports.p50k_base = exports.cl100k_base = void 0;
 /* eslint-disable camelcase */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.chatModelParams = exports.modelToEncodingMap = exports.encodingNames = exports.r50k_base = exports.p50k_edit = exports.p50k_base = exports.cl100k_base = void 0;
+const specialTokens_js_1 = require("./specialTokens.js");
 exports.cl100k_base = 'cl100k_base';
 exports.p50k_base = 'p50k_base';
 exports.p50k_edit = 'p50k_edit';
@@ -15,7 +16,11 @@ exports.encodingNames = [
 exports.modelToEncodingMap = {
     // chat
     'gpt-4': exports.cl100k_base,
+    'gpt-4-32k': exports.cl100k_base,
+    'gpt-4-0314': exports.cl100k_base,
+    'gpt-4-32k-0314': exports.cl100k_base,
     'gpt-3.5-turbo': exports.cl100k_base,
+    'gpt-3.5-turbo-0301': exports.cl100k_base,
     // text
     'text-davinci-003': exports.p50k_base,
     'text-davinci-002': exports.p50k_base,
@@ -51,4 +56,31 @@ exports.modelToEncodingMap = {
     'code-search-babbage-code-001': exports.r50k_base,
     'code-search-ada-code-001': exports.r50k_base,
 };
+const internalChatModelParams = {
+    'gpt-3.5-turbo': {
+        messageSeparator: '\n',
+        roleSeparator: '\n',
+    },
+    'gpt-3.5-turbo-0301': {
+        messageSeparator: '\n',
+        roleSeparator: '\n',
+    },
+    'gpt-4': {
+        messageSeparator: '',
+        roleSeparator: specialTokens_js_1.ImSep,
+    },
+    'gpt-4-0314': {
+        messageSeparator: '',
+        roleSeparator: specialTokens_js_1.ImSep,
+    },
+    'gpt-4-32k': {
+        messageSeparator: '',
+        roleSeparator: specialTokens_js_1.ImSep,
+    },
+    'gpt-4-32k-0314': {
+        messageSeparator: '',
+        roleSeparator: specialTokens_js_1.ImSep,
+    },
+};
+exports.chatModelParams = internalChatModelParams;
 //# sourceMappingURL=mapping.js.map

package/cjs/mapping.js.map

@@ -1 +1 @@
-{"version":3,"file":"mapping.js","sourceRoot":"","sources":["../src/mapping.ts"],"names":[],"mappings":";;;AAAA,8BAA8B;AACjB,QAAA,WAAW,GAAG,aAAa,CAAA;AAC3B,QAAA,SAAS,GAAG,WAAW,CAAA;AACvB,QAAA,SAAS,GAAG,WAAW,CAAA;AACvB,QAAA,SAAS,GAAG,WAAW,CAAA;AAEvB,QAAA,aAAa,GAAG;IAC3B,mBAAW;IACX,iBAAS;IACT,iBAAS;IACT,iBAAS;CACD,CAAA;AAEG,QAAA,kBAAkB,GAAG;IAChC,OAAO;IACP,OAAO,EAAE,mBAAW;IACpB,eAAe,EAAE,mBAAW;IAC5B,OAAO;IACP,kBAAkB,EAAE,iBAAS;IAC7B,kBAAkB,EAAE,iBAAS;IAC7B,kBAAkB,EAAE,iBAAS;IAC7B,gBAAgB,EAAE,iBAAS;IAC3B,kBAAkB,EAAE,iBAAS;IAC7B,cAAc,EAAE,iBAAS;IACzB,OAAO,EAAE,iBAAS;IAClB,KAAK,EAAE,iBAAS;IAChB,OAAO,EAAE,iBAAS;IAClB,GAAG,EAAE,iBAAS;IACd,OAAO;IACP,kBAAkB,EAAE,iBAAS;IAC7B,kBAAkB,EAAE,iBAAS;IAC7B,kBAAkB,EAAE,iBAAS;IAC7B,kBAAkB,EAAE,iBAAS;IAC7B,eAAe,EAAE,iBAAS;IAC1B,eAAe,EAAE,iBAAS;IAC1B,OAAO;IACP,uBAAuB,EAAE,iBAAS;IAClC,uBAAuB,EAAE,iBAAS;IAClC,aAAa;IACb,wBAAwB,EAAE,mBAAW;IACrC,iBAAiB;IACjB,6BAA6B,EAAE,iBAAS;IACxC,2BAA2B,EAAE,iBAAS;IACtC,6BAA6B,EAAE,iBAAS;IACxC,yBAAyB,EAAE,iBAAS;IACpC,6BAA6B,EAAE,iBAAS;IACxC,2BAA2B,EAAE,iBAAS;IACtC,6BAA6B,EAAE,iBAAS;IACxC,yBAAyB,EAAE,iBAAS;IACpC,8BAA8B,EAAE,iBAAS;IACzC,0BAA0B,EAAE,iBAAS;CAC7B,CAAA"}
+{"version":3,"file":"mapping.js","sourceRoot":"","sources":["../src/mapping.ts"],"names":[],"mappings":";AAAA,8BAA8B;;;AAE9B,yDAA0C;AAE7B,QAAA,WAAW,GAAG,aAAa,CAAA;AAC3B,QAAA,SAAS,GAAG,WAAW,CAAA;AACvB,QAAA,SAAS,GAAG,WAAW,CAAA;AACvB,QAAA,SAAS,GAAG,WAAW,CAAA;AAEvB,QAAA,aAAa,GAAG;IAC3B,mBAAW;IACX,iBAAS;IACT,iBAAS;IACT,iBAAS;CACD,CAAA;AAEG,QAAA,kBAAkB,GAAG;IAChC,OAAO;IACP,OAAO,EAAE,mBAAW;IACpB,WAAW,EAAE,mBAAW;IACxB,YAAY,EAAE,mBAAW;IACzB,gBAAgB,EAAE,mBAAW;IAC7B,eAAe,EAAE,mBAAW;IAC5B,oBAAoB,EAAE,mBAAW;IACjC,OAAO;IACP,kBAAkB,EAAE,iBAAS;IAC7B,kBAAkB,EAAE,iBAAS;IAC7B,kBAAkB,EAAE,iBAAS;IAC7B,gBAAgB,EAAE,iBAAS;IAC3B,kBAAkB,EAAE,iBAAS;IAC7B,cAAc,EAAE,iBAAS;IACzB,OAAO,EAAE,iBAAS;IAClB,KAAK,EAAE,iBAAS;IAChB,OAAO,EAAE,iBAAS;IAClB,GAAG,EAAE,iBAAS;IACd,OAAO;IACP,kBAAkB,EAAE,iBAAS;IAC7B,kBAAkB,EAAE,iBAAS;IAC7B,kBAAkB,EAAE,iBAAS;IAC7B,kBAAkB,EAAE,iBAAS;IAC7B,eAAe,EAAE,iBAAS;IAC1B,eAAe,EAAE,iBAAS;IAC1B,OAAO;IACP,uBAAuB,EAAE,iBAAS;IAClC,uBAAuB,EAAE,iBAAS;IAClC,aAAa;IACb,wBAAwB,EAAE,mBAAW;IACrC,iBAAiB;IACjB,6BAA6B,EAAE,iBAAS;IACxC,2BAA2B,EAAE,iBAAS;IACtC,6BAA6B,EAAE,iBAAS;IACxC,yBAAyB,EAAE,iBAAS;IACpC,6BAA6B,EAAE,iBAAS;IACxC,2BAA2B,EAAE,iBAAS;IACtC,6BAA6B,EAAE,iBAAS;IACxC,yBAAyB,EAAE,iBAAS;IACpC,8BAA8B,EAAE,iBAAS;IACzC,0BAA0B,EAAE,iBAAS;CAC7B,CAAA;AAOV,MAAM,uBAAuB,GAAG;IAC9B,eAAe,EAAE;QACf,gBAAgB,EAAE,IAAI;QACtB,aAAa,EAAE,IAAI;KACpB;IACD,oBAAoB,EAAE;QACpB,gBAAgB,EAAE,IAAI;QACtB,aAAa,EAAE,IAAI;KACpB;IACD,OAAO,EAAE;QACP,gBAAgB,EAAE,EAAE;QACpB,aAAa,EAAE,wBAAK;KACrB;IACD,YAAY,EAAE;QACZ,gBAAgB,EAAE,EAAE;QACpB,aAAa,EAAE,wBAAK;KACrB;IACD,WAAW,EAAE;QACX,gBAAgB,EAAE,EAAE;QACpB,aAAa,EAAE,wBAAK;KACrB;IACD,gBAAgB,EAAE;QAChB,gBAAgB,EAAE,EAAE;QACpB,aAAa,EAAE,wBAAK;KACrB;CACF,CAAA;AAEY,QAAA,eAAe,GAC1B,uBAAuB,CAAA"}

package/cjs/model/gpt-3.5-turbo-0301.d.ts

@@ -0,0 +1,6 @@
+import { GptEncoding } from '../GptEncoding.js';
+export * from '../specialTokens.js';
+declare const api: GptEncoding;
+declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void, unknown>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, unknown>, encode: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, { allowedSpecial, disallowedSpecial, }?: import("../GptEncoding.js").EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, tokenLimit: number) => number | false, encodeChat: (chat: readonly import("../GptEncoding.js").ChatMessage[], model?: "gpt-4" | "gpt-4-32k" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo" | "gpt-3.5-turbo-0301" | "text-davinci-003" | "text-davinci-002" | "text-davinci-001" | "text-curie-001" | "text-babbage-001" | "text-ada-001" | "davinci" | "curie" | "babbage" | "ada" | "code-davinci-002" | "code-davinci-001" | "code-cushman-002" | "code-cushman-001" | "davinci-codex" | "cushman-codex" | "text-davinci-edit-001" | "code-davinci-edit-001" | "text-embedding-ada-002" | "text-similarity-davinci-001" | "text-similarity-curie-001" | "text-similarity-babbage-001" | "text-similarity-ada-001" | "text-search-davinci-doc-001" | "text-search-curie-doc-001" | "text-search-babbage-doc-001" | "text-search-ada-doc-001" | "code-search-babbage-code-001" | "code-search-ada-code-001" | undefined) => number[], encodeChatGenerator: (chat: Iterable<import("../GptEncoding.js").ChatMessage>, model?: "gpt-4" | "gpt-4-32k" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo" | "gpt-3.5-turbo-0301" | "text-davinci-003" | "text-davinci-002" | "text-davinci-001" | "text-curie-001" | "text-babbage-001" | "text-ada-001" | "davinci" | "curie" | "babbage" | "ada" | "code-davinci-002" | "code-davinci-001" | "code-cushman-002" | "code-cushman-001" | "davinci-codex" | "cushman-codex" | "text-davinci-edit-001" | "code-davinci-edit-001" | "text-embedding-ada-002" | "text-similarity-davinci-001" | "text-similarity-curie-001" | "text-similarity-babbage-001" | "text-similarity-ada-001" | "text-search-davinci-doc-001" | "text-search-curie-doc-001" | "text-search-babbage-doc-001" | "text-search-ada-doc-001" | "code-search-babbage-code-001" | "code-search-ada-code-001" | undefined) => Generator<number[], void, undefined>;
+export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, };
+export default api;

package/cjs/model/gpt-3.5-turbo-0301.js

@@ -0,0 +1,38 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.isWithinTokenLimit = exports.encodeGenerator = exports.encodeChatGenerator = exports.encodeChat = exports.encode = exports.decodeGenerator = exports.decodeAsyncGenerator = exports.decode = void 0;
+/* eslint-disable import/extensions */
+const convertTokenBytePairEncodingFromTuples_js_1 = require("../convertTokenBytePairEncodingFromTuples.js");
+const cl100k_base_js_1 = __importDefault(require("../encodings/cl100k_base.js"));
+const GptEncoding_js_1 = require("../GptEncoding.js");
+__exportStar(require("../specialTokens.js"), exports);
+const api = GptEncoding_js_1.GptEncoding.getEncodingApiForModel('gpt-3.5-turbo-0301', () => (0, convertTokenBytePairEncodingFromTuples_js_1.convertTokenBytePairEncodingFromTuples)(cl100k_base_js_1.default));
+const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, encodeChat, encodeChatGenerator, } = api;
+exports.decode = decode;
+exports.decodeAsyncGenerator = decodeAsyncGenerator;
+exports.decodeGenerator = decodeGenerator;
+exports.encode = encode;
+exports.encodeGenerator = encodeGenerator;
+exports.isWithinTokenLimit = isWithinTokenLimit;
+exports.encodeChat = encodeChat;
+exports.encodeChatGenerator = encodeChatGenerator;
+// eslint-disable-next-line import/no-default-export
+exports.default = api;
+//# sourceMappingURL=gpt-3.5-turbo-0301.js.map

package/cjs/model/gpt-3.5-turbo-0301.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"gpt-3.5-turbo-0301.js","sourceRoot":"","sources":["../../src/model/gpt-3.5-turbo-0301.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA,sCAAsC;AACtC,4GAAqG;AACrG,iFAAiD;AACjD,sDAA+C;AAE/C,sDAAmC;AAEnC,MAAM,GAAG,GAAG,4BAAW,CAAC,sBAAsB,CAAC,oBAAoB,EAAE,GAAG,EAAE,CACxE,IAAA,kFAAsC,EAAC,wBAAO,CAAC,CAChD,CAAA;AACD,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,EAClB,UAAU,EACV,mBAAmB,GACpB,GAAG,GAAG,CAAA;AAEL,wBAAM;AACN,oDAAoB;AACpB,0CAAe;AACf,wBAAM;AAGN,0CAAe;AACf,gDAAkB;AAHlB,gCAAU;AACV,kDAAmB;AAIrB,oDAAoD;AACpD,kBAAe,GAAG,CAAA"}

package/cjs/model/gpt-3.5-turbo.d.ts

@@ -1 +1,6 @@
-export * from '../encoding/cl100k_base.js';
+import { GptEncoding } from '../GptEncoding.js';
+export * from '../specialTokens.js';
+declare const api: GptEncoding;
+declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void, unknown>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, unknown>, encode: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, { allowedSpecial, disallowedSpecial, }?: import("../GptEncoding.js").EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, tokenLimit: number) => number | false, encodeChat: (chat: readonly import("../GptEncoding.js").ChatMessage[], model?: "gpt-4" | "gpt-4-32k" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo" | "gpt-3.5-turbo-0301" | "text-davinci-003" | "text-davinci-002" | "text-davinci-001" | "text-curie-001" | "text-babbage-001" | "text-ada-001" | "davinci" | "curie" | "babbage" | "ada" | "code-davinci-002" | "code-davinci-001" | "code-cushman-002" | "code-cushman-001" | "davinci-codex" | "cushman-codex" | "text-davinci-edit-001" | "code-davinci-edit-001" | "text-embedding-ada-002" | "text-similarity-davinci-001" | "text-similarity-curie-001" | "text-similarity-babbage-001" | "text-similarity-ada-001" | "text-search-davinci-doc-001" | "text-search-curie-doc-001" | "text-search-babbage-doc-001" | "text-search-ada-doc-001" | "code-search-babbage-code-001" | "code-search-ada-code-001" | undefined) => number[], encodeChatGenerator: (chat: Iterable<import("../GptEncoding.js").ChatMessage>, model?: "gpt-4" | "gpt-4-32k" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo" | "gpt-3.5-turbo-0301" | "text-davinci-003" | "text-davinci-002" | "text-davinci-001" | "text-curie-001" | "text-babbage-001" | "text-ada-001" | "davinci" | "curie" | "babbage" | "ada" | "code-davinci-002" | "code-davinci-001" | "code-cushman-002" | "code-cushman-001" | "davinci-codex" | "cushman-codex" | "text-davinci-edit-001" | "code-davinci-edit-001" | "text-embedding-ada-002" | "text-similarity-davinci-001" | "text-similarity-curie-001" | "text-similarity-babbage-001" | "text-similarity-ada-001" | "text-search-davinci-doc-001" | "text-search-curie-doc-001" | "text-search-babbage-doc-001" | "text-search-ada-doc-001" | "code-search-babbage-code-001" | "code-search-ada-code-001" | undefined) => Generator<number[], void, undefined>;
+export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, };
+export default api;

package/cjs/model/gpt-3.5-turbo.js

@@ -13,6 +13,26 @@ var __createBinding = (this && this.__createBinding) || (Object.create ? (functi
 var __exportStar = (this && this.__exportStar) || function(m, exports) {
     for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
 };
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
-__exportStar(require("../encoding/cl100k_base.js"), exports);
+exports.isWithinTokenLimit = exports.encodeGenerator = exports.encodeChatGenerator = exports.encodeChat = exports.encode = exports.decodeGenerator = exports.decodeAsyncGenerator = exports.decode = void 0;
+/* eslint-disable import/extensions */
+const convertTokenBytePairEncodingFromTuples_js_1 = require("../convertTokenBytePairEncodingFromTuples.js");
+const cl100k_base_js_1 = __importDefault(require("../encodings/cl100k_base.js"));
+const GptEncoding_js_1 = require("../GptEncoding.js");
+__exportStar(require("../specialTokens.js"), exports);
+const api = GptEncoding_js_1.GptEncoding.getEncodingApiForModel('gpt-3.5-turbo', () => (0, convertTokenBytePairEncodingFromTuples_js_1.convertTokenBytePairEncodingFromTuples)(cl100k_base_js_1.default));
+const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, encodeChat, encodeChatGenerator, } = api;
+exports.decode = decode;
+exports.decodeAsyncGenerator = decodeAsyncGenerator;
+exports.decodeGenerator = decodeGenerator;
+exports.encode = encode;
+exports.encodeGenerator = encodeGenerator;
+exports.isWithinTokenLimit = isWithinTokenLimit;
+exports.encodeChat = encodeChat;
+exports.encodeChatGenerator = encodeChatGenerator;
+// eslint-disable-next-line import/no-default-export
+exports.default = api;
 //# sourceMappingURL=gpt-3.5-turbo.js.map

package/cjs/model/gpt-3.5-turbo.js.map

@@ -1 +1 @@
-{"version":3,"file":"gpt-3.5-turbo.js","sourceRoot":"","sources":["../../src/model/gpt-3.5-turbo.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,6DAA0C"}
+{"version":3,"file":"gpt-3.5-turbo.js","sourceRoot":"","sources":["../../src/model/gpt-3.5-turbo.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA,sCAAsC;AACtC,4GAAqG;AACrG,iFAAiD;AACjD,sDAA+C;AAE/C,sDAAmC;AAEnC,MAAM,GAAG,GAAG,4BAAW,CAAC,sBAAsB,CAAC,eAAe,EAAE,GAAG,EAAE,CACnE,IAAA,kFAAsC,EAAC,wBAAO,CAAC,CAChD,CAAA;AACD,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,EAClB,UAAU,EACV,mBAAmB,GACpB,GAAG,GAAG,CAAA;AAEL,wBAAM;AACN,oDAAoB;AACpB,0CAAe;AACf,wBAAM;AAGN,0CAAe;AACf,gDAAkB;AAHlB,gCAAU;AACV,kDAAmB;AAIrB,oDAAoD;AACpD,kBAAe,GAAG,CAAA"}

package/cjs/model/gpt-4-0314.d.ts

@@ -0,0 +1,6 @@
+import { GptEncoding } from '../GptEncoding.js';
+export * from '../specialTokens.js';
+declare const api: GptEncoding;
+declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void, unknown>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, unknown>, encode: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, { allowedSpecial, disallowedSpecial, }?: import("../GptEncoding.js").EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, tokenLimit: number) => number | false, encodeChat: (chat: readonly import("../GptEncoding.js").ChatMessage[], model?: "gpt-4" | "gpt-4-32k" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo" | "gpt-3.5-turbo-0301" | "text-davinci-003" | "text-davinci-002" | "text-davinci-001" | "text-curie-001" | "text-babbage-001" | "text-ada-001" | "davinci" | "curie" | "babbage" | "ada" | "code-davinci-002" | "code-davinci-001" | "code-cushman-002" | "code-cushman-001" | "davinci-codex" | "cushman-codex" | "text-davinci-edit-001" | "code-davinci-edit-001" | "text-embedding-ada-002" | "text-similarity-davinci-001" | "text-similarity-curie-001" | "text-similarity-babbage-001" | "text-similarity-ada-001" | "text-search-davinci-doc-001" | "text-search-curie-doc-001" | "text-search-babbage-doc-001" | "text-search-ada-doc-001" | "code-search-babbage-code-001" | "code-search-ada-code-001" | undefined) => number[], encodeChatGenerator: (chat: Iterable<import("../GptEncoding.js").ChatMessage>, model?: "gpt-4" | "gpt-4-32k" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo" | "gpt-3.5-turbo-0301" | "text-davinci-003" | "text-davinci-002" | "text-davinci-001" | "text-curie-001" | "text-babbage-001" | "text-ada-001" | "davinci" | "curie" | "babbage" | "ada" | "code-davinci-002" | "code-davinci-001" | "code-cushman-002" | "code-cushman-001" | "davinci-codex" | "cushman-codex" | "text-davinci-edit-001" | "code-davinci-edit-001" | "text-embedding-ada-002" | "text-similarity-davinci-001" | "text-similarity-curie-001" | "text-similarity-babbage-001" | "text-similarity-ada-001" | "text-search-davinci-doc-001" | "text-search-curie-doc-001" | "text-search-babbage-doc-001" | "text-search-ada-doc-001" | "code-search-babbage-code-001" | "code-search-ada-code-001" | undefined) => Generator<number[], void, undefined>;
+export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, };
+export default api;

package/cjs/model/gpt-4-0314.js

@@ -0,0 +1,38 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.isWithinTokenLimit = exports.encodeGenerator = exports.encodeChatGenerator = exports.encodeChat = exports.encode = exports.decodeGenerator = exports.decodeAsyncGenerator = exports.decode = void 0;
+/* eslint-disable import/extensions */
+const convertTokenBytePairEncodingFromTuples_js_1 = require("../convertTokenBytePairEncodingFromTuples.js");
+const cl100k_base_js_1 = __importDefault(require("../encodings/cl100k_base.js"));
+const GptEncoding_js_1 = require("../GptEncoding.js");
+__exportStar(require("../specialTokens.js"), exports);
+const api = GptEncoding_js_1.GptEncoding.getEncodingApiForModel('gpt-4-0314', () => (0, convertTokenBytePairEncodingFromTuples_js_1.convertTokenBytePairEncodingFromTuples)(cl100k_base_js_1.default));
+const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, encodeChat, encodeChatGenerator, } = api;
+exports.decode = decode;
+exports.decodeAsyncGenerator = decodeAsyncGenerator;
+exports.decodeGenerator = decodeGenerator;
+exports.encode = encode;
+exports.encodeGenerator = encodeGenerator;
+exports.isWithinTokenLimit = isWithinTokenLimit;
+exports.encodeChat = encodeChat;
+exports.encodeChatGenerator = encodeChatGenerator;
+// eslint-disable-next-line import/no-default-export
+exports.default = api;
+//# sourceMappingURL=gpt-4-0314.js.map

package/cjs/model/gpt-4-0314.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"gpt-4-0314.js","sourceRoot":"","sources":["../../src/model/gpt-4-0314.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA,sCAAsC;AACtC,4GAAqG;AACrG,iFAAiD;AACjD,sDAA+C;AAE/C,sDAAmC;AAEnC,MAAM,GAAG,GAAG,4BAAW,CAAC,sBAAsB,CAAC,YAAY,EAAE,GAAG,EAAE,CAChE,IAAA,kFAAsC,EAAC,wBAAO,CAAC,CAChD,CAAA;AACD,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,EAClB,UAAU,EACV,mBAAmB,GACpB,GAAG,GAAG,CAAA;AAEL,wBAAM;AACN,oDAAoB;AACpB,0CAAe;AACf,wBAAM;AAGN,0CAAe;AACf,gDAAkB;AAHlB,gCAAU;AACV,kDAAmB;AAIrB,oDAAoD;AACpD,kBAAe,GAAG,CAAA"}

package/cjs/model/gpt-4-32k-0314.d.ts

@@ -0,0 +1,6 @@
+import { GptEncoding } from '../GptEncoding.js';
+export * from '../specialTokens.js';
+declare const api: GptEncoding;
+declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void, unknown>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, unknown>, encode: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, { allowedSpecial, disallowedSpecial, }?: import("../GptEncoding.js").EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, tokenLimit: number) => number | false, encodeChat: (chat: readonly import("../GptEncoding.js").ChatMessage[], model?: "gpt-4" | "gpt-4-32k" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo" | "gpt-3.5-turbo-0301" | "text-davinci-003" | "text-davinci-002" | "text-davinci-001" | "text-curie-001" | "text-babbage-001" | "text-ada-001" | "davinci" | "curie" | "babbage" | "ada" | "code-davinci-002" | "code-davinci-001" | "code-cushman-002" | "code-cushman-001" | "davinci-codex" | "cushman-codex" | "text-davinci-edit-001" | "code-davinci-edit-001" | "text-embedding-ada-002" | "text-similarity-davinci-001" | "text-similarity-curie-001" | "text-similarity-babbage-001" | "text-similarity-ada-001" | "text-search-davinci-doc-001" | "text-search-curie-doc-001" | "text-search-babbage-doc-001" | "text-search-ada-doc-001" | "code-search-babbage-code-001" | "code-search-ada-code-001" | undefined) => number[], encodeChatGenerator: (chat: Iterable<import("../GptEncoding.js").ChatMessage>, model?: "gpt-4" | "gpt-4-32k" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo" | "gpt-3.5-turbo-0301" | "text-davinci-003" | "text-davinci-002" | "text-davinci-001" | "text-curie-001" | "text-babbage-001" | "text-ada-001" | "davinci" | "curie" | "babbage" | "ada" | "code-davinci-002" | "code-davinci-001" | "code-cushman-002" | "code-cushman-001" | "davinci-codex" | "cushman-codex" | "text-davinci-edit-001" | "code-davinci-edit-001" | "text-embedding-ada-002" | "text-similarity-davinci-001" | "text-similarity-curie-001" | "text-similarity-babbage-001" | "text-similarity-ada-001" | "text-search-davinci-doc-001" | "text-search-curie-doc-001" | "text-search-babbage-doc-001" | "text-search-ada-doc-001" | "code-search-babbage-code-001" | "code-search-ada-code-001" | undefined) => Generator<number[], void, undefined>;
+export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, };
+export default api;

package/cjs/model/gpt-4-32k-0314.js

@@ -0,0 +1,38 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.isWithinTokenLimit = exports.encodeGenerator = exports.encodeChatGenerator = exports.encodeChat = exports.encode = exports.decodeGenerator = exports.decodeAsyncGenerator = exports.decode = void 0;
+/* eslint-disable import/extensions */
+const convertTokenBytePairEncodingFromTuples_js_1 = require("../convertTokenBytePairEncodingFromTuples.js");
+const cl100k_base_js_1 = __importDefault(require("../encodings/cl100k_base.js"));
+const GptEncoding_js_1 = require("../GptEncoding.js");
+__exportStar(require("../specialTokens.js"), exports);
+const api = GptEncoding_js_1.GptEncoding.getEncodingApiForModel('gpt-4-32k-0314', () => (0, convertTokenBytePairEncodingFromTuples_js_1.convertTokenBytePairEncodingFromTuples)(cl100k_base_js_1.default));
+const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, encodeChat, encodeChatGenerator, } = api;
+exports.decode = decode;
+exports.decodeAsyncGenerator = decodeAsyncGenerator;
+exports.decodeGenerator = decodeGenerator;
+exports.encode = encode;
+exports.encodeGenerator = encodeGenerator;
+exports.isWithinTokenLimit = isWithinTokenLimit;
+exports.encodeChat = encodeChat;
+exports.encodeChatGenerator = encodeChatGenerator;
+// eslint-disable-next-line import/no-default-export
+exports.default = api;
+//# sourceMappingURL=gpt-4-32k-0314.js.map

package/cjs/model/gpt-4-32k-0314.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"gpt-4-32k-0314.js","sourceRoot":"","sources":["../../src/model/gpt-4-32k-0314.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA,sCAAsC;AACtC,4GAAqG;AACrG,iFAAiD;AACjD,sDAA+C;AAE/C,sDAAmC;AAEnC,MAAM,GAAG,GAAG,4BAAW,CAAC,sBAAsB,CAAC,gBAAgB,EAAE,GAAG,EAAE,CACpE,IAAA,kFAAsC,EAAC,wBAAO,CAAC,CAChD,CAAA;AACD,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,EAClB,UAAU,EACV,mBAAmB,GACpB,GAAG,GAAG,CAAA;AAEL,wBAAM;AACN,oDAAoB;AACpB,0CAAe;AACf,wBAAM;AAGN,0CAAe;AACf,gDAAkB;AAHlB,gCAAU;AACV,kDAAmB;AAIrB,oDAAoD;AACpD,kBAAe,GAAG,CAAA"}

package/cjs/model/gpt-4-32k.d.ts

@@ -0,0 +1,6 @@
+import { GptEncoding } from '../GptEncoding.js';
+export * from '../specialTokens.js';
+declare const api: GptEncoding;
+declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void, unknown>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, unknown>, encode: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, { allowedSpecial, disallowedSpecial, }?: import("../GptEncoding.js").EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, tokenLimit: number) => number | false, encodeChat: (chat: readonly import("../GptEncoding.js").ChatMessage[], model?: "gpt-4" | "gpt-4-32k" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo" | "gpt-3.5-turbo-0301" | "text-davinci-003" | "text-davinci-002" | "text-davinci-001" | "text-curie-001" | "text-babbage-001" | "text-ada-001" | "davinci" | "curie" | "babbage" | "ada" | "code-davinci-002" | "code-davinci-001" | "code-cushman-002" | "code-cushman-001" | "davinci-codex" | "cushman-codex" | "text-davinci-edit-001" | "code-davinci-edit-001" | "text-embedding-ada-002" | "text-similarity-davinci-001" | "text-similarity-curie-001" | "text-similarity-babbage-001" | "text-similarity-ada-001" | "text-search-davinci-doc-001" | "text-search-curie-doc-001" | "text-search-babbage-doc-001" | "text-search-ada-doc-001" | "code-search-babbage-code-001" | "code-search-ada-code-001" | undefined) => number[], encodeChatGenerator: (chat: Iterable<import("../GptEncoding.js").ChatMessage>, model?: "gpt-4" | "gpt-4-32k" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo" | "gpt-3.5-turbo-0301" | "text-davinci-003" | "text-davinci-002" | "text-davinci-001" | "text-curie-001" | "text-babbage-001" | "text-ada-001" | "davinci" | "curie" | "babbage" | "ada" | "code-davinci-002" | "code-davinci-001" | "code-cushman-002" | "code-cushman-001" | "davinci-codex" | "cushman-codex" | "text-davinci-edit-001" | "code-davinci-edit-001" | "text-embedding-ada-002" | "text-similarity-davinci-001" | "text-similarity-curie-001" | "text-similarity-babbage-001" | "text-similarity-ada-001" | "text-search-davinci-doc-001" | "text-search-curie-doc-001" | "text-search-babbage-doc-001" | "text-search-ada-doc-001" | "code-search-babbage-code-001" | "code-search-ada-code-001" | undefined) => Generator<number[], void, undefined>;
+export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, };
+export default api;

package/cjs/model/gpt-4-32k.js

@@ -0,0 +1,38 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.isWithinTokenLimit = exports.encodeGenerator = exports.encodeChatGenerator = exports.encodeChat = exports.encode = exports.decodeGenerator = exports.decodeAsyncGenerator = exports.decode = void 0;
+/* eslint-disable import/extensions */
+const convertTokenBytePairEncodingFromTuples_js_1 = require("../convertTokenBytePairEncodingFromTuples.js");
+const cl100k_base_js_1 = __importDefault(require("../encodings/cl100k_base.js"));
+const GptEncoding_js_1 = require("../GptEncoding.js");
+__exportStar(require("../specialTokens.js"), exports);
+const api = GptEncoding_js_1.GptEncoding.getEncodingApiForModel('gpt-4-32k', () => (0, convertTokenBytePairEncodingFromTuples_js_1.convertTokenBytePairEncodingFromTuples)(cl100k_base_js_1.default));
+const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, encodeChat, encodeChatGenerator, } = api;
+exports.decode = decode;
+exports.decodeAsyncGenerator = decodeAsyncGenerator;
+exports.decodeGenerator = decodeGenerator;
+exports.encode = encode;
+exports.encodeGenerator = encodeGenerator;
+exports.isWithinTokenLimit = isWithinTokenLimit;
+exports.encodeChat = encodeChat;
+exports.encodeChatGenerator = encodeChatGenerator;
+// eslint-disable-next-line import/no-default-export
+exports.default = api;
+//# sourceMappingURL=gpt-4-32k.js.map

package/cjs/model/gpt-4-32k.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"gpt-4-32k.js","sourceRoot":"","sources":["../../src/model/gpt-4-32k.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA,sCAAsC;AACtC,4GAAqG;AACrG,iFAAiD;AACjD,sDAA+C;AAE/C,sDAAmC;AAEnC,MAAM,GAAG,GAAG,4BAAW,CAAC,sBAAsB,CAAC,WAAW,EAAE,GAAG,EAAE,CAC/D,IAAA,kFAAsC,EAAC,wBAAO,CAAC,CAChD,CAAA;AACD,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,EAClB,UAAU,EACV,mBAAmB,GACpB,GAAG,GAAG,CAAA;AAEL,wBAAM;AACN,oDAAoB;AACpB,0CAAe;AACf,wBAAM;AAGN,0CAAe;AACf,gDAAkB;AAHlB,gCAAU;AACV,kDAAmB;AAIrB,oDAAoD;AACpD,kBAAe,GAAG,CAAA"}

package/cjs/model/gpt-4.d.ts

@@ -1 +1,6 @@
-export * from '../encoding/cl100k_base.js';
+import { GptEncoding } from '../GptEncoding.js';
+export * from '../specialTokens.js';
+declare const api: GptEncoding;
+declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void, unknown>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, unknown>, encode: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, { allowedSpecial, disallowedSpecial, }?: import("../GptEncoding.js").EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, tokenLimit: number) => number | false, encodeChat: (chat: readonly import("../GptEncoding.js").ChatMessage[], model?: "gpt-4" | "gpt-4-32k" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo" | "gpt-3.5-turbo-0301" | "text-davinci-003" | "text-davinci-002" | "text-davinci-001" | "text-curie-001" | "text-babbage-001" | "text-ada-001" | "davinci" | "curie" | "babbage" | "ada" | "code-davinci-002" | "code-davinci-001" | "code-cushman-002" | "code-cushman-001" | "davinci-codex" | "cushman-codex" | "text-davinci-edit-001" | "code-davinci-edit-001" | "text-embedding-ada-002" | "text-similarity-davinci-001" | "text-similarity-curie-001" | "text-similarity-babbage-001" | "text-similarity-ada-001" | "text-search-davinci-doc-001" | "text-search-curie-doc-001" | "text-search-babbage-doc-001" | "text-search-ada-doc-001" | "code-search-babbage-code-001" | "code-search-ada-code-001" | undefined) => number[], encodeChatGenerator: (chat: Iterable<import("../GptEncoding.js").ChatMessage>, model?: "gpt-4" | "gpt-4-32k" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo" | "gpt-3.5-turbo-0301" | "text-davinci-003" | "text-davinci-002" | "text-davinci-001" | "text-curie-001" | "text-babbage-001" | "text-ada-001" | "davinci" | "curie" | "babbage" | "ada" | "code-davinci-002" | "code-davinci-001" | "code-cushman-002" | "code-cushman-001" | "davinci-codex" | "cushman-codex" | "text-davinci-edit-001" | "code-davinci-edit-001" | "text-embedding-ada-002" | "text-similarity-davinci-001" | "text-similarity-curie-001" | "text-similarity-babbage-001" | "text-similarity-ada-001" | "text-search-davinci-doc-001" | "text-search-curie-doc-001" | "text-search-babbage-doc-001" | "text-search-ada-doc-001" | "code-search-babbage-code-001" | "code-search-ada-code-001" | undefined) => Generator<number[], void, undefined>;
+export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, };
+export default api;

package/cjs/model/gpt-4.js

@@ -13,6 +13,26 @@ var __createBinding = (this && this.__createBinding) || (Object.create ? (functi
 var __exportStar = (this && this.__exportStar) || function(m, exports) {
     for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
 };
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
-__exportStar(require("../encoding/cl100k_base.js"), exports);
+exports.isWithinTokenLimit = exports.encodeGenerator = exports.encodeChatGenerator = exports.encodeChat = exports.encode = exports.decodeGenerator = exports.decodeAsyncGenerator = exports.decode = void 0;
+/* eslint-disable import/extensions */
+const convertTokenBytePairEncodingFromTuples_js_1 = require("../convertTokenBytePairEncodingFromTuples.js");
+const cl100k_base_js_1 = __importDefault(require("../encodings/cl100k_base.js"));
+const GptEncoding_js_1 = require("../GptEncoding.js");
+__exportStar(require("../specialTokens.js"), exports);
+const api = GptEncoding_js_1.GptEncoding.getEncodingApiForModel('gpt-4', () => (0, convertTokenBytePairEncodingFromTuples_js_1.convertTokenBytePairEncodingFromTuples)(cl100k_base_js_1.default));
+const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, encodeChat, encodeChatGenerator, } = api;
+exports.decode = decode;
+exports.decodeAsyncGenerator = decodeAsyncGenerator;
+exports.decodeGenerator = decodeGenerator;
+exports.encode = encode;
+exports.encodeGenerator = encodeGenerator;
+exports.isWithinTokenLimit = isWithinTokenLimit;
+exports.encodeChat = encodeChat;
+exports.encodeChatGenerator = encodeChatGenerator;
+// eslint-disable-next-line import/no-default-export
+exports.default = api;
 //# sourceMappingURL=gpt-4.js.map

package/cjs/model/gpt-4.js.map

@@ -1 +1 @@
-{"version":3,"file":"gpt-4.js","sourceRoot":"","sources":["../../src/model/gpt-4.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,6DAA0C"}
+{"version":3,"file":"gpt-4.js","sourceRoot":"","sources":["../../src/model/gpt-4.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA,sCAAsC;AACtC,4GAAqG;AACrG,iFAAiD;AACjD,sDAA+C;AAE/C,sDAAmC;AAEnC,MAAM,GAAG,GAAG,4BAAW,CAAC,sBAAsB,CAAC,OAAO,EAAE,GAAG,EAAE,CAC3D,IAAA,kFAAsC,EAAC,wBAAO,CAAC,CAChD,CAAA;AACD,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,EAClB,UAAU,EACV,mBAAmB,GACpB,GAAG,GAAG,CAAA;AAEL,wBAAM;AACN,oDAAoB;AACpB,0CAAe;AACf,wBAAM;AAGN,0CAAe;AACf,gDAAkB;AAHlB,gCAAU;AACV,kDAAmB;AAIrB,oDAAoD;AACpD,kBAAe,GAAG,CAAA"}

package/cjs/modelParams.d.ts

@@ -1,6 +1,6 @@
 import { EncoderMap } from './EncoderMap.js';
-import type { EncodingName } from './mapping.js';
-export interface ModelParams {
+import type { EncodingName, ModelName } from './mapping.js';
+export interface EncodingParams {
     /**
      * The expected total number of tokens in the vocabulary, including both regular and special tokens.
      * This parameter is used to ensure that the combined number of regular and special tokens matches the expected total.
@@ -15,8 +15,9 @@ export interface ModelParams {
     tokenSplitRegex: RegExp;
     mergeableBytePairRanks: EncoderMap;
     specialTokenMapping: Map<string, number>;
+    modelName?: ModelName;
 }
 export type GetMergeableRanksFn = (encodingName: EncodingName) => EncoderMap;
 export type GetMergeableRanksAsyncFn = (encodingName: EncodingName) => Promise<EncoderMap>;
-export declare function getModelParams(encodingName: EncodingName, getMergeableRanks: GetMergeableRanksFn): ModelParams;
-export declare function getModelParamsAsync(encodingName: EncodingName, getMergeableRanks: GetMergeableRanksAsyncFn): Promise<ModelParams>;
+export declare function getEncodingParams(encodingName: EncodingName, getMergeableRanks: GetMergeableRanksFn): EncodingParams;
+export declare function getModelParamsAsync(encodingName: EncodingName, getMergeableRanks: GetMergeableRanksAsyncFn): Promise<EncodingParams>;

package/cjs/modelParams.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.getModelParamsAsync = exports.getModelParams = void 0;
+exports.getModelParamsAsync = exports.getEncodingParams = void 0;
 const specialTokens_js_1 = require("./specialTokens.js");
 const tokenSplitRegex = /'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+/gu;
 function R50KBase(mergeableBytePairRanks) {
@@ -38,6 +38,9 @@ function Cl100KBase(mergeableBytePairRanks) {
         [specialTokens_js_1.FimPrefix, 100258],
         [specialTokens_js_1.FimMiddle, 100259],
         [specialTokens_js_1.FimSuffix, 100260],
+        [specialTokens_js_1.ImStart, 100264],
+        [specialTokens_js_1.ImEnd, 100265],
+        [specialTokens_js_1.ImSep, 100266],
         [specialTokens_js_1.EndOfPrompt, 100276],
     ]);
     return {
@@ -46,7 +49,7 @@ function Cl100KBase(mergeableBytePairRanks) {
         specialTokenMapping,
     };
 }
-function getModelParams(encodingName, getMergeableRanks) {
+function getEncodingParams(encodingName, getMergeableRanks) {
     const mergeableBytePairRanks = getMergeableRanks(encodingName);
     switch (encodingName.toLowerCase()) {
         case 'r50k_base':
@@ -61,10 +64,10 @@ function getModelParams(encodingName, getMergeableRanks) {
             throw new Error(`Unknown encoding name: ${encodingName}`);
     }
 }
-exports.getModelParams = getModelParams;
+exports.getEncodingParams = getEncodingParams;
 async function getModelParamsAsync(encodingName, getMergeableRanks) {
     const mergeableBytePairRanks = await getMergeableRanks(encodingName);
-    return getModelParams(encodingName, () => mergeableBytePairRanks);
+    return getEncodingParams(encodingName, () => mergeableBytePairRanks);
 }
 exports.getModelParamsAsync = getModelParamsAsync;
 //# sourceMappingURL=modelParams.js.map

package/cjs/modelParams.js.map

@@ -1 +1 @@
-{"version":3,"file":"modelParams.js","sourceRoot":"","sources":["../src/modelParams.ts"],"names":[],"mappings":";;;AAGA,yDAM2B;AAmB3B,MAAM,eAAe,GACnB,8EAA8E,CAAA;AAEhF,SAAS,QAAQ,CAAC,sBAAkC;IAClD,OAAO;QACL,sBAAsB,EAAE,KAAM;QAC9B,eAAe;QACf,sBAAsB;QACtB,mBAAmB,EAAE,IAAI,GAAG,CAAiB,CAAC,CAAC,4BAAS,EAAE,KAAM,CAAC,CAAC,CAAC;KACpE,CAAA;AACH,CAAC;AAED,SAAS,QAAQ,CAAC,sBAAkC;IAClD,OAAO;QACL,sBAAsB,EAAE,KAAM;QAC9B,eAAe;QACf,sBAAsB;QACtB,mBAAmB,EAAE,IAAI,GAAG,CAAiB,CAAC,CAAC,4BAAS,EAAE,KAAM,CAAC,CAAC,CAAC;KACpE,CAAA;AACH,CAAC;AAED,SAAS,QAAQ,CAAC,sBAAkC;IAClD,MAAM,mBAAmB,GAAG,IAAI,GAAG,CAAiB;QAClD,CAAC,4BAAS,EAAE,KAAM,CAAC;QACnB,CAAC,4BAAS,EAAE,KAAM,CAAC;QACnB,CAAC,4BAAS,EAAE,KAAM,CAAC;QACnB,CAAC,4BAAS,EAAE,KAAM,CAAC;KACpB,CAAC,CAAA;IAEF,OAAO;QACL,eAAe;QACf,sBAAsB;QACtB,mBAAmB;KACpB,CAAA;AACH,CAAC;AAED,SAAS,UAAU,CAAC,sBAAkC;IACpD,MAAM,mBAAmB,GAAG,IAAI,GAAG,CAAiB;QAClD,CAAC,4BAAS,EAAE,MAAO,CAAC;QACpB,CAAC,4BAAS,EAAE,MAAO,CAAC;QACpB,CAAC,4BAAS,EAAE,MAAO,CAAC;QACpB,CAAC,4BAAS,EAAE,MAAO,CAAC;QACpB,CAAC,8BAAW,EAAE,MAAO,CAAC;KACvB,CAAC,CAAA;IAEF,OAAO;QACL,eAAe,EACb,uHAAuH;QACzH,sBAAsB;QACtB,mBAAmB;KACpB,CAAA;AACH,CAAC;AAOD,SAAgB,cAAc,CAC5B,YAA0B,EAC1B,iBAAsC;IAEtC,MAAM,sBAAsB,GAAG,iBAAiB,CAAC,YAAY,CAAC,CAAA;IAC9D,QAAQ,YAAY,CAAC,WAAW,EAAE,EAAE;QAClC,KAAK,WAAW;YACd,OAAO,QAAQ,CAAC,sBAAsB,CAAC,CAAA;QAEzC,KAAK,WAAW;YACd,OAAO,QAAQ,CAAC,sBAAsB,CAAC,CAAA;QAEzC,KAAK,WAAW;YACd,OAAO,QAAQ,CAAC,sBAAsB,CAAC,CAAA;QAEzC,KAAK,aAAa;YAChB,OAAO,UAAU,CAAC,sBAAsB,CAAC,CAAA;QAE3C;YACE,MAAM,IAAI,KAAK,CAAC,0BAA0B,YAAY,EAAE,CAAC,CAAA;KAC5D;AACH,CAAC;AArBD,wCAqBC;AAEM,KAAK,UAAU,mBAAmB,CACvC,YAA0B,EAC1B,iBAA2C;IAE3C,MAAM,sBAAsB,GAAG,MAAM,iBAAiB,CAAC,YAAY,CAAC,CAAA;IACpE,OAAO,cAAc,CAAC,YAAY,EAAE,GAAG,EAAE,CAAC,sBAAsB,CAAC,CAAA;AACnE,CAAC;AAND,kDAMC"}
+{"version":3,"file":"modelParams.js","sourceRoot":"","sources":["../src/modelParams.ts"],"names":[],"mappings":";;;AAGA,yDAS2B;AAoB3B,MAAM,eAAe,GACnB,8EAA8E,CAAA;AAEhF,SAAS,QAAQ,CAAC,sBAAkC;IAClD,OAAO;QACL,sBAAsB,EAAE,KAAM;QAC9B,eAAe;QACf,sBAAsB;QACtB,mBAAmB,EAAE,IAAI,GAAG,CAAiB,CAAC,CAAC,4BAAS,EAAE,KAAM,CAAC,CAAC,CAAC;KACpE,CAAA;AACH,CAAC;AAED,SAAS,QAAQ,CAAC,sBAAkC;IAClD,OAAO;QACL,sBAAsB,EAAE,KAAM;QAC9B,eAAe;QACf,sBAAsB;QACtB,mBAAmB,EAAE,IAAI,GAAG,CAAiB,CAAC,CAAC,4BAAS,EAAE,KAAM,CAAC,CAAC,CAAC;KACpE,CAAA;AACH,CAAC;AAED,SAAS,QAAQ,CAAC,sBAAkC;IAClD,MAAM,mBAAmB,GAAG,IAAI,GAAG,CAAiB;QAClD,CAAC,4BAAS,EAAE,KAAM,CAAC;QACnB,CAAC,4BAAS,EAAE,KAAM,CAAC;QACnB,CAAC,4BAAS,EAAE,KAAM,CAAC;QACnB,CAAC,4BAAS,EAAE,KAAM,CAAC;KACpB,CAAC,CAAA;IAEF,OAAO;QACL,eAAe;QACf,sBAAsB;QACtB,mBAAmB;KACpB,CAAA;AACH,CAAC;AAED,SAAS,UAAU,CAAC,sBAAkC;IACpD,MAAM,mBAAmB,GAAG,IAAI,GAAG,CAAiB;QAClD,CAAC,4BAAS,EAAE,MAAO,CAAC;QACpB,CAAC,4BAAS,EAAE,MAAO,CAAC;QACpB,CAAC,4BAAS,EAAE,MAAO,CAAC;QACpB,CAAC,4BAAS,EAAE,MAAO,CAAC;QACpB,CAAC,0BAAO,EAAE,MAAO,CAAC;QAClB,CAAC,wBAAK,EAAE,MAAO,CAAC;QAChB,CAAC,wBAAK,EAAE,MAAO,CAAC;QAChB,CAAC,8BAAW,EAAE,MAAO,CAAC;KACvB,CAAC,CAAA;IAEF,OAAO;QACL,eAAe,EACb,uHAAuH;QACzH,sBAAsB;QACtB,mBAAmB;KACpB,CAAA;AACH,CAAC;AAOD,SAAgB,iBAAiB,CAC/B,YAA0B,EAC1B,iBAAsC;IAEtC,MAAM,sBAAsB,GAAG,iBAAiB,CAAC,YAAY,CAAC,CAAA;IAC9D,QAAQ,YAAY,CAAC,WAAW,EAAE,EAAE;QAClC,KAAK,WAAW;YACd,OAAO,QAAQ,CAAC,sBAAsB,CAAC,CAAA;QAEzC,KAAK,WAAW;YACd,OAAO,QAAQ,CAAC,sBAAsB,CAAC,CAAA;QAEzC,KAAK,WAAW;YACd,OAAO,QAAQ,CAAC,sBAAsB,CAAC,CAAA;QAEzC,KAAK,aAAa;YAChB,OAAO,UAAU,CAAC,sBAAsB,CAAC,CAAA;QAE3C;YACE,MAAM,IAAI,KAAK,CAAC,0BAA0B,YAAY,EAAE,CAAC,CAAA;KAC5D;AACH,CAAC;AArBD,8CAqBC;AAEM,KAAK,UAAU,mBAAmB,CACvC,YAA0B,EAC1B,iBAA2C;IAE3C,MAAM,sBAAsB,GAAG,MAAM,iBAAiB,CAAC,YAAY,CAAC,CAAA;IACpE,OAAO,iBAAiB,CAAC,YAAY,EAAE,GAAG,EAAE,CAAC,sBAAsB,CAAC,CAAA;AACtE,CAAC;AAND,kDAMC"}

package/cjs/specialTokens.d.ts

@@ -2,4 +2,7 @@ export declare const EndOfText = "<|endoftext|>";
 export declare const FimPrefix = "<|fim_prefix|>";
 export declare const FimMiddle = "<|fim_middle|>";
 export declare const FimSuffix = "<|fim_suffix|>";
+export declare const ImStart = "<|im_start|>";
+export declare const ImEnd = "<|im_end|>";
+export declare const ImSep = "<|im_sep|>";
 export declare const EndOfPrompt = "<|endofprompt|>";