gpt-tokenizer 2.0.0 → 2.1.1

package/cjs/mapping.d.ts

@@ -5,7 +5,11 @@ export declare const r50k_base = "r50k_base";
 export declare const encodingNames: readonly ["cl100k_base", "p50k_base", "r50k_base", "p50k_edit"];
 export declare const modelToEncodingMap: {
     readonly 'gpt-4': "cl100k_base";
+    readonly 'gpt-4-32k': "cl100k_base";
+    readonly 'gpt-4-0314': "cl100k_base";
+    readonly 'gpt-4-32k-0314': "cl100k_base";
     readonly 'gpt-3.5-turbo': "cl100k_base";
+    readonly 'gpt-3.5-turbo-0301': "cl100k_base";
     readonly 'text-davinci-003': "p50k_base";
     readonly 'text-davinci-002': "p50k_base";
     readonly 'text-davinci-001': "r50k_base";
@@ -36,5 +40,38 @@ export declare const modelToEncodingMap: {
     readonly 'code-search-babbage-code-001': "r50k_base";
     readonly 'code-search-ada-code-001': "r50k_base";
 };
+export interface ChatParameters {
+    messageSeparator: string;
+    roleSeparator: string;
+}
+declare const internalChatModelParams: {
+    'gpt-3.5-turbo': {
+        messageSeparator: string;
+        roleSeparator: string;
+    };
+    'gpt-3.5-turbo-0301': {
+        messageSeparator: string;
+        roleSeparator: string;
+    };
+    'gpt-4': {
+        messageSeparator: string;
+        roleSeparator: string;
+    };
+    'gpt-4-0314': {
+        messageSeparator: string;
+        roleSeparator: string;
+    };
+    'gpt-4-32k': {
+        messageSeparator: string;
+        roleSeparator: string;
+    };
+    'gpt-4-32k-0314': {
+        messageSeparator: string;
+        roleSeparator: string;
+    };
+};
+export declare const chatModelParams: Partial<Record<ModelName, ChatParameters>>;
 export type ModelName = keyof typeof modelToEncodingMap;
+export type ChatModelName = keyof typeof internalChatModelParams;
 export type EncodingName = (typeof modelToEncodingMap)[ModelName];
+export {};

package/cjs/mapping.js

@@ -1,7 +1,8 @@
 "use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.modelToEncodingMap = exports.encodingNames = exports.r50k_base = exports.p50k_edit = exports.p50k_base = exports.cl100k_base = void 0;
 /* eslint-disable camelcase */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.chatModelParams = exports.modelToEncodingMap = exports.encodingNames = exports.r50k_base = exports.p50k_edit = exports.p50k_base = exports.cl100k_base = void 0;
+const specialTokens_js_1 = require("./specialTokens.js");
 exports.cl100k_base = 'cl100k_base';
 exports.p50k_base = 'p50k_base';
 exports.p50k_edit = 'p50k_edit';
@@ -15,7 +16,11 @@ exports.encodingNames = [
 exports.modelToEncodingMap = {
     // chat
     'gpt-4': exports.cl100k_base,
+    'gpt-4-32k': exports.cl100k_base,
+    'gpt-4-0314': exports.cl100k_base,
+    'gpt-4-32k-0314': exports.cl100k_base,
     'gpt-3.5-turbo': exports.cl100k_base,
+    'gpt-3.5-turbo-0301': exports.cl100k_base,
     // text
     'text-davinci-003': exports.p50k_base,
     'text-davinci-002': exports.p50k_base,
@@ -51,4 +56,31 @@ exports.modelToEncodingMap = {
     'code-search-babbage-code-001': exports.r50k_base,
     'code-search-ada-code-001': exports.r50k_base,
 };
+const internalChatModelParams = {
+    'gpt-3.5-turbo': {
+        messageSeparator: '\n',
+        roleSeparator: '\n',
+    },
+    'gpt-3.5-turbo-0301': {
+        messageSeparator: '\n',
+        roleSeparator: '\n',
+    },
+    'gpt-4': {
+        messageSeparator: '',
+        roleSeparator: specialTokens_js_1.ImSep,
+    },
+    'gpt-4-0314': {
+        messageSeparator: '',
+        roleSeparator: specialTokens_js_1.ImSep,
+    },
+    'gpt-4-32k': {
+        messageSeparator: '',
+        roleSeparator: specialTokens_js_1.ImSep,
+    },
+    'gpt-4-32k-0314': {
+        messageSeparator: '',
+        roleSeparator: specialTokens_js_1.ImSep,
+    },
+};
+exports.chatModelParams = internalChatModelParams;
 //# sourceMappingURL=mapping.js.map

package/cjs/mapping.js.map

@@ -1 +1 @@
-{"version":3,"file":"mapping.js","sourceRoot":"","sources":["../src/mapping.ts"],"names":[],"mappings":";;;AAAA,8BAA8B;AACjB,QAAA,WAAW,GAAG,aAAa,CAAA;AAC3B,QAAA,SAAS,GAAG,WAAW,CAAA;AACvB,QAAA,SAAS,GAAG,WAAW,CAAA;AACvB,QAAA,SAAS,GAAG,WAAW,CAAA;AAEvB,QAAA,aAAa,GAAG;IAC3B,mBAAW;IACX,iBAAS;IACT,iBAAS;IACT,iBAAS;CACD,CAAA;AAEG,QAAA,kBAAkB,GAAG;IAChC,OAAO;IACP,OAAO,EAAE,mBAAW;IACpB,eAAe,EAAE,mBAAW;IAC5B,OAAO;IACP,kBAAkB,EAAE,iBAAS;IAC7B,kBAAkB,EAAE,iBAAS;IAC7B,kBAAkB,EAAE,iBAAS;IAC7B,gBAAgB,EAAE,iBAAS;IAC3B,kBAAkB,EAAE,iBAAS;IAC7B,cAAc,EAAE,iBAAS;IACzB,OAAO,EAAE,iBAAS;IAClB,KAAK,EAAE,iBAAS;IAChB,OAAO,EAAE,iBAAS;IAClB,GAAG,EAAE,iBAAS;IACd,OAAO;IACP,kBAAkB,EAAE,iBAAS;IAC7B,kBAAkB,EAAE,iBAAS;IAC7B,kBAAkB,EAAE,iBAAS;IAC7B,kBAAkB,EAAE,iBAAS;IAC7B,eAAe,EAAE,iBAAS;IAC1B,eAAe,EAAE,iBAAS;IAC1B,OAAO;IACP,uBAAuB,EAAE,iBAAS;IAClC,uBAAuB,EAAE,iBAAS;IAClC,aAAa;IACb,wBAAwB,EAAE,mBAAW;IACrC,iBAAiB;IACjB,6BAA6B,EAAE,iBAAS;IACxC,2BAA2B,EAAE,iBAAS;IACtC,6BAA6B,EAAE,iBAAS;IACxC,yBAAyB,EAAE,iBAAS;IACpC,6BAA6B,EAAE,iBAAS;IACxC,2BAA2B,EAAE,iBAAS;IACtC,6BAA6B,EAAE,iBAAS;IACxC,yBAAyB,EAAE,iBAAS;IACpC,8BAA8B,EAAE,iBAAS;IACzC,0BAA0B,EAAE,iBAAS;CAC7B,CAAA"}
+{"version":3,"file":"mapping.js","sourceRoot":"","sources":["../src/mapping.ts"],"names":[],"mappings":";AAAA,8BAA8B;;;AAE9B,yDAA0C;AAE7B,QAAA,WAAW,GAAG,aAAa,CAAA;AAC3B,QAAA,SAAS,GAAG,WAAW,CAAA;AACvB,QAAA,SAAS,GAAG,WAAW,CAAA;AACvB,QAAA,SAAS,GAAG,WAAW,CAAA;AAEvB,QAAA,aAAa,GAAG;IAC3B,mBAAW;IACX,iBAAS;IACT,iBAAS;IACT,iBAAS;CACD,CAAA;AAEG,QAAA,kBAAkB,GAAG;IAChC,OAAO;IACP,OAAO,EAAE,mBAAW;IACpB,WAAW,EAAE,mBAAW;IACxB,YAAY,EAAE,mBAAW;IACzB,gBAAgB,EAAE,mBAAW;IAC7B,eAAe,EAAE,mBAAW;IAC5B,oBAAoB,EAAE,mBAAW;IACjC,OAAO;IACP,kBAAkB,EAAE,iBAAS;IAC7B,kBAAkB,EAAE,iBAAS;IAC7B,kBAAkB,EAAE,iBAAS;IAC7B,gBAAgB,EAAE,iBAAS;IAC3B,kBAAkB,EAAE,iBAAS;IAC7B,cAAc,EAAE,iBAAS;IACzB,OAAO,EAAE,iBAAS;IAClB,KAAK,EAAE,iBAAS;IAChB,OAAO,EAAE,iBAAS;IAClB,GAAG,EAAE,iBAAS;IACd,OAAO;IACP,kBAAkB,EAAE,iBAAS;IAC7B,kBAAkB,EAAE,iBAAS;IAC7B,kBAAkB,EAAE,iBAAS;IAC7B,kBAAkB,EAAE,iBAAS;IAC7B,eAAe,EAAE,iBAAS;IAC1B,eAAe,EAAE,iBAAS;IAC1B,OAAO;IACP,uBAAuB,EAAE,iBAAS;IAClC,uBAAuB,EAAE,iBAAS;IAClC,aAAa;IACb,wBAAwB,EAAE,mBAAW;IACrC,iBAAiB;IACjB,6BAA6B,EAAE,iBAAS;IACxC,2BAA2B,EAAE,iBAAS;IACtC,6BAA6B,EAAE,iBAAS;IACxC,yBAAyB,EAAE,iBAAS;IACpC,6BAA6B,EAAE,iBAAS;IACxC,2BAA2B,EAAE,iBAAS;IACtC,6BAA6B,EAAE,iBAAS;IACxC,yBAAyB,EAAE,iBAAS;IACpC,8BAA8B,EAAE,iBAAS;IACzC,0BAA0B,EAAE,iBAAS;CAC7B,CAAA;AAOV,MAAM,uBAAuB,GAAG;IAC9B,eAAe,EAAE;QACf,gBAAgB,EAAE,IAAI;QACtB,aAAa,EAAE,IAAI;KACpB;IACD,oBAAoB,EAAE;QACpB,gBAAgB,EAAE,IAAI;QACtB,aAAa,EAAE,IAAI;KACpB;IACD,OAAO,EAAE;QACP,gBAAgB,EAAE,EAAE;QACpB,aAAa,EAAE,wBAAK;KACrB;IACD,YAAY,EAAE;QACZ,gBAAgB,EAAE,EAAE;QACpB,aAAa,EAAE,wBAAK;KACrB;IACD,WAAW,EAAE;QACX,gBAAgB,EAAE,EAAE;QACpB,aAAa,EAAE,wBAAK;KACrB;IACD,gBAAgB,EAAE;QAChB,gBAAgB,EAAE,EAAE;QACpB,aAAa,EAAE,wBAAK;KACrB;CACF,CAAA;AAEY,QAAA,eAAe,GAC1B,uBAAuB,CAAA"}

package/cjs/model/ada.d.ts

@@ -1 +1,2 @@
+export { default } from '../encoding/r50k_base.js';
 export * from '../encoding/r50k_base.js';

package/cjs/model/ada.js

@@ -13,6 +13,13 @@ var __createBinding = (this && this.__createBinding) || (Object.create ? (functi
 var __exportStar = (this && this.__exportStar) || function(m, exports) {
     for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
 };
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = void 0;
+// eslint-disable-next-line no-restricted-exports, import/no-default-export
+var r50k_base_js_1 = require("../encoding/r50k_base.js");
+Object.defineProperty(exports, "default", { enumerable: true, get: function () { return __importDefault(r50k_base_js_1).default; } });
 __exportStar(require("../encoding/r50k_base.js"), exports);
 //# sourceMappingURL=ada.js.map

package/cjs/model/ada.js.map

@@ -1 +1 @@
-{"version":3,"file":"ada.js","sourceRoot":"","sources":["../../src/model/ada.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,2DAAwC"}
+{"version":3,"file":"ada.js","sourceRoot":"","sources":["../../src/model/ada.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA,2EAA2E;AAC3E,yDAAkD;AAAzC,wHAAA,OAAO,OAAA;AAChB,2DAAwC"}

package/cjs/model/babbage.d.ts

@@ -1 +1,2 @@
+export { default } from '../encoding/r50k_base.js';
 export * from '../encoding/r50k_base.js';

package/cjs/model/babbage.js

@@ -13,6 +13,13 @@ var __createBinding = (this && this.__createBinding) || (Object.create ? (functi
 var __exportStar = (this && this.__exportStar) || function(m, exports) {
     for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
 };
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = void 0;
+// eslint-disable-next-line no-restricted-exports, import/no-default-export
+var r50k_base_js_1 = require("../encoding/r50k_base.js");
+Object.defineProperty(exports, "default", { enumerable: true, get: function () { return __importDefault(r50k_base_js_1).default; } });
 __exportStar(require("../encoding/r50k_base.js"), exports);
 //# sourceMappingURL=babbage.js.map

package/cjs/model/babbage.js.map

@@ -1 +1 @@
-{"version":3,"file":"babbage.js","sourceRoot":"","sources":["../../src/model/babbage.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,2DAAwC"}
+{"version":3,"file":"babbage.js","sourceRoot":"","sources":["../../src/model/babbage.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA,2EAA2E;AAC3E,yDAAkD;AAAzC,wHAAA,OAAO,OAAA;AAChB,2DAAwC"}

package/cjs/model/code-cushman-001.d.ts

@@ -1 +1,2 @@
+export { default } from '../encoding/p50k_base.js';
 export * from '../encoding/p50k_base.js';

package/cjs/model/code-cushman-001.js

@@ -13,6 +13,13 @@ var __createBinding = (this && this.__createBinding) || (Object.create ? (functi
 var __exportStar = (this && this.__exportStar) || function(m, exports) {
     for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
 };
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = void 0;
+// eslint-disable-next-line no-restricted-exports, import/no-default-export
+var p50k_base_js_1 = require("../encoding/p50k_base.js");
+Object.defineProperty(exports, "default", { enumerable: true, get: function () { return __importDefault(p50k_base_js_1).default; } });
 __exportStar(require("../encoding/p50k_base.js"), exports);
 //# sourceMappingURL=code-cushman-001.js.map

package/cjs/model/code-cushman-001.js.map

@@ -1 +1 @@
-{"version":3,"file":"code-cushman-001.js","sourceRoot":"","sources":["../../src/model/code-cushman-001.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,2DAAwC"}
+{"version":3,"file":"code-cushman-001.js","sourceRoot":"","sources":["../../src/model/code-cushman-001.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA,2EAA2E;AAC3E,yDAAkD;AAAzC,wHAAA,OAAO,OAAA;AAChB,2DAAwC"}

package/cjs/model/code-cushman-002.d.ts

@@ -1 +1,2 @@
+export { default } from '../encoding/p50k_base.js';
 export * from '../encoding/p50k_base.js';

package/cjs/model/code-cushman-002.js

@@ -13,6 +13,13 @@ var __createBinding = (this && this.__createBinding) || (Object.create ? (functi
 var __exportStar = (this && this.__exportStar) || function(m, exports) {
     for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
 };
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = void 0;
+// eslint-disable-next-line no-restricted-exports, import/no-default-export
+var p50k_base_js_1 = require("../encoding/p50k_base.js");
+Object.defineProperty(exports, "default", { enumerable: true, get: function () { return __importDefault(p50k_base_js_1).default; } });
 __exportStar(require("../encoding/p50k_base.js"), exports);
 //# sourceMappingURL=code-cushman-002.js.map

package/cjs/model/code-cushman-002.js.map

@@ -1 +1 @@
-{"version":3,"file":"code-cushman-002.js","sourceRoot":"","sources":["../../src/model/code-cushman-002.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,2DAAwC"}
+{"version":3,"file":"code-cushman-002.js","sourceRoot":"","sources":["../../src/model/code-cushman-002.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA,2EAA2E;AAC3E,yDAAkD;AAAzC,wHAAA,OAAO,OAAA;AAChB,2DAAwC"}

package/cjs/model/code-davinci-001.d.ts

@@ -1 +1,2 @@
+export { default } from '../encoding/p50k_base.js';
 export * from '../encoding/p50k_base.js';

package/cjs/model/code-davinci-001.js

@@ -13,6 +13,13 @@ var __createBinding = (this && this.__createBinding) || (Object.create ? (functi
 var __exportStar = (this && this.__exportStar) || function(m, exports) {
     for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
 };
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = void 0;
+// eslint-disable-next-line no-restricted-exports, import/no-default-export
+var p50k_base_js_1 = require("../encoding/p50k_base.js");
+Object.defineProperty(exports, "default", { enumerable: true, get: function () { return __importDefault(p50k_base_js_1).default; } });
 __exportStar(require("../encoding/p50k_base.js"), exports);
 //# sourceMappingURL=code-davinci-001.js.map

package/cjs/model/code-davinci-001.js.map

@@ -1 +1 @@
-{"version":3,"file":"code-davinci-001.js","sourceRoot":"","sources":["../../src/model/code-davinci-001.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,2DAAwC"}
+{"version":3,"file":"code-davinci-001.js","sourceRoot":"","sources":["../../src/model/code-davinci-001.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA,2EAA2E;AAC3E,yDAAkD;AAAzC,wHAAA,OAAO,OAAA;AAChB,2DAAwC"}

package/cjs/model/code-davinci-002.d.ts

@@ -1 +1,2 @@
+export { default } from '../encoding/p50k_base.js';
 export * from '../encoding/p50k_base.js';

package/cjs/model/code-davinci-002.js

@@ -13,6 +13,13 @@ var __createBinding = (this && this.__createBinding) || (Object.create ? (functi
 var __exportStar = (this && this.__exportStar) || function(m, exports) {
     for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
 };
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = void 0;
+// eslint-disable-next-line no-restricted-exports, import/no-default-export
+var p50k_base_js_1 = require("../encoding/p50k_base.js");
+Object.defineProperty(exports, "default", { enumerable: true, get: function () { return __importDefault(p50k_base_js_1).default; } });
 __exportStar(require("../encoding/p50k_base.js"), exports);
 //# sourceMappingURL=code-davinci-002.js.map

package/cjs/model/code-davinci-002.js.map

@@ -1 +1 @@
-{"version":3,"file":"code-davinci-002.js","sourceRoot":"","sources":["../../src/model/code-davinci-002.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,2DAAwC"}
+{"version":3,"file":"code-davinci-002.js","sourceRoot":"","sources":["../../src/model/code-davinci-002.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA,2EAA2E;AAC3E,yDAAkD;AAAzC,wHAAA,OAAO,OAAA;AAChB,2DAAwC"}

package/cjs/model/code-davinci-edit-001.d.ts

@@ -1 +1,2 @@
+export { default } from '../encoding/p50k_edit.js';
 export * from '../encoding/p50k_edit.js';

package/cjs/model/code-davinci-edit-001.js

@@ -13,6 +13,13 @@ var __createBinding = (this && this.__createBinding) || (Object.create ? (functi
 var __exportStar = (this && this.__exportStar) || function(m, exports) {
     for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
 };
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = void 0;
+// eslint-disable-next-line no-restricted-exports, import/no-default-export
+var p50k_edit_js_1 = require("../encoding/p50k_edit.js");
+Object.defineProperty(exports, "default", { enumerable: true, get: function () { return __importDefault(p50k_edit_js_1).default; } });
 __exportStar(require("../encoding/p50k_edit.js"), exports);
 //# sourceMappingURL=code-davinci-edit-001.js.map

package/cjs/model/code-davinci-edit-001.js.map

@@ -1 +1 @@
-{"version":3,"file":"code-davinci-edit-001.js","sourceRoot":"","sources":["../../src/model/code-davinci-edit-001.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,2DAAwC"}
+{"version":3,"file":"code-davinci-edit-001.js","sourceRoot":"","sources":["../../src/model/code-davinci-edit-001.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA,2EAA2E;AAC3E,yDAAkD;AAAzC,wHAAA,OAAO,OAAA;AAChB,2DAAwC"}

package/cjs/model/code-search-ada-code-001.d.ts

@@ -1 +1,2 @@
+export { default } from '../encoding/r50k_base.js';
 export * from '../encoding/r50k_base.js';

package/cjs/model/code-search-ada-code-001.js

@@ -13,6 +13,13 @@ var __createBinding = (this && this.__createBinding) || (Object.create ? (functi
 var __exportStar = (this && this.__exportStar) || function(m, exports) {
     for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
 };
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = void 0;
+// eslint-disable-next-line no-restricted-exports, import/no-default-export
+var r50k_base_js_1 = require("../encoding/r50k_base.js");
+Object.defineProperty(exports, "default", { enumerable: true, get: function () { return __importDefault(r50k_base_js_1).default; } });
 __exportStar(require("../encoding/r50k_base.js"), exports);
 //# sourceMappingURL=code-search-ada-code-001.js.map

package/cjs/model/code-search-ada-code-001.js.map

@@ -1 +1 @@
-{"version":3,"file":"code-search-ada-code-001.js","sourceRoot":"","sources":["../../src/model/code-search-ada-code-001.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,2DAAwC"}
+{"version":3,"file":"code-search-ada-code-001.js","sourceRoot":"","sources":["../../src/model/code-search-ada-code-001.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA,2EAA2E;AAC3E,yDAAkD;AAAzC,wHAAA,OAAO,OAAA;AAChB,2DAAwC"}

package/cjs/model/code-search-babbage-code-001.d.ts

@@ -1 +1,2 @@
+export { default } from '../encoding/r50k_base.js';
 export * from '../encoding/r50k_base.js';

package/cjs/model/code-search-babbage-code-001.js

@@ -13,6 +13,13 @@ var __createBinding = (this && this.__createBinding) || (Object.create ? (functi
 var __exportStar = (this && this.__exportStar) || function(m, exports) {
     for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
 };
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = void 0;
+// eslint-disable-next-line no-restricted-exports, import/no-default-export
+var r50k_base_js_1 = require("../encoding/r50k_base.js");
+Object.defineProperty(exports, "default", { enumerable: true, get: function () { return __importDefault(r50k_base_js_1).default; } });
 __exportStar(require("../encoding/r50k_base.js"), exports);
 //# sourceMappingURL=code-search-babbage-code-001.js.map

package/cjs/model/code-search-babbage-code-001.js.map

@@ -1 +1 @@
-{"version":3,"file":"code-search-babbage-code-001.js","sourceRoot":"","sources":["../../src/model/code-search-babbage-code-001.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,2DAAwC"}
+{"version":3,"file":"code-search-babbage-code-001.js","sourceRoot":"","sources":["../../src/model/code-search-babbage-code-001.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA,2EAA2E;AAC3E,yDAAkD;AAAzC,wHAAA,OAAO,OAAA;AAChB,2DAAwC"}

package/cjs/model/curie.d.ts

@@ -1 +1,2 @@
+export { default } from '../encoding/r50k_base.js';
 export * from '../encoding/r50k_base.js';

package/cjs/model/curie.js

@@ -13,6 +13,13 @@ var __createBinding = (this && this.__createBinding) || (Object.create ? (functi
 var __exportStar = (this && this.__exportStar) || function(m, exports) {
     for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
 };
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = void 0;
+// eslint-disable-next-line no-restricted-exports, import/no-default-export
+var r50k_base_js_1 = require("../encoding/r50k_base.js");
+Object.defineProperty(exports, "default", { enumerable: true, get: function () { return __importDefault(r50k_base_js_1).default; } });
 __exportStar(require("../encoding/r50k_base.js"), exports);
 //# sourceMappingURL=curie.js.map

package/cjs/model/curie.js.map

@@ -1 +1 @@
-{"version":3,"file":"curie.js","sourceRoot":"","sources":["../../src/model/curie.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,2DAAwC"}
+{"version":3,"file":"curie.js","sourceRoot":"","sources":["../../src/model/curie.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA,2EAA2E;AAC3E,yDAAkD;AAAzC,wHAAA,OAAO,OAAA;AAChB,2DAAwC"}

package/cjs/model/cushman-codex.d.ts

@@ -1 +1,2 @@
+export { default } from '../encoding/p50k_base.js';
 export * from '../encoding/p50k_base.js';

package/cjs/model/cushman-codex.js

@@ -13,6 +13,13 @@ var __createBinding = (this && this.__createBinding) || (Object.create ? (functi
 var __exportStar = (this && this.__exportStar) || function(m, exports) {
     for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
 };
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = void 0;
+// eslint-disable-next-line no-restricted-exports, import/no-default-export
+var p50k_base_js_1 = require("../encoding/p50k_base.js");
+Object.defineProperty(exports, "default", { enumerable: true, get: function () { return __importDefault(p50k_base_js_1).default; } });
 __exportStar(require("../encoding/p50k_base.js"), exports);
 //# sourceMappingURL=cushman-codex.js.map

package/cjs/model/cushman-codex.js.map

@@ -1 +1 @@
-{"version":3,"file":"cushman-codex.js","sourceRoot":"","sources":["../../src/model/cushman-codex.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,2DAAwC"}
+{"version":3,"file":"cushman-codex.js","sourceRoot":"","sources":["../../src/model/cushman-codex.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA,2EAA2E;AAC3E,yDAAkD;AAAzC,wHAAA,OAAO,OAAA;AAChB,2DAAwC"}

package/cjs/model/davinci-codex.d.ts

@@ -1 +1,2 @@
+export { default } from '../encoding/p50k_base.js';
 export * from '../encoding/p50k_base.js';

package/cjs/model/davinci-codex.js

@@ -13,6 +13,13 @@ var __createBinding = (this && this.__createBinding) || (Object.create ? (functi
 var __exportStar = (this && this.__exportStar) || function(m, exports) {
     for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
 };
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = void 0;
+// eslint-disable-next-line no-restricted-exports, import/no-default-export
+var p50k_base_js_1 = require("../encoding/p50k_base.js");
+Object.defineProperty(exports, "default", { enumerable: true, get: function () { return __importDefault(p50k_base_js_1).default; } });
 __exportStar(require("../encoding/p50k_base.js"), exports);
 //# sourceMappingURL=davinci-codex.js.map

package/cjs/model/davinci-codex.js.map

@@ -1 +1 @@
-{"version":3,"file":"davinci-codex.js","sourceRoot":"","sources":["../../src/model/davinci-codex.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,2DAAwC"}
+{"version":3,"file":"davinci-codex.js","sourceRoot":"","sources":["../../src/model/davinci-codex.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA,2EAA2E;AAC3E,yDAAkD;AAAzC,wHAAA,OAAO,OAAA;AAChB,2DAAwC"}

package/cjs/model/davinci.d.ts

@@ -1 +1,2 @@
+export { default } from '../encoding/r50k_base.js';
 export * from '../encoding/r50k_base.js';

package/cjs/model/davinci.js

@@ -13,6 +13,13 @@ var __createBinding = (this && this.__createBinding) || (Object.create ? (functi
 var __exportStar = (this && this.__exportStar) || function(m, exports) {
     for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
 };
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = void 0;
+// eslint-disable-next-line no-restricted-exports, import/no-default-export
+var r50k_base_js_1 = require("../encoding/r50k_base.js");
+Object.defineProperty(exports, "default", { enumerable: true, get: function () { return __importDefault(r50k_base_js_1).default; } });
 __exportStar(require("../encoding/r50k_base.js"), exports);
 //# sourceMappingURL=davinci.js.map

package/cjs/model/davinci.js.map

@@ -1 +1 @@
-{"version":3,"file":"davinci.js","sourceRoot":"","sources":["../../src/model/davinci.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,2DAAwC"}
+{"version":3,"file":"davinci.js","sourceRoot":"","sources":["../../src/model/davinci.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA,2EAA2E;AAC3E,yDAAkD;AAAzC,wHAAA,OAAO,OAAA;AAChB,2DAAwC"}

package/cjs/model/gpt-3.5-turbo-0301.d.ts

@@ -0,0 +1,6 @@
+import { GptEncoding } from '../GptEncoding.js';
+export * from '../specialTokens.js';
+declare const api: GptEncoding;
+declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void, unknown>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, unknown>, encode: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, { allowedSpecial, disallowedSpecial, }?: import("../GptEncoding.js").EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, tokenLimit: number) => number | false, encodeChat: (chat: readonly import("../GptEncoding.js").ChatMessage[], model?: "gpt-4" | "gpt-4-32k" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo" | "gpt-3.5-turbo-0301" | "text-davinci-003" | "text-davinci-002" | "text-davinci-001" | "text-curie-001" | "text-babbage-001" | "text-ada-001" | "davinci" | "curie" | "babbage" | "ada" | "code-davinci-002" | "code-davinci-001" | "code-cushman-002" | "code-cushman-001" | "davinci-codex" | "cushman-codex" | "text-davinci-edit-001" | "code-davinci-edit-001" | "text-embedding-ada-002" | "text-similarity-davinci-001" | "text-similarity-curie-001" | "text-similarity-babbage-001" | "text-similarity-ada-001" | "text-search-davinci-doc-001" | "text-search-curie-doc-001" | "text-search-babbage-doc-001" | "text-search-ada-doc-001" | "code-search-babbage-code-001" | "code-search-ada-code-001" | undefined) => number[], encodeChatGenerator: (chat: Iterable<import("../GptEncoding.js").ChatMessage>, model?: "gpt-4" | "gpt-4-32k" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo" | "gpt-3.5-turbo-0301" | "text-davinci-003" | "text-davinci-002" | "text-davinci-001" | "text-curie-001" | "text-babbage-001" | "text-ada-001" | "davinci" | "curie" | "babbage" | "ada" | "code-davinci-002" | "code-davinci-001" | "code-cushman-002" | "code-cushman-001" | "davinci-codex" | "cushman-codex" | "text-davinci-edit-001" | "code-davinci-edit-001" | "text-embedding-ada-002" | "text-similarity-davinci-001" | "text-similarity-curie-001" | "text-similarity-babbage-001" | "text-similarity-ada-001" | "text-search-davinci-doc-001" | "text-search-curie-doc-001" | "text-search-babbage-doc-001" | "text-search-ada-doc-001" | "code-search-babbage-code-001" | "code-search-ada-code-001" | undefined) => Generator<number[], void, undefined>;
+export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, };
+export default api;

package/cjs/model/gpt-3.5-turbo-0301.js

@@ -0,0 +1,38 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.isWithinTokenLimit = exports.encodeGenerator = exports.encodeChatGenerator = exports.encodeChat = exports.encode = exports.decodeGenerator = exports.decodeAsyncGenerator = exports.decode = void 0;
+/* eslint-disable import/extensions */
+const convertTokenBytePairEncodingFromTuples_js_1 = require("../convertTokenBytePairEncodingFromTuples.js");
+const cl100k_base_js_1 = __importDefault(require("../encodings/cl100k_base.js"));
+const GptEncoding_js_1 = require("../GptEncoding.js");
+__exportStar(require("../specialTokens.js"), exports);
+const api = GptEncoding_js_1.GptEncoding.getEncodingApiForModel('gpt-3.5-turbo-0301', () => (0, convertTokenBytePairEncodingFromTuples_js_1.convertTokenBytePairEncodingFromTuples)(cl100k_base_js_1.default));
+const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, encodeChat, encodeChatGenerator, } = api;
+exports.decode = decode;
+exports.decodeAsyncGenerator = decodeAsyncGenerator;
+exports.decodeGenerator = decodeGenerator;
+exports.encode = encode;
+exports.encodeGenerator = encodeGenerator;
+exports.isWithinTokenLimit = isWithinTokenLimit;
+exports.encodeChat = encodeChat;
+exports.encodeChatGenerator = encodeChatGenerator;
+// eslint-disable-next-line import/no-default-export
+exports.default = api;
+//# sourceMappingURL=gpt-3.5-turbo-0301.js.map

package/cjs/model/gpt-3.5-turbo-0301.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"gpt-3.5-turbo-0301.js","sourceRoot":"","sources":["../../src/model/gpt-3.5-turbo-0301.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA,sCAAsC;AACtC,4GAAqG;AACrG,iFAAiD;AACjD,sDAA+C;AAE/C,sDAAmC;AAEnC,MAAM,GAAG,GAAG,4BAAW,CAAC,sBAAsB,CAAC,oBAAoB,EAAE,GAAG,EAAE,CACxE,IAAA,kFAAsC,EAAC,wBAAO,CAAC,CAChD,CAAA;AACD,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,EAClB,UAAU,EACV,mBAAmB,GACpB,GAAG,GAAG,CAAA;AAEL,wBAAM;AACN,oDAAoB;AACpB,0CAAe;AACf,wBAAM;AAGN,0CAAe;AACf,gDAAkB;AAHlB,gCAAU;AACV,kDAAmB;AAIrB,oDAAoD;AACpD,kBAAe,GAAG,CAAA"}

package/cjs/model/gpt-3.5-turbo.d.ts

@@ -1 +1,6 @@
-export * from '../encoding/cl100k_base.js';
+import { GptEncoding } from '../GptEncoding.js';
+export * from '../specialTokens.js';
+declare const api: GptEncoding;
+declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void, unknown>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, unknown>, encode: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, { allowedSpecial, disallowedSpecial, }?: import("../GptEncoding.js").EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, tokenLimit: number) => number | false, encodeChat: (chat: readonly import("../GptEncoding.js").ChatMessage[], model?: "gpt-4" | "gpt-4-32k" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo" | "gpt-3.5-turbo-0301" | "text-davinci-003" | "text-davinci-002" | "text-davinci-001" | "text-curie-001" | "text-babbage-001" | "text-ada-001" | "davinci" | "curie" | "babbage" | "ada" | "code-davinci-002" | "code-davinci-001" | "code-cushman-002" | "code-cushman-001" | "davinci-codex" | "cushman-codex" | "text-davinci-edit-001" | "code-davinci-edit-001" | "text-embedding-ada-002" | "text-similarity-davinci-001" | "text-similarity-curie-001" | "text-similarity-babbage-001" | "text-similarity-ada-001" | "text-search-davinci-doc-001" | "text-search-curie-doc-001" | "text-search-babbage-doc-001" | "text-search-ada-doc-001" | "code-search-babbage-code-001" | "code-search-ada-code-001" | undefined) => number[], encodeChatGenerator: (chat: Iterable<import("../GptEncoding.js").ChatMessage>, model?: "gpt-4" | "gpt-4-32k" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo" | "gpt-3.5-turbo-0301" | "text-davinci-003" | "text-davinci-002" | "text-davinci-001" | "text-curie-001" | "text-babbage-001" | "text-ada-001" | "davinci" | "curie" | "babbage" | "ada" | "code-davinci-002" | "code-davinci-001" | "code-cushman-002" | "code-cushman-001" | "davinci-codex" | "cushman-codex" | "text-davinci-edit-001" | "code-davinci-edit-001" | "text-embedding-ada-002" | "text-similarity-davinci-001" | "text-similarity-curie-001" | "text-similarity-babbage-001" | "text-similarity-ada-001" | "text-search-davinci-doc-001" | "text-search-curie-doc-001" | "text-search-babbage-doc-001" | "text-search-ada-doc-001" | "code-search-babbage-code-001" | "code-search-ada-code-001" | undefined) => Generator<number[], void, undefined>;
+export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, };
+export default api;

package/cjs/model/gpt-3.5-turbo.js

@@ -13,6 +13,26 @@ var __createBinding = (this && this.__createBinding) || (Object.create ? (functi
 var __exportStar = (this && this.__exportStar) || function(m, exports) {
     for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
 };
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
-__exportStar(require("../encoding/cl100k_base.js"), exports);
+exports.isWithinTokenLimit = exports.encodeGenerator = exports.encodeChatGenerator = exports.encodeChat = exports.encode = exports.decodeGenerator = exports.decodeAsyncGenerator = exports.decode = void 0;
+/* eslint-disable import/extensions */
+const convertTokenBytePairEncodingFromTuples_js_1 = require("../convertTokenBytePairEncodingFromTuples.js");
+const cl100k_base_js_1 = __importDefault(require("../encodings/cl100k_base.js"));
+const GptEncoding_js_1 = require("../GptEncoding.js");
+__exportStar(require("../specialTokens.js"), exports);
+const api = GptEncoding_js_1.GptEncoding.getEncodingApiForModel('gpt-3.5-turbo', () => (0, convertTokenBytePairEncodingFromTuples_js_1.convertTokenBytePairEncodingFromTuples)(cl100k_base_js_1.default));
+const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, encodeChat, encodeChatGenerator, } = api;
+exports.decode = decode;
+exports.decodeAsyncGenerator = decodeAsyncGenerator;
+exports.decodeGenerator = decodeGenerator;
+exports.encode = encode;
+exports.encodeGenerator = encodeGenerator;
+exports.isWithinTokenLimit = isWithinTokenLimit;
+exports.encodeChat = encodeChat;
+exports.encodeChatGenerator = encodeChatGenerator;
+// eslint-disable-next-line import/no-default-export
+exports.default = api;
 //# sourceMappingURL=gpt-3.5-turbo.js.map

package/cjs/model/gpt-3.5-turbo.js.map

@@ -1 +1 @@
-{"version":3,"file":"gpt-3.5-turbo.js","sourceRoot":"","sources":["../../src/model/gpt-3.5-turbo.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,6DAA0C"}
+{"version":3,"file":"gpt-3.5-turbo.js","sourceRoot":"","sources":["../../src/model/gpt-3.5-turbo.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA,sCAAsC;AACtC,4GAAqG;AACrG,iFAAiD;AACjD,sDAA+C;AAE/C,sDAAmC;AAEnC,MAAM,GAAG,GAAG,4BAAW,CAAC,sBAAsB,CAAC,eAAe,EAAE,GAAG,EAAE,CACnE,IAAA,kFAAsC,EAAC,wBAAO,CAAC,CAChD,CAAA;AACD,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,EAClB,UAAU,EACV,mBAAmB,GACpB,GAAG,GAAG,CAAA;AAEL,wBAAM;AACN,oDAAoB;AACpB,0CAAe;AACf,wBAAM;AAGN,0CAAe;AACf,gDAAkB;AAHlB,gCAAU;AACV,kDAAmB;AAIrB,oDAAoD;AACpD,kBAAe,GAAG,CAAA"}

package/cjs/model/gpt-4-0314.d.ts

@@ -0,0 +1,6 @@
+import { GptEncoding } from '../GptEncoding.js';
+export * from '../specialTokens.js';
+declare const api: GptEncoding;
+declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void, unknown>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, unknown>, encode: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, { allowedSpecial, disallowedSpecial, }?: import("../GptEncoding.js").EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, tokenLimit: number) => number | false, encodeChat: (chat: readonly import("../GptEncoding.js").ChatMessage[], model?: "gpt-4" | "gpt-4-32k" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo" | "gpt-3.5-turbo-0301" | "text-davinci-003" | "text-davinci-002" | "text-davinci-001" | "text-curie-001" | "text-babbage-001" | "text-ada-001" | "davinci" | "curie" | "babbage" | "ada" | "code-davinci-002" | "code-davinci-001" | "code-cushman-002" | "code-cushman-001" | "davinci-codex" | "cushman-codex" | "text-davinci-edit-001" | "code-davinci-edit-001" | "text-embedding-ada-002" | "text-similarity-davinci-001" | "text-similarity-curie-001" | "text-similarity-babbage-001" | "text-similarity-ada-001" | "text-search-davinci-doc-001" | "text-search-curie-doc-001" | "text-search-babbage-doc-001" | "text-search-ada-doc-001" | "code-search-babbage-code-001" | "code-search-ada-code-001" | undefined) => number[], encodeChatGenerator: (chat: Iterable<import("../GptEncoding.js").ChatMessage>, model?: "gpt-4" | "gpt-4-32k" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo" | "gpt-3.5-turbo-0301" | "text-davinci-003" | "text-davinci-002" | "text-davinci-001" | "text-curie-001" | "text-babbage-001" | "text-ada-001" | "davinci" | "curie" | "babbage" | "ada" | "code-davinci-002" | "code-davinci-001" | "code-cushman-002" | "code-cushman-001" | "davinci-codex" | "cushman-codex" | "text-davinci-edit-001" | "code-davinci-edit-001" | "text-embedding-ada-002" | "text-similarity-davinci-001" | "text-similarity-curie-001" | "text-similarity-babbage-001" | "text-similarity-ada-001" | "text-search-davinci-doc-001" | "text-search-curie-doc-001" | "text-search-babbage-doc-001" | "text-search-ada-doc-001" | "code-search-babbage-code-001" | "code-search-ada-code-001" | undefined) => Generator<number[], void, undefined>;
+export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, };
+export default api;

package/cjs/model/gpt-4-0314.js

@@ -0,0 +1,38 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.isWithinTokenLimit = exports.encodeGenerator = exports.encodeChatGenerator = exports.encodeChat = exports.encode = exports.decodeGenerator = exports.decodeAsyncGenerator = exports.decode = void 0;
+/* eslint-disable import/extensions */
+const convertTokenBytePairEncodingFromTuples_js_1 = require("../convertTokenBytePairEncodingFromTuples.js");
+const cl100k_base_js_1 = __importDefault(require("../encodings/cl100k_base.js"));
+const GptEncoding_js_1 = require("../GptEncoding.js");
+__exportStar(require("../specialTokens.js"), exports);
+const api = GptEncoding_js_1.GptEncoding.getEncodingApiForModel('gpt-4-0314', () => (0, convertTokenBytePairEncodingFromTuples_js_1.convertTokenBytePairEncodingFromTuples)(cl100k_base_js_1.default));
+const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, encodeChat, encodeChatGenerator, } = api;
+exports.decode = decode;
+exports.decodeAsyncGenerator = decodeAsyncGenerator;
+exports.decodeGenerator = decodeGenerator;
+exports.encode = encode;
+exports.encodeGenerator = encodeGenerator;
+exports.isWithinTokenLimit = isWithinTokenLimit;
+exports.encodeChat = encodeChat;
+exports.encodeChatGenerator = encodeChatGenerator;
+// eslint-disable-next-line import/no-default-export
+exports.default = api;
+//# sourceMappingURL=gpt-4-0314.js.map

package/cjs/model/gpt-4-0314.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"gpt-4-0314.js","sourceRoot":"","sources":["../../src/model/gpt-4-0314.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA,sCAAsC;AACtC,4GAAqG;AACrG,iFAAiD;AACjD,sDAA+C;AAE/C,sDAAmC;AAEnC,MAAM,GAAG,GAAG,4BAAW,CAAC,sBAAsB,CAAC,YAAY,EAAE,GAAG,EAAE,CAChE,IAAA,kFAAsC,EAAC,wBAAO,CAAC,CAChD,CAAA;AACD,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,EAClB,UAAU,EACV,mBAAmB,GACpB,GAAG,GAAG,CAAA;AAEL,wBAAM;AACN,oDAAoB;AACpB,0CAAe;AACf,wBAAM;AAGN,0CAAe;AACf,gDAAkB;AAHlB,gCAAU;AACV,kDAAmB;AAIrB,oDAAoD;AACpD,kBAAe,GAAG,CAAA"}