gpt-tokenizer 2.0.0 → 2.1.1

package/esm/mapping.js

@@ -1,4 +1,5 @@
 /* eslint-disable camelcase */
+import { ImSep } from './specialTokens.js';
 export const cl100k_base = 'cl100k_base';
 export const p50k_base = 'p50k_base';
 export const p50k_edit = 'p50k_edit';
@@ -12,7 +13,11 @@ export const encodingNames = [
 export const modelToEncodingMap = {
     // chat
     'gpt-4': cl100k_base,
+    'gpt-4-32k': cl100k_base,
+    'gpt-4-0314': cl100k_base,
+    'gpt-4-32k-0314': cl100k_base,
     'gpt-3.5-turbo': cl100k_base,
+    'gpt-3.5-turbo-0301': cl100k_base,
     // text
     'text-davinci-003': p50k_base,
     'text-davinci-002': p50k_base,
@@ -48,4 +53,31 @@ export const modelToEncodingMap = {
     'code-search-babbage-code-001': r50k_base,
     'code-search-ada-code-001': r50k_base,
 };
+const internalChatModelParams = {
+    'gpt-3.5-turbo': {
+        messageSeparator: '\n',
+        roleSeparator: '\n',
+    },
+    'gpt-3.5-turbo-0301': {
+        messageSeparator: '\n',
+        roleSeparator: '\n',
+    },
+    'gpt-4': {
+        messageSeparator: '',
+        roleSeparator: ImSep,
+    },
+    'gpt-4-0314': {
+        messageSeparator: '',
+        roleSeparator: ImSep,
+    },
+    'gpt-4-32k': {
+        messageSeparator: '',
+        roleSeparator: ImSep,
+    },
+    'gpt-4-32k-0314': {
+        messageSeparator: '',
+        roleSeparator: ImSep,
+    },
+};
+export const chatModelParams = internalChatModelParams;
 //# sourceMappingURL=mapping.js.map

package/esm/mapping.js.map

@@ -1 +1 @@
-{"version":3,"file":"mapping.js","sourceRoot":"","sources":["../src/mapping.ts"],"names":[],"mappings":"AAAA,8BAA8B;AAC9B,MAAM,CAAC,MAAM,WAAW,GAAG,aAAa,CAAA;AACxC,MAAM,CAAC,MAAM,SAAS,GAAG,WAAW,CAAA;AACpC,MAAM,CAAC,MAAM,SAAS,GAAG,WAAW,CAAA;AACpC,MAAM,CAAC,MAAM,SAAS,GAAG,WAAW,CAAA;AAEpC,MAAM,CAAC,MAAM,aAAa,GAAG;IAC3B,WAAW;IACX,SAAS;IACT,SAAS;IACT,SAAS;CACD,CAAA;AAEV,MAAM,CAAC,MAAM,kBAAkB,GAAG;IAChC,OAAO;IACP,OAAO,EAAE,WAAW;IACpB,eAAe,EAAE,WAAW;IAC5B,OAAO;IACP,kBAAkB,EAAE,SAAS;IAC7B,kBAAkB,EAAE,SAAS;IAC7B,kBAAkB,EAAE,SAAS;IAC7B,gBAAgB,EAAE,SAAS;IAC3B,kBAAkB,EAAE,SAAS;IAC7B,cAAc,EAAE,SAAS;IACzB,OAAO,EAAE,SAAS;IAClB,KAAK,EAAE,SAAS;IAChB,OAAO,EAAE,SAAS;IAClB,GAAG,EAAE,SAAS;IACd,OAAO;IACP,kBAAkB,EAAE,SAAS;IAC7B,kBAAkB,EAAE,SAAS;IAC7B,kBAAkB,EAAE,SAAS;IAC7B,kBAAkB,EAAE,SAAS;IAC7B,eAAe,EAAE,SAAS;IAC1B,eAAe,EAAE,SAAS;IAC1B,OAAO;IACP,uBAAuB,EAAE,SAAS;IAClC,uBAAuB,EAAE,SAAS;IAClC,aAAa;IACb,wBAAwB,EAAE,WAAW;IACrC,iBAAiB;IACjB,6BAA6B,EAAE,SAAS;IACxC,2BAA2B,EAAE,SAAS;IACtC,6BAA6B,EAAE,SAAS;IACxC,yBAAyB,EAAE,SAAS;IACpC,6BAA6B,EAAE,SAAS;IACxC,2BAA2B,EAAE,SAAS;IACtC,6BAA6B,EAAE,SAAS;IACxC,yBAAyB,EAAE,SAAS;IACpC,8BAA8B,EAAE,SAAS;IACzC,0BAA0B,EAAE,SAAS;CAC7B,CAAA"}
+{"version":3,"file":"mapping.js","sourceRoot":"","sources":["../src/mapping.ts"],"names":[],"mappings":"AAAA,8BAA8B;AAE9B,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAA;AAE1C,MAAM,CAAC,MAAM,WAAW,GAAG,aAAa,CAAA;AACxC,MAAM,CAAC,MAAM,SAAS,GAAG,WAAW,CAAA;AACpC,MAAM,CAAC,MAAM,SAAS,GAAG,WAAW,CAAA;AACpC,MAAM,CAAC,MAAM,SAAS,GAAG,WAAW,CAAA;AAEpC,MAAM,CAAC,MAAM,aAAa,GAAG;IAC3B,WAAW;IACX,SAAS;IACT,SAAS;IACT,SAAS;CACD,CAAA;AAEV,MAAM,CAAC,MAAM,kBAAkB,GAAG;IAChC,OAAO;IACP,OAAO,EAAE,WAAW;IACpB,WAAW,EAAE,WAAW;IACxB,YAAY,EAAE,WAAW;IACzB,gBAAgB,EAAE,WAAW;IAC7B,eAAe,EAAE,WAAW;IAC5B,oBAAoB,EAAE,WAAW;IACjC,OAAO;IACP,kBAAkB,EAAE,SAAS;IAC7B,kBAAkB,EAAE,SAAS;IAC7B,kBAAkB,EAAE,SAAS;IAC7B,gBAAgB,EAAE,SAAS;IAC3B,kBAAkB,EAAE,SAAS;IAC7B,cAAc,EAAE,SAAS;IACzB,OAAO,EAAE,SAAS;IAClB,KAAK,EAAE,SAAS;IAChB,OAAO,EAAE,SAAS;IAClB,GAAG,EAAE,SAAS;IACd,OAAO;IACP,kBAAkB,EAAE,SAAS;IAC7B,kBAAkB,EAAE,SAAS;IAC7B,kBAAkB,EAAE,SAAS;IAC7B,kBAAkB,EAAE,SAAS;IAC7B,eAAe,EAAE,SAAS;IAC1B,eAAe,EAAE,SAAS;IAC1B,OAAO;IACP,uBAAuB,EAAE,SAAS;IAClC,uBAAuB,EAAE,SAAS;IAClC,aAAa;IACb,wBAAwB,EAAE,WAAW;IACrC,iBAAiB;IACjB,6BAA6B,EAAE,SAAS;IACxC,2BAA2B,EAAE,SAAS;IACtC,6BAA6B,EAAE,SAAS;IACxC,yBAAyB,EAAE,SAAS;IACpC,6BAA6B,EAAE,SAAS;IACxC,2BAA2B,EAAE,SAAS;IACtC,6BAA6B,EAAE,SAAS;IACxC,yBAAyB,EAAE,SAAS;IACpC,8BAA8B,EAAE,SAAS;IACzC,0BAA0B,EAAE,SAAS;CAC7B,CAAA;AAOV,MAAM,uBAAuB,GAAG;IAC9B,eAAe,EAAE;QACf,gBAAgB,EAAE,IAAI;QACtB,aAAa,EAAE,IAAI;KACpB;IACD,oBAAoB,EAAE;QACpB,gBAAgB,EAAE,IAAI;QACtB,aAAa,EAAE,IAAI;KACpB;IACD,OAAO,EAAE;QACP,gBAAgB,EAAE,EAAE;QACpB,aAAa,EAAE,KAAK;KACrB;IACD,YAAY,EAAE;QACZ,gBAAgB,EAAE,EAAE;QACpB,aAAa,EAAE,KAAK;KACrB;IACD,WAAW,EAAE;QACX,gBAAgB,EAAE,EAAE;QACpB,aAAa,EAAE,KAAK;KACrB;IACD,gBAAgB,EAAE;QAChB,gBAAgB,EAAE,EAAE;QACpB,aAAa,EAAE,KAAK;KACrB;CACF,CAAA;AAED,MAAM,CAAC,MAAM,eAAe,GAC1B,uBAAuB,CAAA"}

package/esm/model/ada.d.ts

@@ -1 +1,2 @@
+export { default } from '../encoding/r50k_base.js';
 export * from '../encoding/r50k_base.js';

package/esm/model/ada.js

@@ -1,2 +1,4 @@
+// eslint-disable-next-line no-restricted-exports, import/no-default-export
+export { default } from '../encoding/r50k_base.js';
 export * from '../encoding/r50k_base.js';
 //# sourceMappingURL=ada.js.map

package/esm/model/ada.js.map

@@ -1 +1 @@
-{"version":3,"file":"ada.js","sourceRoot":"","sources":["../../src/model/ada.ts"],"names":[],"mappings":"AAAA,cAAc,0BAA0B,CAAA"}
+{"version":3,"file":"ada.js","sourceRoot":"","sources":["../../src/model/ada.ts"],"names":[],"mappings":"AAAA,2EAA2E;AAC3E,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAA;AAClD,cAAc,0BAA0B,CAAA"}

package/esm/model/babbage.d.ts

@@ -1 +1,2 @@
+export { default } from '../encoding/r50k_base.js';
 export * from '../encoding/r50k_base.js';

package/esm/model/babbage.js

@@ -1,2 +1,4 @@
+// eslint-disable-next-line no-restricted-exports, import/no-default-export
+export { default } from '../encoding/r50k_base.js';
 export * from '../encoding/r50k_base.js';
 //# sourceMappingURL=babbage.js.map

package/esm/model/babbage.js.map

@@ -1 +1 @@
-{"version":3,"file":"babbage.js","sourceRoot":"","sources":["../../src/model/babbage.ts"],"names":[],"mappings":"AAAA,cAAc,0BAA0B,CAAA"}
+{"version":3,"file":"babbage.js","sourceRoot":"","sources":["../../src/model/babbage.ts"],"names":[],"mappings":"AAAA,2EAA2E;AAC3E,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAA;AAClD,cAAc,0BAA0B,CAAA"}

package/esm/model/code-cushman-001.d.ts

@@ -1 +1,2 @@
+export { default } from '../encoding/p50k_base.js';
 export * from '../encoding/p50k_base.js';

package/esm/model/code-cushman-001.js

@@ -1,2 +1,4 @@
+// eslint-disable-next-line no-restricted-exports, import/no-default-export
+export { default } from '../encoding/p50k_base.js';
 export * from '../encoding/p50k_base.js';
 //# sourceMappingURL=code-cushman-001.js.map

package/esm/model/code-cushman-001.js.map

@@ -1 +1 @@
-{"version":3,"file":"code-cushman-001.js","sourceRoot":"","sources":["../../src/model/code-cushman-001.ts"],"names":[],"mappings":"AAAA,cAAc,0BAA0B,CAAA"}
+{"version":3,"file":"code-cushman-001.js","sourceRoot":"","sources":["../../src/model/code-cushman-001.ts"],"names":[],"mappings":"AAAA,2EAA2E;AAC3E,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAA;AAClD,cAAc,0BAA0B,CAAA"}

package/esm/model/code-cushman-002.d.ts

@@ -1 +1,2 @@
+export { default } from '../encoding/p50k_base.js';
 export * from '../encoding/p50k_base.js';

package/esm/model/code-cushman-002.js

@@ -1,2 +1,4 @@
+// eslint-disable-next-line no-restricted-exports, import/no-default-export
+export { default } from '../encoding/p50k_base.js';
 export * from '../encoding/p50k_base.js';
 //# sourceMappingURL=code-cushman-002.js.map

package/esm/model/code-cushman-002.js.map

@@ -1 +1 @@
-{"version":3,"file":"code-cushman-002.js","sourceRoot":"","sources":["../../src/model/code-cushman-002.ts"],"names":[],"mappings":"AAAA,cAAc,0BAA0B,CAAA"}
+{"version":3,"file":"code-cushman-002.js","sourceRoot":"","sources":["../../src/model/code-cushman-002.ts"],"names":[],"mappings":"AAAA,2EAA2E;AAC3E,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAA;AAClD,cAAc,0BAA0B,CAAA"}

package/esm/model/code-davinci-001.d.ts

@@ -1 +1,2 @@
+export { default } from '../encoding/p50k_base.js';
 export * from '../encoding/p50k_base.js';

package/esm/model/code-davinci-001.js

@@ -1,2 +1,4 @@
+// eslint-disable-next-line no-restricted-exports, import/no-default-export
+export { default } from '../encoding/p50k_base.js';
 export * from '../encoding/p50k_base.js';
 //# sourceMappingURL=code-davinci-001.js.map

package/esm/model/code-davinci-001.js.map

@@ -1 +1 @@
-{"version":3,"file":"code-davinci-001.js","sourceRoot":"","sources":["../../src/model/code-davinci-001.ts"],"names":[],"mappings":"AAAA,cAAc,0BAA0B,CAAA"}
+{"version":3,"file":"code-davinci-001.js","sourceRoot":"","sources":["../../src/model/code-davinci-001.ts"],"names":[],"mappings":"AAAA,2EAA2E;AAC3E,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAA;AAClD,cAAc,0BAA0B,CAAA"}

package/esm/model/code-davinci-002.d.ts

@@ -1 +1,2 @@
+export { default } from '../encoding/p50k_base.js';
 export * from '../encoding/p50k_base.js';

package/esm/model/code-davinci-002.js

@@ -1,2 +1,4 @@
+// eslint-disable-next-line no-restricted-exports, import/no-default-export
+export { default } from '../encoding/p50k_base.js';
 export * from '../encoding/p50k_base.js';
 //# sourceMappingURL=code-davinci-002.js.map

package/esm/model/code-davinci-002.js.map

@@ -1 +1 @@
-{"version":3,"file":"code-davinci-002.js","sourceRoot":"","sources":["../../src/model/code-davinci-002.ts"],"names":[],"mappings":"AAAA,cAAc,0BAA0B,CAAA"}
+{"version":3,"file":"code-davinci-002.js","sourceRoot":"","sources":["../../src/model/code-davinci-002.ts"],"names":[],"mappings":"AAAA,2EAA2E;AAC3E,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAA;AAClD,cAAc,0BAA0B,CAAA"}

package/esm/model/code-davinci-edit-001.d.ts

@@ -1 +1,2 @@
+export { default } from '../encoding/p50k_edit.js';
 export * from '../encoding/p50k_edit.js';

package/esm/model/code-davinci-edit-001.js

@@ -1,2 +1,4 @@
+// eslint-disable-next-line no-restricted-exports, import/no-default-export
+export { default } from '../encoding/p50k_edit.js';
 export * from '../encoding/p50k_edit.js';
 //# sourceMappingURL=code-davinci-edit-001.js.map

package/esm/model/code-davinci-edit-001.js.map

@@ -1 +1 @@
-{"version":3,"file":"code-davinci-edit-001.js","sourceRoot":"","sources":["../../src/model/code-davinci-edit-001.ts"],"names":[],"mappings":"AAAA,cAAc,0BAA0B,CAAA"}
+{"version":3,"file":"code-davinci-edit-001.js","sourceRoot":"","sources":["../../src/model/code-davinci-edit-001.ts"],"names":[],"mappings":"AAAA,2EAA2E;AAC3E,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAA;AAClD,cAAc,0BAA0B,CAAA"}

package/esm/model/code-search-ada-code-001.d.ts

@@ -1 +1,2 @@
+export { default } from '../encoding/r50k_base.js';
 export * from '../encoding/r50k_base.js';

package/esm/model/code-search-ada-code-001.js

@@ -1,2 +1,4 @@
+// eslint-disable-next-line no-restricted-exports, import/no-default-export
+export { default } from '../encoding/r50k_base.js';
 export * from '../encoding/r50k_base.js';
 //# sourceMappingURL=code-search-ada-code-001.js.map

package/esm/model/code-search-ada-code-001.js.map

@@ -1 +1 @@
-{"version":3,"file":"code-search-ada-code-001.js","sourceRoot":"","sources":["../../src/model/code-search-ada-code-001.ts"],"names":[],"mappings":"AAAA,cAAc,0BAA0B,CAAA"}
+{"version":3,"file":"code-search-ada-code-001.js","sourceRoot":"","sources":["../../src/model/code-search-ada-code-001.ts"],"names":[],"mappings":"AAAA,2EAA2E;AAC3E,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAA;AAClD,cAAc,0BAA0B,CAAA"}

package/esm/model/code-search-babbage-code-001.d.ts

@@ -1 +1,2 @@
+export { default } from '../encoding/r50k_base.js';
 export * from '../encoding/r50k_base.js';

package/esm/model/code-search-babbage-code-001.js

@@ -1,2 +1,4 @@
+// eslint-disable-next-line no-restricted-exports, import/no-default-export
+export { default } from '../encoding/r50k_base.js';
 export * from '../encoding/r50k_base.js';
 //# sourceMappingURL=code-search-babbage-code-001.js.map

package/esm/model/code-search-babbage-code-001.js.map

@@ -1 +1 @@
-{"version":3,"file":"code-search-babbage-code-001.js","sourceRoot":"","sources":["../../src/model/code-search-babbage-code-001.ts"],"names":[],"mappings":"AAAA,cAAc,0BAA0B,CAAA"}
+{"version":3,"file":"code-search-babbage-code-001.js","sourceRoot":"","sources":["../../src/model/code-search-babbage-code-001.ts"],"names":[],"mappings":"AAAA,2EAA2E;AAC3E,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAA;AAClD,cAAc,0BAA0B,CAAA"}

package/esm/model/curie.d.ts

@@ -1 +1,2 @@
+export { default } from '../encoding/r50k_base.js';
 export * from '../encoding/r50k_base.js';

package/esm/model/curie.js

@@ -1,2 +1,4 @@
+// eslint-disable-next-line no-restricted-exports, import/no-default-export
+export { default } from '../encoding/r50k_base.js';
 export * from '../encoding/r50k_base.js';
 //# sourceMappingURL=curie.js.map

package/esm/model/curie.js.map

@@ -1 +1 @@
-{"version":3,"file":"curie.js","sourceRoot":"","sources":["../../src/model/curie.ts"],"names":[],"mappings":"AAAA,cAAc,0BAA0B,CAAA"}
+{"version":3,"file":"curie.js","sourceRoot":"","sources":["../../src/model/curie.ts"],"names":[],"mappings":"AAAA,2EAA2E;AAC3E,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAA;AAClD,cAAc,0BAA0B,CAAA"}

package/esm/model/cushman-codex.d.ts

@@ -1 +1,2 @@
+export { default } from '../encoding/p50k_base.js';
 export * from '../encoding/p50k_base.js';

package/esm/model/cushman-codex.js

@@ -1,2 +1,4 @@
+// eslint-disable-next-line no-restricted-exports, import/no-default-export
+export { default } from '../encoding/p50k_base.js';
 export * from '../encoding/p50k_base.js';
 //# sourceMappingURL=cushman-codex.js.map

package/esm/model/cushman-codex.js.map

@@ -1 +1 @@
-{"version":3,"file":"cushman-codex.js","sourceRoot":"","sources":["../../src/model/cushman-codex.ts"],"names":[],"mappings":"AAAA,cAAc,0BAA0B,CAAA"}
+{"version":3,"file":"cushman-codex.js","sourceRoot":"","sources":["../../src/model/cushman-codex.ts"],"names":[],"mappings":"AAAA,2EAA2E;AAC3E,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAA;AAClD,cAAc,0BAA0B,CAAA"}

package/esm/model/davinci-codex.d.ts

@@ -1 +1,2 @@
+export { default } from '../encoding/p50k_base.js';
 export * from '../encoding/p50k_base.js';

package/esm/model/davinci-codex.js

@@ -1,2 +1,4 @@
+// eslint-disable-next-line no-restricted-exports, import/no-default-export
+export { default } from '../encoding/p50k_base.js';
 export * from '../encoding/p50k_base.js';
 //# sourceMappingURL=davinci-codex.js.map

package/esm/model/davinci-codex.js.map

@@ -1 +1 @@
-{"version":3,"file":"davinci-codex.js","sourceRoot":"","sources":["../../src/model/davinci-codex.ts"],"names":[],"mappings":"AAAA,cAAc,0BAA0B,CAAA"}
+{"version":3,"file":"davinci-codex.js","sourceRoot":"","sources":["../../src/model/davinci-codex.ts"],"names":[],"mappings":"AAAA,2EAA2E;AAC3E,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAA;AAClD,cAAc,0BAA0B,CAAA"}

package/esm/model/davinci.d.ts

@@ -1 +1,2 @@
+export { default } from '../encoding/r50k_base.js';
 export * from '../encoding/r50k_base.js';

package/esm/model/davinci.js

@@ -1,2 +1,4 @@
+// eslint-disable-next-line no-restricted-exports, import/no-default-export
+export { default } from '../encoding/r50k_base.js';
 export * from '../encoding/r50k_base.js';
 //# sourceMappingURL=davinci.js.map

package/esm/model/davinci.js.map

@@ -1 +1 @@
-{"version":3,"file":"davinci.js","sourceRoot":"","sources":["../../src/model/davinci.ts"],"names":[],"mappings":"AAAA,cAAc,0BAA0B,CAAA"}
+{"version":3,"file":"davinci.js","sourceRoot":"","sources":["../../src/model/davinci.ts"],"names":[],"mappings":"AAAA,2EAA2E;AAC3E,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAA;AAClD,cAAc,0BAA0B,CAAA"}

package/esm/model/gpt-3.5-turbo-0301.d.ts

@@ -0,0 +1,6 @@
+import { GptEncoding } from '../GptEncoding.js';
+export * from '../specialTokens.js';
+declare const api: GptEncoding;
+declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void, unknown>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, unknown>, encode: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, { allowedSpecial, disallowedSpecial, }?: import("../GptEncoding.js").EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, tokenLimit: number) => number | false, encodeChat: (chat: readonly import("../GptEncoding.js").ChatMessage[], model?: "gpt-4" | "gpt-4-32k" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo" | "gpt-3.5-turbo-0301" | "text-davinci-003" | "text-davinci-002" | "text-davinci-001" | "text-curie-001" | "text-babbage-001" | "text-ada-001" | "davinci" | "curie" | "babbage" | "ada" | "code-davinci-002" | "code-davinci-001" | "code-cushman-002" | "code-cushman-001" | "davinci-codex" | "cushman-codex" | "text-davinci-edit-001" | "code-davinci-edit-001" | "text-embedding-ada-002" | "text-similarity-davinci-001" | "text-similarity-curie-001" | "text-similarity-babbage-001" | "text-similarity-ada-001" | "text-search-davinci-doc-001" | "text-search-curie-doc-001" | "text-search-babbage-doc-001" | "text-search-ada-doc-001" | "code-search-babbage-code-001" | "code-search-ada-code-001" | undefined) => number[], encodeChatGenerator: (chat: Iterable<import("../GptEncoding.js").ChatMessage>, model?: "gpt-4" | "gpt-4-32k" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo" | "gpt-3.5-turbo-0301" | "text-davinci-003" | "text-davinci-002" | "text-davinci-001" | "text-curie-001" | "text-babbage-001" | "text-ada-001" | "davinci" | "curie" | "babbage" | "ada" | "code-davinci-002" | "code-davinci-001" | "code-cushman-002" | "code-cushman-001" | "davinci-codex" | "cushman-codex" | "text-davinci-edit-001" | "code-davinci-edit-001" | "text-embedding-ada-002" | "text-similarity-davinci-001" | "text-similarity-curie-001" | "text-similarity-babbage-001" | "text-similarity-ada-001" | "text-search-davinci-doc-001" | "text-search-curie-doc-001" | "text-search-babbage-doc-001" | "text-search-ada-doc-001" | "code-search-babbage-code-001" | "code-search-ada-code-001" | undefined) => Generator<number[], void, undefined>;
+export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, };
+export default api;

package/esm/model/gpt-3.5-turbo-0301.js

@@ -0,0 +1,11 @@
+/* eslint-disable import/extensions */
+import { convertTokenBytePairEncodingFromTuples } from '../convertTokenBytePairEncodingFromTuples.js';
+import encoder from '../encodings/cl100k_base.js';
+import { GptEncoding } from '../GptEncoding.js';
+export * from '../specialTokens.js';
+const api = GptEncoding.getEncodingApiForModel('gpt-3.5-turbo-0301', () => convertTokenBytePairEncodingFromTuples(encoder));
+const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, encodeChat, encodeChatGenerator, } = api;
+export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, };
+// eslint-disable-next-line import/no-default-export
+export default api;
+//# sourceMappingURL=gpt-3.5-turbo-0301.js.map

package/esm/model/gpt-3.5-turbo-0301.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"gpt-3.5-turbo-0301.js","sourceRoot":"","sources":["../../src/model/gpt-3.5-turbo-0301.ts"],"names":[],"mappings":"AAAA,sCAAsC;AACtC,OAAO,EAAE,sCAAsC,EAAE,MAAM,8CAA8C,CAAA;AACrG,OAAO,OAAO,MAAM,6BAA6B,CAAA;AACjD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAA;AAE/C,cAAc,qBAAqB,CAAA;AAEnC,MAAM,GAAG,GAAG,WAAW,CAAC,sBAAsB,CAAC,oBAAoB,EAAE,GAAG,EAAE,CACxE,sCAAsC,CAAC,OAAO,CAAC,CAChD,CAAA;AACD,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,EAClB,UAAU,EACV,mBAAmB,GACpB,GAAG,GAAG,CAAA;AACP,OAAO,EACL,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,UAAU,EACV,mBAAmB,EACnB,eAAe,EACf,kBAAkB,GACnB,CAAA;AACD,oDAAoD;AACpD,eAAe,GAAG,CAAA"}

package/esm/model/gpt-3.5-turbo.d.ts

@@ -1 +1,6 @@
-export * from '../encoding/cl100k_base.js';
+import { GptEncoding } from '../GptEncoding.js';
+export * from '../specialTokens.js';
+declare const api: GptEncoding;
+declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void, unknown>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, unknown>, encode: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, { allowedSpecial, disallowedSpecial, }?: import("../GptEncoding.js").EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, tokenLimit: number) => number | false, encodeChat: (chat: readonly import("../GptEncoding.js").ChatMessage[], model?: "gpt-4" | "gpt-4-32k" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo" | "gpt-3.5-turbo-0301" | "text-davinci-003" | "text-davinci-002" | "text-davinci-001" | "text-curie-001" | "text-babbage-001" | "text-ada-001" | "davinci" | "curie" | "babbage" | "ada" | "code-davinci-002" | "code-davinci-001" | "code-cushman-002" | "code-cushman-001" | "davinci-codex" | "cushman-codex" | "text-davinci-edit-001" | "code-davinci-edit-001" | "text-embedding-ada-002" | "text-similarity-davinci-001" | "text-similarity-curie-001" | "text-similarity-babbage-001" | "text-similarity-ada-001" | "text-search-davinci-doc-001" | "text-search-curie-doc-001" | "text-search-babbage-doc-001" | "text-search-ada-doc-001" | "code-search-babbage-code-001" | "code-search-ada-code-001" | undefined) => number[], encodeChatGenerator: (chat: Iterable<import("../GptEncoding.js").ChatMessage>, model?: "gpt-4" | "gpt-4-32k" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo" | "gpt-3.5-turbo-0301" | "text-davinci-003" | "text-davinci-002" | "text-davinci-001" | "text-curie-001" | "text-babbage-001" | "text-ada-001" | "davinci" | "curie" | "babbage" | "ada" | "code-davinci-002" | "code-davinci-001" | "code-cushman-002" | "code-cushman-001" | "davinci-codex" | "cushman-codex" | "text-davinci-edit-001" | "code-davinci-edit-001" | "text-embedding-ada-002" | "text-similarity-davinci-001" | "text-similarity-curie-001" | "text-similarity-babbage-001" | "text-similarity-ada-001" | "text-search-davinci-doc-001" | "text-search-curie-doc-001" | "text-search-babbage-doc-001" | "text-search-ada-doc-001" | "code-search-babbage-code-001" | "code-search-ada-code-001" | undefined) => Generator<number[], void, undefined>;
+export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, };
+export default api;

package/esm/model/gpt-3.5-turbo.js

@@ -1,2 +1,11 @@
-export * from '../encoding/cl100k_base.js';
+/* eslint-disable import/extensions */
+import { convertTokenBytePairEncodingFromTuples } from '../convertTokenBytePairEncodingFromTuples.js';
+import encoder from '../encodings/cl100k_base.js';
+import { GptEncoding } from '../GptEncoding.js';
+export * from '../specialTokens.js';
+const api = GptEncoding.getEncodingApiForModel('gpt-3.5-turbo', () => convertTokenBytePairEncodingFromTuples(encoder));
+const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, encodeChat, encodeChatGenerator, } = api;
+export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, };
+// eslint-disable-next-line import/no-default-export
+export default api;
 //# sourceMappingURL=gpt-3.5-turbo.js.map

package/esm/model/gpt-3.5-turbo.js.map

@@ -1 +1 @@
-{"version":3,"file":"gpt-3.5-turbo.js","sourceRoot":"","sources":["../../src/model/gpt-3.5-turbo.ts"],"names":[],"mappings":"AAAA,cAAc,4BAA4B,CAAA"}
+{"version":3,"file":"gpt-3.5-turbo.js","sourceRoot":"","sources":["../../src/model/gpt-3.5-turbo.ts"],"names":[],"mappings":"AAAA,sCAAsC;AACtC,OAAO,EAAE,sCAAsC,EAAE,MAAM,8CAA8C,CAAA;AACrG,OAAO,OAAO,MAAM,6BAA6B,CAAA;AACjD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAA;AAE/C,cAAc,qBAAqB,CAAA;AAEnC,MAAM,GAAG,GAAG,WAAW,CAAC,sBAAsB,CAAC,eAAe,EAAE,GAAG,EAAE,CACnE,sCAAsC,CAAC,OAAO,CAAC,CAChD,CAAA;AACD,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,EAClB,UAAU,EACV,mBAAmB,GACpB,GAAG,GAAG,CAAA;AACP,OAAO,EACL,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,UAAU,EACV,mBAAmB,EACnB,eAAe,EACf,kBAAkB,GACnB,CAAA;AACD,oDAAoD;AACpD,eAAe,GAAG,CAAA"}

package/esm/model/gpt-4-0314.d.ts

@@ -0,0 +1,6 @@
+import { GptEncoding } from '../GptEncoding.js';
+export * from '../specialTokens.js';
+declare const api: GptEncoding;
+declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void, unknown>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, unknown>, encode: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, { allowedSpecial, disallowedSpecial, }?: import("../GptEncoding.js").EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, tokenLimit: number) => number | false, encodeChat: (chat: readonly import("../GptEncoding.js").ChatMessage[], model?: "gpt-4" | "gpt-4-32k" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo" | "gpt-3.5-turbo-0301" | "text-davinci-003" | "text-davinci-002" | "text-davinci-001" | "text-curie-001" | "text-babbage-001" | "text-ada-001" | "davinci" | "curie" | "babbage" | "ada" | "code-davinci-002" | "code-davinci-001" | "code-cushman-002" | "code-cushman-001" | "davinci-codex" | "cushman-codex" | "text-davinci-edit-001" | "code-davinci-edit-001" | "text-embedding-ada-002" | "text-similarity-davinci-001" | "text-similarity-curie-001" | "text-similarity-babbage-001" | "text-similarity-ada-001" | "text-search-davinci-doc-001" | "text-search-curie-doc-001" | "text-search-babbage-doc-001" | "text-search-ada-doc-001" | "code-search-babbage-code-001" | "code-search-ada-code-001" | undefined) => number[], encodeChatGenerator: (chat: Iterable<import("../GptEncoding.js").ChatMessage>, model?: "gpt-4" | "gpt-4-32k" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo" | "gpt-3.5-turbo-0301" | "text-davinci-003" | "text-davinci-002" | "text-davinci-001" | "text-curie-001" | "text-babbage-001" | "text-ada-001" | "davinci" | "curie" | "babbage" | "ada" | "code-davinci-002" | "code-davinci-001" | "code-cushman-002" | "code-cushman-001" | "davinci-codex" | "cushman-codex" | "text-davinci-edit-001" | "code-davinci-edit-001" | "text-embedding-ada-002" | "text-similarity-davinci-001" | "text-similarity-curie-001" | "text-similarity-babbage-001" | "text-similarity-ada-001" | "text-search-davinci-doc-001" | "text-search-curie-doc-001" | "text-search-babbage-doc-001" | "text-search-ada-doc-001" | "code-search-babbage-code-001" | "code-search-ada-code-001" | undefined) => Generator<number[], void, undefined>;
+export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, };
+export default api;

package/esm/model/gpt-4-0314.js

@@ -0,0 +1,11 @@
+/* eslint-disable import/extensions */
+import { convertTokenBytePairEncodingFromTuples } from '../convertTokenBytePairEncodingFromTuples.js';
+import encoder from '../encodings/cl100k_base.js';
+import { GptEncoding } from '../GptEncoding.js';
+export * from '../specialTokens.js';
+const api = GptEncoding.getEncodingApiForModel('gpt-4-0314', () => convertTokenBytePairEncodingFromTuples(encoder));
+const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, encodeChat, encodeChatGenerator, } = api;
+export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, };
+// eslint-disable-next-line import/no-default-export
+export default api;
+//# sourceMappingURL=gpt-4-0314.js.map

package/esm/model/gpt-4-0314.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"gpt-4-0314.js","sourceRoot":"","sources":["../../src/model/gpt-4-0314.ts"],"names":[],"mappings":"AAAA,sCAAsC;AACtC,OAAO,EAAE,sCAAsC,EAAE,MAAM,8CAA8C,CAAA;AACrG,OAAO,OAAO,MAAM,6BAA6B,CAAA;AACjD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAA;AAE/C,cAAc,qBAAqB,CAAA;AAEnC,MAAM,GAAG,GAAG,WAAW,CAAC,sBAAsB,CAAC,YAAY,EAAE,GAAG,EAAE,CAChE,sCAAsC,CAAC,OAAO,CAAC,CAChD,CAAA;AACD,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,EAClB,UAAU,EACV,mBAAmB,GACpB,GAAG,GAAG,CAAA;AACP,OAAO,EACL,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,UAAU,EACV,mBAAmB,EACnB,eAAe,EACf,kBAAkB,GACnB,CAAA;AACD,oDAAoD;AACpD,eAAe,GAAG,CAAA"}

package/esm/model/gpt-4-32k-0314.d.ts

@@ -0,0 +1,6 @@
+import { GptEncoding } from '../GptEncoding.js';
+export * from '../specialTokens.js';
+declare const api: GptEncoding;
+declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void, unknown>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, unknown>, encode: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, { allowedSpecial, disallowedSpecial, }?: import("../GptEncoding.js").EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, tokenLimit: number) => number | false, encodeChat: (chat: readonly import("../GptEncoding.js").ChatMessage[], model?: "gpt-4" | "gpt-4-32k" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo" | "gpt-3.5-turbo-0301" | "text-davinci-003" | "text-davinci-002" | "text-davinci-001" | "text-curie-001" | "text-babbage-001" | "text-ada-001" | "davinci" | "curie" | "babbage" | "ada" | "code-davinci-002" | "code-davinci-001" | "code-cushman-002" | "code-cushman-001" | "davinci-codex" | "cushman-codex" | "text-davinci-edit-001" | "code-davinci-edit-001" | "text-embedding-ada-002" | "text-similarity-davinci-001" | "text-similarity-curie-001" | "text-similarity-babbage-001" | "text-similarity-ada-001" | "text-search-davinci-doc-001" | "text-search-curie-doc-001" | "text-search-babbage-doc-001" | "text-search-ada-doc-001" | "code-search-babbage-code-001" | "code-search-ada-code-001" | undefined) => number[], encodeChatGenerator: (chat: Iterable<import("../GptEncoding.js").ChatMessage>, model?: "gpt-4" | "gpt-4-32k" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo" | "gpt-3.5-turbo-0301" | "text-davinci-003" | "text-davinci-002" | "text-davinci-001" | "text-curie-001" | "text-babbage-001" | "text-ada-001" | "davinci" | "curie" | "babbage" | "ada" | "code-davinci-002" | "code-davinci-001" | "code-cushman-002" | "code-cushman-001" | "davinci-codex" | "cushman-codex" | "text-davinci-edit-001" | "code-davinci-edit-001" | "text-embedding-ada-002" | "text-similarity-davinci-001" | "text-similarity-curie-001" | "text-similarity-babbage-001" | "text-similarity-ada-001" | "text-search-davinci-doc-001" | "text-search-curie-doc-001" | "text-search-babbage-doc-001" | "text-search-ada-doc-001" | "code-search-babbage-code-001" | "code-search-ada-code-001" | undefined) => Generator<number[], void, undefined>;
+export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, };
+export default api;

package/esm/model/gpt-4-32k-0314.js

@@ -0,0 +1,11 @@
+/* eslint-disable import/extensions */
+import { convertTokenBytePairEncodingFromTuples } from '../convertTokenBytePairEncodingFromTuples.js';
+import encoder from '../encodings/cl100k_base.js';
+import { GptEncoding } from '../GptEncoding.js';
+export * from '../specialTokens.js';
+const api = GptEncoding.getEncodingApiForModel('gpt-4-32k-0314', () => convertTokenBytePairEncodingFromTuples(encoder));
+const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, encodeChat, encodeChatGenerator, } = api;
+export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, };
+// eslint-disable-next-line import/no-default-export
+export default api;
+//# sourceMappingURL=gpt-4-32k-0314.js.map

package/esm/model/gpt-4-32k-0314.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"gpt-4-32k-0314.js","sourceRoot":"","sources":["../../src/model/gpt-4-32k-0314.ts"],"names":[],"mappings":"AAAA,sCAAsC;AACtC,OAAO,EAAE,sCAAsC,EAAE,MAAM,8CAA8C,CAAA;AACrG,OAAO,OAAO,MAAM,6BAA6B,CAAA;AACjD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAA;AAE/C,cAAc,qBAAqB,CAAA;AAEnC,MAAM,GAAG,GAAG,WAAW,CAAC,sBAAsB,CAAC,gBAAgB,EAAE,GAAG,EAAE,CACpE,sCAAsC,CAAC,OAAO,CAAC,CAChD,CAAA;AACD,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,EAClB,UAAU,EACV,mBAAmB,GACpB,GAAG,GAAG,CAAA;AACP,OAAO,EACL,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,UAAU,EACV,mBAAmB,EACnB,eAAe,EACf,kBAAkB,GACnB,CAAA;AACD,oDAAoD;AACpD,eAAe,GAAG,CAAA"}

package/esm/model/gpt-4-32k.d.ts

@@ -0,0 +1,6 @@
+import { GptEncoding } from '../GptEncoding.js';
+export * from '../specialTokens.js';
+declare const api: GptEncoding;
+declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void, unknown>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, unknown>, encode: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, { allowedSpecial, disallowedSpecial, }?: import("../GptEncoding.js").EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, tokenLimit: number) => number | false, encodeChat: (chat: readonly import("../GptEncoding.js").ChatMessage[], model?: "gpt-4" | "gpt-4-32k" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo" | "gpt-3.5-turbo-0301" | "text-davinci-003" | "text-davinci-002" | "text-davinci-001" | "text-curie-001" | "text-babbage-001" | "text-ada-001" | "davinci" | "curie" | "babbage" | "ada" | "code-davinci-002" | "code-davinci-001" | "code-cushman-002" | "code-cushman-001" | "davinci-codex" | "cushman-codex" | "text-davinci-edit-001" | "code-davinci-edit-001" | "text-embedding-ada-002" | "text-similarity-davinci-001" | "text-similarity-curie-001" | "text-similarity-babbage-001" | "text-similarity-ada-001" | "text-search-davinci-doc-001" | "text-search-curie-doc-001" | "text-search-babbage-doc-001" | "text-search-ada-doc-001" | "code-search-babbage-code-001" | "code-search-ada-code-001" | undefined) => number[], encodeChatGenerator: (chat: Iterable<import("../GptEncoding.js").ChatMessage>, model?: "gpt-4" | "gpt-4-32k" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo" | "gpt-3.5-turbo-0301" | "text-davinci-003" | "text-davinci-002" | "text-davinci-001" | "text-curie-001" | "text-babbage-001" | "text-ada-001" | "davinci" | "curie" | "babbage" | "ada" | "code-davinci-002" | "code-davinci-001" | "code-cushman-002" | "code-cushman-001" | "davinci-codex" | "cushman-codex" | "text-davinci-edit-001" | "code-davinci-edit-001" | "text-embedding-ada-002" | "text-similarity-davinci-001" | "text-similarity-curie-001" | "text-similarity-babbage-001" | "text-similarity-ada-001" | "text-search-davinci-doc-001" | "text-search-curie-doc-001" | "text-search-babbage-doc-001" | "text-search-ada-doc-001" | "code-search-babbage-code-001" | "code-search-ada-code-001" | undefined) => Generator<number[], void, undefined>;
+export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, };
+export default api;

package/esm/model/gpt-4-32k.js

@@ -0,0 +1,11 @@
+/* eslint-disable import/extensions */
+import { convertTokenBytePairEncodingFromTuples } from '../convertTokenBytePairEncodingFromTuples.js';
+import encoder from '../encodings/cl100k_base.js';
+import { GptEncoding } from '../GptEncoding.js';
+export * from '../specialTokens.js';
+const api = GptEncoding.getEncodingApiForModel('gpt-4-32k', () => convertTokenBytePairEncodingFromTuples(encoder));
+const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, encodeChat, encodeChatGenerator, } = api;
+export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, };
+// eslint-disable-next-line import/no-default-export
+export default api;
+//# sourceMappingURL=gpt-4-32k.js.map

package/esm/model/gpt-4-32k.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"gpt-4-32k.js","sourceRoot":"","sources":["../../src/model/gpt-4-32k.ts"],"names":[],"mappings":"AAAA,sCAAsC;AACtC,OAAO,EAAE,sCAAsC,EAAE,MAAM,8CAA8C,CAAA;AACrG,OAAO,OAAO,MAAM,6BAA6B,CAAA;AACjD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAA;AAE/C,cAAc,qBAAqB,CAAA;AAEnC,MAAM,GAAG,GAAG,WAAW,CAAC,sBAAsB,CAAC,WAAW,EAAE,GAAG,EAAE,CAC/D,sCAAsC,CAAC,OAAO,CAAC,CAChD,CAAA;AACD,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,EAClB,UAAU,EACV,mBAAmB,GACpB,GAAG,GAAG,CAAA;AACP,OAAO,EACL,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,UAAU,EACV,mBAAmB,EACnB,eAAe,EACf,kBAAkB,GACnB,CAAA;AACD,oDAAoD;AACpD,eAAe,GAAG,CAAA"}

package/esm/model/gpt-4.d.ts

@@ -1 +1,6 @@
-export * from '../encoding/cl100k_base.js';
+import { GptEncoding } from '../GptEncoding.js';
+export * from '../specialTokens.js';
+declare const api: GptEncoding;
+declare const decode: (inputTokensToDecode: Iterable<number>) => string, decodeAsyncGenerator: (inputTokensToDecode: AsyncIterable<number>) => AsyncGenerator<string, void, unknown>, decodeGenerator: (inputTokensToDecode: Iterable<number>) => Generator<string, void, unknown>, encode: (lineToEncode: string, encodeOptions?: import("../GptEncoding.js").EncodeOptions) => number[], encodeGenerator: (lineToEncode: string, { allowedSpecial, disallowedSpecial, }?: import("../GptEncoding.js").EncodeOptions) => Generator<number[], number, undefined>, isWithinTokenLimit: (input: string | Iterable<import("../GptEncoding.js").ChatMessage>, tokenLimit: number) => number | false, encodeChat: (chat: readonly import("../GptEncoding.js").ChatMessage[], model?: "gpt-4" | "gpt-4-32k" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo" | "gpt-3.5-turbo-0301" | "text-davinci-003" | "text-davinci-002" | "text-davinci-001" | "text-curie-001" | "text-babbage-001" | "text-ada-001" | "davinci" | "curie" | "babbage" | "ada" | "code-davinci-002" | "code-davinci-001" | "code-cushman-002" | "code-cushman-001" | "davinci-codex" | "cushman-codex" | "text-davinci-edit-001" | "code-davinci-edit-001" | "text-embedding-ada-002" | "text-similarity-davinci-001" | "text-similarity-curie-001" | "text-similarity-babbage-001" | "text-similarity-ada-001" | "text-search-davinci-doc-001" | "text-search-curie-doc-001" | "text-search-babbage-doc-001" | "text-search-ada-doc-001" | "code-search-babbage-code-001" | "code-search-ada-code-001" | undefined) => number[], encodeChatGenerator: (chat: Iterable<import("../GptEncoding.js").ChatMessage>, model?: "gpt-4" | "gpt-4-32k" | "gpt-4-0314" | "gpt-4-32k-0314" | "gpt-3.5-turbo" | "gpt-3.5-turbo-0301" | "text-davinci-003" | "text-davinci-002" | "text-davinci-001" | "text-curie-001" | "text-babbage-001" | "text-ada-001" | "davinci" | "curie" | "babbage" | "ada" | "code-davinci-002" | "code-davinci-001" | "code-cushman-002" | "code-cushman-001" | "davinci-codex" | "cushman-codex" | "text-davinci-edit-001" | "code-davinci-edit-001" | "text-embedding-ada-002" | "text-similarity-davinci-001" | "text-similarity-curie-001" | "text-similarity-babbage-001" | "text-similarity-ada-001" | "text-search-davinci-doc-001" | "text-search-curie-doc-001" | "text-search-babbage-doc-001" | "text-search-ada-doc-001" | "code-search-babbage-code-001" | "code-search-ada-code-001" | undefined) => Generator<number[], void, undefined>;
+export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, };
+export default api;

package/esm/model/gpt-4.js

@@ -1,2 +1,11 @@
-export * from '../encoding/cl100k_base.js';
+/* eslint-disable import/extensions */
+import { convertTokenBytePairEncodingFromTuples } from '../convertTokenBytePairEncodingFromTuples.js';
+import encoder from '../encodings/cl100k_base.js';
+import { GptEncoding } from '../GptEncoding.js';
+export * from '../specialTokens.js';
+const api = GptEncoding.getEncodingApiForModel('gpt-4', () => convertTokenBytePairEncodingFromTuples(encoder));
+const { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeGenerator, isWithinTokenLimit, encodeChat, encodeChatGenerator, } = api;
+export { decode, decodeAsyncGenerator, decodeGenerator, encode, encodeChat, encodeChatGenerator, encodeGenerator, isWithinTokenLimit, };
+// eslint-disable-next-line import/no-default-export
+export default api;
 //# sourceMappingURL=gpt-4.js.map

package/esm/model/gpt-4.js.map

@@ -1 +1 @@
-{"version":3,"file":"gpt-4.js","sourceRoot":"","sources":["../../src/model/gpt-4.ts"],"names":[],"mappings":"AAAA,cAAc,4BAA4B,CAAA"}
+{"version":3,"file":"gpt-4.js","sourceRoot":"","sources":["../../src/model/gpt-4.ts"],"names":[],"mappings":"AAAA,sCAAsC;AACtC,OAAO,EAAE,sCAAsC,EAAE,MAAM,8CAA8C,CAAA;AACrG,OAAO,OAAO,MAAM,6BAA6B,CAAA;AACjD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAA;AAE/C,cAAc,qBAAqB,CAAA;AAEnC,MAAM,GAAG,GAAG,WAAW,CAAC,sBAAsB,CAAC,OAAO,EAAE,GAAG,EAAE,CAC3D,sCAAsC,CAAC,OAAO,CAAC,CAChD,CAAA;AACD,MAAM,EACJ,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,eAAe,EACf,kBAAkB,EAClB,UAAU,EACV,mBAAmB,GACpB,GAAG,GAAG,CAAA;AACP,OAAO,EACL,MAAM,EACN,oBAAoB,EACpB,eAAe,EACf,MAAM,EACN,UAAU,EACV,mBAAmB,EACnB,eAAe,EACf,kBAAkB,GACnB,CAAA;AACD,oDAAoD;AACpD,eAAe,GAAG,CAAA"}

package/esm/model/text-ada-001.d.ts

@@ -1 +1,2 @@
+export { default } from '../encoding/r50k_base.js';
 export * from '../encoding/r50k_base.js';

package/esm/model/text-ada-001.js

@@ -1,2 +1,4 @@
+// eslint-disable-next-line no-restricted-exports, import/no-default-export
+export { default } from '../encoding/r50k_base.js';
 export * from '../encoding/r50k_base.js';
 //# sourceMappingURL=text-ada-001.js.map

package/esm/model/text-ada-001.js.map

@@ -1 +1 @@
-{"version":3,"file":"text-ada-001.js","sourceRoot":"","sources":["../../src/model/text-ada-001.ts"],"names":[],"mappings":"AAAA,cAAc,0BAA0B,CAAA"}
+{"version":3,"file":"text-ada-001.js","sourceRoot":"","sources":["../../src/model/text-ada-001.ts"],"names":[],"mappings":"AAAA,2EAA2E;AAC3E,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAA;AAClD,cAAc,0BAA0B,CAAA"}

package/esm/model/text-babbage-001.d.ts

@@ -1 +1,2 @@
+export { default } from '../encoding/r50k_base.js';
 export * from '../encoding/r50k_base.js';

package/esm/model/text-babbage-001.js

@@ -1,2 +1,4 @@
+// eslint-disable-next-line no-restricted-exports, import/no-default-export
+export { default } from '../encoding/r50k_base.js';
 export * from '../encoding/r50k_base.js';
 //# sourceMappingURL=text-babbage-001.js.map

package/esm/model/text-babbage-001.js.map

@@ -1 +1 @@
-{"version":3,"file":"text-babbage-001.js","sourceRoot":"","sources":["../../src/model/text-babbage-001.ts"],"names":[],"mappings":"AAAA,cAAc,0BAA0B,CAAA"}
+{"version":3,"file":"text-babbage-001.js","sourceRoot":"","sources":["../../src/model/text-babbage-001.ts"],"names":[],"mappings":"AAAA,2EAA2E;AAC3E,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAA;AAClD,cAAc,0BAA0B,CAAA"}

package/esm/model/text-curie-001.d.ts

@@ -1 +1,2 @@
+export { default } from '../encoding/r50k_base.js';
 export * from '../encoding/r50k_base.js';

package/esm/model/text-curie-001.js

@@ -1,2 +1,4 @@
+// eslint-disable-next-line no-restricted-exports, import/no-default-export
+export { default } from '../encoding/r50k_base.js';
 export * from '../encoding/r50k_base.js';
 //# sourceMappingURL=text-curie-001.js.map

package/esm/model/text-curie-001.js.map

@@ -1 +1 @@
-{"version":3,"file":"text-curie-001.js","sourceRoot":"","sources":["../../src/model/text-curie-001.ts"],"names":[],"mappings":"AAAA,cAAc,0BAA0B,CAAA"}
+{"version":3,"file":"text-curie-001.js","sourceRoot":"","sources":["../../src/model/text-curie-001.ts"],"names":[],"mappings":"AAAA,2EAA2E;AAC3E,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAA;AAClD,cAAc,0BAA0B,CAAA"}

package/esm/model/text-davinci-001.d.ts

@@ -1 +1,2 @@
+export { default } from '../encoding/r50k_base.js';
 export * from '../encoding/r50k_base.js';

package/esm/model/text-davinci-001.js

@@ -1,2 +1,4 @@
+// eslint-disable-next-line no-restricted-exports, import/no-default-export
+export { default } from '../encoding/r50k_base.js';
 export * from '../encoding/r50k_base.js';
 //# sourceMappingURL=text-davinci-001.js.map

package/esm/model/text-davinci-001.js.map

@@ -1 +1 @@
-{"version":3,"file":"text-davinci-001.js","sourceRoot":"","sources":["../../src/model/text-davinci-001.ts"],"names":[],"mappings":"AAAA,cAAc,0BAA0B,CAAA"}
+{"version":3,"file":"text-davinci-001.js","sourceRoot":"","sources":["../../src/model/text-davinci-001.ts"],"names":[],"mappings":"AAAA,2EAA2E;AAC3E,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAA;AAClD,cAAc,0BAA0B,CAAA"}

package/esm/model/text-davinci-002.d.ts

@@ -1 +1,2 @@
+export { default } from '../encoding/p50k_base.js';
 export * from '../encoding/p50k_base.js';

package/esm/model/text-davinci-002.js

@@ -1,2 +1,4 @@
+// eslint-disable-next-line no-restricted-exports, import/no-default-export
+export { default } from '../encoding/p50k_base.js';
 export * from '../encoding/p50k_base.js';
 //# sourceMappingURL=text-davinci-002.js.map

package/esm/model/text-davinci-002.js.map

@@ -1 +1 @@
-{"version":3,"file":"text-davinci-002.js","sourceRoot":"","sources":["../../src/model/text-davinci-002.ts"],"names":[],"mappings":"AAAA,cAAc,0BAA0B,CAAA"}
+{"version":3,"file":"text-davinci-002.js","sourceRoot":"","sources":["../../src/model/text-davinci-002.ts"],"names":[],"mappings":"AAAA,2EAA2E;AAC3E,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAA;AAClD,cAAc,0BAA0B,CAAA"}

package/esm/model/text-davinci-003.d.ts

@@ -1 +1,2 @@
+export { default } from '../encoding/p50k_base.js';
 export * from '../encoding/p50k_base.js';

package/esm/model/text-davinci-003.js

@@ -1,2 +1,4 @@
+// eslint-disable-next-line no-restricted-exports, import/no-default-export
+export { default } from '../encoding/p50k_base.js';
 export * from '../encoding/p50k_base.js';
 //# sourceMappingURL=text-davinci-003.js.map

package/esm/model/text-davinci-003.js.map

@@ -1 +1 @@
-{"version":3,"file":"text-davinci-003.js","sourceRoot":"","sources":["../../src/model/text-davinci-003.ts"],"names":[],"mappings":"AAAA,cAAc,0BAA0B,CAAA"}
+{"version":3,"file":"text-davinci-003.js","sourceRoot":"","sources":["../../src/model/text-davinci-003.ts"],"names":[],"mappings":"AAAA,2EAA2E;AAC3E,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAA;AAClD,cAAc,0BAA0B,CAAA"}

package/esm/model/text-davinci-edit-001.d.ts

@@ -1 +1,2 @@
+export { default } from '../encoding/p50k_edit.js';
 export * from '../encoding/p50k_edit.js';

package/esm/model/text-davinci-edit-001.js

@@ -1,2 +1,4 @@
+// eslint-disable-next-line no-restricted-exports, import/no-default-export
+export { default } from '../encoding/p50k_edit.js';
 export * from '../encoding/p50k_edit.js';
 //# sourceMappingURL=text-davinci-edit-001.js.map

package/esm/model/text-davinci-edit-001.js.map

@@ -1 +1 @@
-{"version":3,"file":"text-davinci-edit-001.js","sourceRoot":"","sources":["../../src/model/text-davinci-edit-001.ts"],"names":[],"mappings":"AAAA,cAAc,0BAA0B,CAAA"}
+{"version":3,"file":"text-davinci-edit-001.js","sourceRoot":"","sources":["../../src/model/text-davinci-edit-001.ts"],"names":[],"mappings":"AAAA,2EAA2E;AAC3E,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAA;AAClD,cAAc,0BAA0B,CAAA"}

package/esm/model/text-embedding-ada-002.d.ts

@@ -1 +1,2 @@
+export { default } from '../encoding/cl100k_base.js';
 export * from '../encoding/cl100k_base.js';