@jeffreycao/copilot-api 1.9.0-beta.0 → 1.9.0

package/dist/GptEncoding-TiCgsNrQ.js

@@ -1,887 +0,0 @@
-import { __export } from "./chunk-BZ41Y9eH.js";
-
-//#region node_modules/gpt-tokenizer/esm/constants.js
-const ALL_SPECIAL_TOKENS = "all";
-const DEFAULT_MERGE_CACHE_SIZE = 1e5;
-
-//#endregion
-//#region node_modules/gpt-tokenizer/esm/utfUtil.js
-const isAscii = (codePoint) => codePoint <= 127;
-const HIGH_SURROGATE_START = 55296;
-const HIGH_SURROGATE_END = 56319;
-function endsWithIncompleteUtfPairSurrogate(string) {
-	if (string.length === 0) return false;
-	const lastCharCode = string.charCodeAt(string.length - 1);
-	return lastCharCode >= HIGH_SURROGATE_START && lastCharCode <= HIGH_SURROGATE_END;
-}
-function isValidUTF8(bytes) {
-	let i = 0;
-	while (i < bytes.length) {
-		const byte1 = bytes[i];
-		let numBytes = 0;
-		let codePoint = 0;
-		if (byte1 <= 127) {
-			numBytes = 1;
-			codePoint = byte1;
-		} else if ((byte1 & 224) === 192) {
-			numBytes = 2;
-			codePoint = byte1 & 31;
-			if (byte1 <= 193) return false;
-		} else if ((byte1 & 240) === 224) {
-			numBytes = 3;
-			codePoint = byte1 & 15;
-		} else if ((byte1 & 248) === 240) {
-			numBytes = 4;
-			codePoint = byte1 & 7;
-			if (byte1 > 244) return false;
-		} else return false;
-		if (i + numBytes > bytes.length) return false;
-		for (let j = 1; j < numBytes; j++) {
-			const byte = bytes[i + j];
-			if (byte === void 0 || (byte & 192) !== 128) return false;
-			codePoint = codePoint << 6 | byte & 63;
-		}
-		if (numBytes === 2 && codePoint < 128) return false;
-		if (numBytes === 3 && codePoint < 2048) return false;
-		if (numBytes === 4 && codePoint < 65536) return false;
-		if (codePoint >= 55296 && codePoint <= 57343) return false;
-		if (codePoint > 1114111) return false;
-		i += numBytes;
-	}
-	return true;
-}
-const textDecoder = new TextDecoder("utf8", { fatal: false });
-function tryConvertToString(arr) {
-	if (!isValidUTF8(arr)) return;
-	return textDecoder.decode(arr);
-}
-function compareUint8Arrays(a, b) {
-	const len = Math.min(a.length, b.length);
-	for (let i = 0; i < len; i++) if (a[i] !== b[i]) return a[i] - b[i];
-	return a.length - b.length;
-}
-
-//#endregion
-//#region node_modules/gpt-tokenizer/esm/util.js
-function getMaxValueFromMap(map) {
-	let max = 0;
-	map.forEach((val) => {
-		max = Math.max(max, val);
-	});
-	return max;
-}
-function escapeRegExp(string) {
-	return string.replace(/[$()*+.?[\\\]^{|}]/g, "\\$&");
-}
-function getSpecialTokenRegex(tokens) {
-	const inner = [...tokens].map(escapeRegExp).join("|");
-	return /* @__PURE__ */ new RegExp(`(${inner})`);
-}
-
-//#endregion
-//#region node_modules/gpt-tokenizer/esm/BytePairEncodingCore.js
-const emptyBuffer = new Uint8Array(0);
-const decoder = new TextDecoder("utf8");
-var BytePairEncodingCore = class {
-	mergeableBytePairRankCount;
-	/**
-	* an array where the index is the BPE rank,
-	* and the value is the string or the array of bytes that it decodes to
-	* it may contain holes if token is unused
-	*/
-	bytePairRankDecoder;
-	bytePairNonUtfRankDecoder = /* @__PURE__ */ new Map();
-	bytePairNonUtfSortedEncoder;
-	/**
-	* a reverse map of the bytePairRankDecoder,
-	* where the key is the string and the value is the rank
-	* values that cannot be represented as a string are present in `bytePairNonUtfSortedEncoder`
-	*/
-	bytePairStringRankEncoder;
-	tokenSplitRegex;
-	specialTokensEncoder;
-	specialTokensDecoder;
-	specialTokenPatternRegex;
-	textEncoder = new TextEncoder();
-	mergeCache;
-	mergeCacheSize;
-	constructor({ bytePairRankDecoder, specialTokensEncoder, tokenSplitRegex, mergeCacheSize = DEFAULT_MERGE_CACHE_SIZE }) {
-		this.bytePairRankDecoder = bytePairRankDecoder;
-		this.bytePairStringRankEncoder = /* @__PURE__ */ new Map();
-		this.mergeCacheSize = mergeCacheSize;
-		if (mergeCacheSize > 0) this.mergeCache = /* @__PURE__ */ new Map();
-		this.mergeableBytePairRankCount = Object.keys(bytePairRankDecoder).length;
-		const binaryLookup = [];
-		bytePairRankDecoder.forEach((value, rank) => {
-			if (typeof value === "string") {
-				this.bytePairStringRankEncoder.set(value, rank);
-				return;
-			}
-			const byteArray = new Uint8Array(value);
-			binaryLookup.push([byteArray, rank]);
-			this.bytePairNonUtfRankDecoder.set(rank, byteArray);
-		});
-		this.bytePairNonUtfSortedEncoder = binaryLookup.sort((a, b) => compareUint8Arrays(a[0], b[0]));
-		this.specialTokensEncoder = specialTokensEncoder ?? /* @__PURE__ */ new Map();
-		this.specialTokensDecoder = specialTokensEncoder ? new Map([...specialTokensEncoder].map(([key, value]) => [value, key])) : /* @__PURE__ */ new Map();
-		this.tokenSplitRegex = tokenSplitRegex;
-		const allSpecialTokensRegex = [...this.specialTokensEncoder.keys()].map(escapeRegExp).join("|");
-		try {
-			this.specialTokenPatternRegex = new RegExp(allSpecialTokensRegex, "y");
-		} catch {
-			throw new Error("Invalid regular expression pattern.");
-		}
-	}
-	setMergeCacheSize(newSize) {
-		if (this.mergeCacheSize === 0 && newSize > 0) this.mergeCache = /* @__PURE__ */ new Map();
-		this.mergeCacheSize = newSize;
-		if (newSize === 0) this.mergeCache = void 0;
-	}
-	clearMergeCache() {
-		this.mergeCache?.clear();
-	}
-	*encodeNativeGenerator(text, allowedSpecial) {
-		let startIndex = 0;
-		let lastTokenLength = 0;
-		while (true) {
-			const nextSpecialMatch = this.findNextSpecialToken(text, allowedSpecial, startIndex);
-			const nextSpecialStartIndex = nextSpecialMatch?.[0];
-			const endIndex = nextSpecialStartIndex ?? text.length;
-			const textBeforeSpecial = startIndex === 0 && endIndex === text.length ? text : text.slice(startIndex, endIndex);
-			for (const [match] of textBeforeSpecial.matchAll(this.tokenSplitRegex)) {
-				const token = this.getBpeRankFromString(match);
-				if (token !== void 0) {
-					lastTokenLength = 1;
-					yield [token];
-					continue;
-				}
-				const tokens = this.bytePairEncode(match);
-				lastTokenLength = tokens.length;
-				yield tokens;
-			}
-			if (nextSpecialStartIndex !== void 0) {
-				const specialToken = nextSpecialMatch[1];
-				const specialTokenValue = this.specialTokensEncoder.get(specialToken);
-				if (specialTokenValue === void 0) throw new Error(`Special token "${specialToken}" is not in the special token encoder.`);
-				yield [specialTokenValue];
-				startIndex = nextSpecialStartIndex + specialToken.length;
-				lastTokenLength = 1;
-			} else break;
-		}
-		return lastTokenLength;
-	}
-	encodeNative(text, allowedSpecial) {
-		let startIndex = 0;
-		const tokensArray = [];
-		while (true) {
-			const nextSpecialMatch = this.findNextSpecialToken(text, allowedSpecial, startIndex);
-			const nextSpecialStartIndex = nextSpecialMatch?.[0];
-			const endIndex = nextSpecialStartIndex ?? text.length;
-			const textBeforeSpecial = startIndex === 0 && endIndex === text.length ? text : text.slice(startIndex, endIndex);
-			for (const [match] of textBeforeSpecial.matchAll(this.tokenSplitRegex)) {
-				const token = this.getBpeRankFromString(match);
-				if (token !== void 0) {
-					tokensArray.push(token);
-					continue;
-				}
-				const tokens = this.bytePairEncode(match);
-				tokensArray.push(...tokens);
-			}
-			if (nextSpecialStartIndex !== void 0) {
-				const specialToken = nextSpecialMatch[1];
-				const specialTokenValue = this.specialTokensEncoder.get(specialToken);
-				if (specialTokenValue === void 0) throw new Error(`Special token "${specialToken}" is not in the special token encoder.`);
-				tokensArray.push(specialTokenValue);
-				startIndex = nextSpecialStartIndex + specialToken.length;
-			} else break;
-		}
-		return tokensArray;
-	}
-	countNative(text, allowedSpecial) {
-		let startIndex = 0;
-		let tokensCount = 0;
-		while (true) {
-			const nextSpecialMatch = this.findNextSpecialToken(text, allowedSpecial, startIndex);
-			const nextSpecialStartIndex = nextSpecialMatch?.[0];
-			const endIndex = nextSpecialStartIndex ?? text.length;
-			const textBeforeSpecial = startIndex === 0 && endIndex === text.length ? text : text.slice(startIndex, endIndex);
-			for (const [match] of textBeforeSpecial.matchAll(this.tokenSplitRegex)) {
-				if (this.getBpeRankFromString(match) !== void 0) {
-					tokensCount++;
-					continue;
-				}
-				const tokens = this.bytePairEncode(match);
-				tokensCount += tokens.length;
-			}
-			if (nextSpecialStartIndex !== void 0) {
-				const specialToken = nextSpecialMatch[1];
-				if (this.specialTokensEncoder.get(specialToken) === void 0) throw new Error(`Special token "${specialToken}" is not in the special token encoder.`);
-				tokensCount++;
-				startIndex = nextSpecialStartIndex + specialToken.length;
-			} else break;
-		}
-		return tokensCount;
-	}
-	*decodeNativeGenerator(tokens) {
-		for (const token of tokens) {
-			const tokenBytes = this.tryDecodeToken(token);
-			if (tokenBytes) yield tokenBytes;
-		}
-	}
-	decodeNative(tokens) {
-		let decoded = "";
-		let intBuffer = emptyBuffer;
-		for (const token of tokens) {
-			const tokenBytes = this.tryDecodeToken(token);
-			if (tokenBytes === void 0) throw new Error(`Token ${token} is not in the byte pair encoder.`);
-			if (typeof tokenBytes === "string") {
-				if (intBuffer !== emptyBuffer) {
-					decoded += decoder.decode(intBuffer, { stream: true });
-					intBuffer = emptyBuffer;
-				}
-				decoded += tokenBytes;
-			} else {
-				const newBuffer = new Uint8Array(intBuffer.length + tokenBytes.length);
-				newBuffer.set(intBuffer);
-				newBuffer.set(tokenBytes, intBuffer.length);
-				intBuffer = newBuffer;
-			}
-		}
-		if (intBuffer !== emptyBuffer) decoded += decoder.decode(intBuffer, { stream: true });
-		return decoded;
-	}
-	async *decodeNativeAsyncIterable(tokens) {
-		for await (const token of tokens) {
-			const tokenBytesOrString = this.tryDecodeToken(token);
-			if (tokenBytesOrString) yield tokenBytesOrString;
-		}
-	}
-	getBpeRankFromString(key) {
-		return this.bytePairStringRankEncoder.get(key);
-	}
-	getBpeRankFromStringOrThrow(key) {
-		const value = this.getBpeRankFromString(key);
-		if (value === void 0) throw new Error(`The byte-pair encoding does not contain a value for: ${key}`);
-		return value;
-	}
-	getBpeRankFromBytes(key) {
-		const keyAsString = tryConvertToString(key);
-		if (keyAsString !== void 0) return this.getBpeRankFromString(keyAsString);
-		const index = this.binarySearch(key);
-		if (index !== -1) return this.bytePairNonUtfSortedEncoder[index][1];
-	}
-	getBpeRankFromBytesOrThrow(key) {
-		const value = this.getBpeRankFromBytes(key);
-		if (value === void 0) throw new Error(`The byte-pair encoding does not contain a value for: ${key.toString()}`);
-		return value;
-	}
-	binarySearch(key) {
-		let low = 0;
-		let high = this.bytePairNonUtfSortedEncoder.length - 1;
-		while (low <= high) {
-			const mid = low + high >>> 1;
-			const midKey = this.bytePairNonUtfSortedEncoder[mid][0];
-			let cmp = 0;
-			const maxLength = Math.min(midKey.length, key.length);
-			for (let i = 0; i < maxLength; i++) {
-				cmp = midKey[i] - key[i];
-				if (cmp !== 0) break;
-			}
-			if (cmp === 0) cmp = midKey.length - key.length;
-			if (cmp === 0) return mid;
-			if (cmp < 0) low = mid + 1;
-			else high = mid - 1;
-		}
-		return -1;
-	}
-	findNextSpecialToken(text, allowedSpecial, startIndex) {
-		let searchIndex = startIndex;
-		while (true) {
-			this.specialTokenPatternRegex.lastIndex = searchIndex;
-			const nextSpecialMatch = this.specialTokenPatternRegex.exec(text);
-			if (!nextSpecialMatch) return;
-			const specialToken = nextSpecialMatch[0];
-			if (allowedSpecial?.has(specialToken)) return [nextSpecialMatch.index + searchIndex, specialToken];
-			searchIndex = nextSpecialMatch.index + searchIndex + 1;
-		}
-	}
-	tryDecodeToken(tokenRank) {
-		const value = this.bytePairRankDecoder[tokenRank];
-		if (typeof value === "string") return value;
-		if (typeof value === "object") {
-			const fromBinary = this.bytePairNonUtfRankDecoder.get(tokenRank);
-			if (fromBinary) return fromBinary;
-		}
-		return this.specialTokensDecoder.get(tokenRank);
-	}
-	addToMergeCache(key, value) {
-		if (!this.mergeCache) return;
-		if (this.mergeCache.size >= this.mergeCacheSize) {
-			const firstKey = this.mergeCache.keys().next().value;
-			this.mergeCache.delete(firstKey);
-		}
-		this.mergeCache.set(key, value);
-	}
-	bytePairEncode(input) {
-		if (input.length === 1 && isAscii(input.codePointAt(0))) return [this.getBpeRankFromStringOrThrow(input)];
-		if (this.mergeCache?.has(input)) {
-			const result$1 = this.mergeCache.get(input);
-			this.mergeCache.delete(input);
-			this.mergeCache.set(input, result$1);
-			return result$1;
-		}
-		const inputBytes = this.textEncoder.encode(input);
-		const result = this.bytePairMerge(inputBytes);
-		this.addToMergeCache(input, result);
-		return result;
-	}
-	bytePairMerge(piece) {
-		const starts = [];
-		const ranks = [];
-		const getRank = (startIndex, pairStart = starts[startIndex], pairEnd = starts[startIndex + 2]) => {
-			if (pairEnd === void 0) return Number.POSITIVE_INFINITY;
-			const key = piece.subarray(pairStart, pairEnd);
-			return this.getBpeRankFromBytes(key) ?? Number.POSITIVE_INFINITY;
-		};
-		for (let i = 0; i <= piece.length; i++) {
-			starts.push(i);
-			if (i < piece.length - 1) ranks.push(getRank(i, i, i + 2));
-			else ranks.push(Number.POSITIVE_INFINITY);
-		}
-		while (starts.length > 1) {
-			let lowestRank = Number.POSITIVE_INFINITY;
-			let lowestPartitionIndex = -1;
-			for (let i = 0; i < ranks.length - 1; i++) {
-				const rank = ranks[i];
-				if (rank < lowestRank) {
-					lowestRank = rank;
-					lowestPartitionIndex = i;
-				}
-			}
-			if (lowestRank === Number.POSITIVE_INFINITY || lowestPartitionIndex === -1) break;
-			starts.splice(lowestPartitionIndex + 1, 1);
-			ranks.splice(lowestPartitionIndex, 1);
-			ranks[lowestPartitionIndex] = getRank(lowestPartitionIndex);
-			if (lowestPartitionIndex > 0) ranks[lowestPartitionIndex - 1] = getRank(lowestPartitionIndex - 1);
-		}
-		const output = [];
-		for (let i = 0; i < starts.length - 1; i++) {
-			const pairStart = starts[i];
-			const pairEnd = starts[i + 1];
-			const bpeValue = this.getBpeRankFromBytesOrThrow(piece.subarray(pairStart, pairEnd));
-			output.push(bpeValue);
-		}
-		return output;
-	}
-};
-
-//#endregion
-//#region node_modules/gpt-tokenizer/esm/modelsChatEnabled.gen.js
-const chatEnabledModels = [
-	"chatgpt-4o-latest",
-	"codex-mini-latest",
-	"computer-use-preview",
-	"computer-use-preview-2025-03-11",
-	"gpt-3.5",
-	"gpt-3.5-0301",
-	"gpt-3.5-turbo",
-	"gpt-3.5-turbo-0125",
-	"gpt-3.5-turbo-0613",
-	"gpt-3.5-turbo-1106",
-	"gpt-3.5-turbo-16k-0613",
-	"gpt-3.5-turbo-instruct",
-	"gpt-4",
-	"gpt-4-0125-preview",
-	"gpt-4-0314",
-	"gpt-4-0613",
-	"gpt-4-1106-preview",
-	"gpt-4-1106-vision-preview",
-	"gpt-4-32k",
-	"gpt-4-turbo",
-	"gpt-4-turbo-2024-04-09",
-	"gpt-4-turbo-preview",
-	"gpt-4.1",
-	"gpt-4.1-2025-04-14",
-	"gpt-4.1-mini",
-	"gpt-4.1-mini-2025-04-14",
-	"gpt-4.1-nano",
-	"gpt-4.1-nano-2025-04-14",
-	"gpt-4.5-preview",
-	"gpt-4.5-preview-2025-02-27",
-	"gpt-4o",
-	"gpt-4o-2024-05-13",
-	"gpt-4o-2024-08-06",
-	"gpt-4o-2024-11-20",
-	"gpt-4o-audio-preview",
-	"gpt-4o-audio-preview-2024-10-01",
-	"gpt-4o-audio-preview-2024-12-17",
-	"gpt-4o-audio-preview-2025-06-03",
-	"gpt-4o-mini",
-	"gpt-4o-mini-2024-07-18",
-	"gpt-4o-mini-audio-preview",
-	"gpt-4o-mini-audio-preview-2024-12-17",
-	"gpt-4o-mini-search-preview",
-	"gpt-4o-mini-search-preview-2025-03-11",
-	"gpt-4o-search-preview",
-	"gpt-4o-search-preview-2025-03-11",
-	"o1",
-	"o1-2024-12-17",
-	"o1-mini",
-	"o1-mini-2024-09-12",
-	"o1-preview",
-	"o1-preview-2024-09-12",
-	"o1-pro",
-	"o1-pro-2025-03-19",
-	"o3",
-	"o3-2025-04-16",
-	"o3-mini",
-	"o3-mini-2025-01-31",
-	"o3-pro",
-	"o3-pro-2025-06-10",
-	"o4-mini",
-	"o4-mini-2025-04-16"
-];
-
-//#endregion
-//#region node_modules/gpt-tokenizer/esm/modelsMap.js
-var modelsMap_exports = /* @__PURE__ */ __export({
-	cl100k_base: () => cl100k_base,
-	o200k_base: () => o200k_base$1,
-	p50k_base: () => p50k_base,
-	p50k_edit: () => p50k_edit,
-	r50k_base: () => r50k_base
-});
-const p50k_base = [
-	"text-davinci-002",
-	"text-davinci-003",
-	"code-davinci-001",
-	"code-davinci-002",
-	"davinci-codex",
-	"code-cushman-001",
-	"code-cushman-002",
-	"cushman-codex"
-];
-const r50k_base = [
-	"text-ada-001",
-	"text-babbage-001",
-	"text-curie-001",
-	"text-davinci-001",
-	"ada",
-	"babbage",
-	"curie",
-	"davinci",
-	"code-search-ada-code-001",
-	"code-search-ada-text-001",
-	"text-similarity-ada-001",
-	"text-search-ada-doc-001",
-	"text-search-ada-query-001",
-	"text-similarity-babbage-001",
-	"text-search-babbage-doc-001",
-	"text-search-babbage-query-001",
-	"code-search-babbage-code-001",
-	"code-search-babbage-text-001",
-	"text-similarity-curie-001",
-	"text-search-curie-doc-001",
-	"text-search-curie-query-001",
-	"text-similarity-davinci-001",
-	"text-search-davinci-doc-001",
-	"text-search-davinci-query-001"
-];
-const p50k_edit = ["code-davinci-edit-001", "text-davinci-edit-001"];
-const cl100k_base = [
-	"gpt-3.5",
-	"gpt-3.5-0301",
-	"gpt-3.5-turbo",
-	"gpt-3.5-turbo-0125",
-	"gpt-3.5-turbo-0613",
-	"gpt-3.5-turbo-1106",
-	"gpt-3.5-turbo-16k-0613",
-	"gpt-3.5-turbo-instruct",
-	"gpt-4",
-	"gpt-4-0125-preview",
-	"gpt-4-0314",
-	"gpt-4-0613",
-	"gpt-4-1106-preview",
-	"gpt-4-1106-vision-preview",
-	"gpt-4-32k",
-	"gpt-4-turbo",
-	"gpt-4-turbo-2024-04-09",
-	"gpt-4-turbo-preview",
-	"text-embedding-3-large",
-	"text-embedding-3-small",
-	"text-embedding-ada-002",
-	"babbage-002",
-	"davinci-002"
-];
-const o200k_base$1 = [];
-
-//#endregion
-//#region node_modules/gpt-tokenizer/esm/specialTokens.js
-const EndOfText = "<|endoftext|>";
-const FimPrefix = "<|fim_prefix|>";
-const FimMiddle = "<|fim_middle|>";
-const FimSuffix = "<|fim_suffix|>";
-const ImStart = "<|im_start|>";
-const ImEnd = "<|im_end|>";
-const ImSep = "<|im_sep|>";
-const EndOfPrompt = "<|endofprompt|>";
-
-//#endregion
-//#region node_modules/gpt-tokenizer/esm/mapping.js
-const o200k_base = "o200k_base";
-const DEFAULT_ENCODING = o200k_base;
-/**
-* maps model names to encoding names
-* if a model is not listed, it uses the default encoding for new models
-* which is `o200k_base`
-*/
-const modelToEncodingMap = Object.fromEntries(Object.entries(modelsMap_exports).flatMap(([encodingName, models]) => models.map((modelName) => [modelName, encodingName])));
-const gpt3params = {
-	messageSeparator: "\n",
-	roleSeparator: "\n"
-};
-const gpt4params = {
-	messageSeparator: "",
-	roleSeparator: ImSep
-};
-const chatModelParams = Object.fromEntries(chatEnabledModels.flatMap((modelName) => modelName.startsWith("gpt-3.5") ? [[modelName, gpt3params]] : [[modelName, gpt4params]]));
-
-//#endregion
-//#region node_modules/gpt-tokenizer/esm/encodingParams/constants.js
-const R50K_TOKEN_SPLIT_REGEX = /'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+/gu;
-const CL_AND_O_TOKEN_SPLIT_PATTERN = /(?:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}{1,3}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+/giu;
-
-//#endregion
-//#region node_modules/gpt-tokenizer/esm/encodingParams/cl100k_base.js
-function Cl100KBase(bytePairRankDecoder) {
-	return {
-		tokenSplitRegex: CL_AND_O_TOKEN_SPLIT_PATTERN,
-		bytePairRankDecoder,
-		specialTokensEncoder: new Map([
-			[EndOfText, 100257],
-			[FimPrefix, 100258],
-			[FimMiddle, 100259],
-			[FimSuffix, 100260],
-			[ImStart, 100264],
-			[ImEnd, 100265],
-			[ImSep, 100266],
-			[EndOfPrompt, 100276]
-		])
-	};
-}
-
-//#endregion
-//#region node_modules/gpt-tokenizer/esm/encodingParams/o200k_base.js
-function O200KBase(bytePairRankDecoder) {
-	return {
-		tokenSplitRegex: CL_AND_O_TOKEN_SPLIT_PATTERN,
-		bytePairRankDecoder,
-		specialTokensEncoder: new Map([
-			[EndOfText, 199999],
-			[FimPrefix, 2e5],
-			[FimMiddle, 200001],
-			[FimSuffix, 200002],
-			[ImStart, 200003],
-			[ImEnd, 200004],
-			[ImSep, 200005],
-			[EndOfPrompt, 200006]
-		])
-	};
-}
-
-//#endregion
-//#region node_modules/gpt-tokenizer/esm/encodingParams/p50k_base.js
-function P50KBase(bytePairRankDecoder) {
-	return {
-		expectedVocabularySize: 50281,
-		tokenSplitRegex: R50K_TOKEN_SPLIT_REGEX,
-		bytePairRankDecoder,
-		specialTokensEncoder: new Map([[EndOfText, 50256]])
-	};
-}
-
-//#endregion
-//#region node_modules/gpt-tokenizer/esm/encodingParams/p50k_edit.js
-function P50KEdit(bytePairRankDecoder) {
-	return {
-		tokenSplitRegex: R50K_TOKEN_SPLIT_REGEX,
-		bytePairRankDecoder,
-		specialTokensEncoder: new Map([
-			[EndOfText, 50256],
-			[FimPrefix, 50281],
-			[FimMiddle, 50282],
-			[FimSuffix, 50283]
-		])
-	};
-}
-
-//#endregion
-//#region node_modules/gpt-tokenizer/esm/encodingParams/r50k_base.js
-function R50KBase(bytePairRankDecoder) {
-	return {
-		expectedVocabularySize: 50257,
-		tokenSplitRegex: R50K_TOKEN_SPLIT_REGEX,
-		bytePairRankDecoder,
-		specialTokensEncoder: new Map([[EndOfText, 50256]])
-	};
-}
-
-//#endregion
-//#region node_modules/gpt-tokenizer/esm/modelParams.js
-function getEncodingParams(encodingName, getMergeableRanks) {
-	const mergeableBytePairRanks = getMergeableRanks(encodingName);
-	switch (encodingName.toLowerCase()) {
-		case "r50k_base": return R50KBase(mergeableBytePairRanks);
-		case "p50k_base": return P50KBase(mergeableBytePairRanks);
-		case "p50k_edit": return P50KEdit(mergeableBytePairRanks);
-		case "cl100k_base": return Cl100KBase(mergeableBytePairRanks);
-		case "o200k_base": return O200KBase(mergeableBytePairRanks);
-		default: throw new Error(`Unknown encoding name: ${encodingName}`);
-	}
-}
-
-//#endregion
-//#region node_modules/gpt-tokenizer/esm/GptEncoding.js
-var GptEncoding = class GptEncoding {
-	static EndOfPrompt = EndOfPrompt;
-	static EndOfText = EndOfText;
-	static FimMiddle = FimMiddle;
-	static FimPrefix = FimPrefix;
-	static FimSuffix = FimSuffix;
-	modelName;
-	modelSpec;
-	bytePairEncodingCoreProcessor;
-	specialTokensEncoder;
-	specialTokensSet;
-	allSpecialTokenRegex;
-	defaultSpecialTokenConfig;
-	vocabularySize;
-	constructor({ bytePairRankDecoder: mergeableBytePairRanks, specialTokensEncoder, expectedVocabularySize, modelName, modelSpec,...rest }) {
-		this.specialTokensEncoder = specialTokensEncoder;
-		this.specialTokensSet = new Set(this.specialTokensEncoder.keys());
-		this.allSpecialTokenRegex = getSpecialTokenRegex(this.specialTokensSet);
-		this.bytePairEncodingCoreProcessor = new BytePairEncodingCore({
-			bytePairRankDecoder: mergeableBytePairRanks,
-			specialTokensEncoder,
-			...rest
-		});
-		this.defaultSpecialTokenConfig = this.processSpecialTokens();
-		const maxTokenValue = Math.max(mergeableBytePairRanks.length - 1, getMaxValueFromMap(specialTokensEncoder));
-		this.vocabularySize = this.bytePairEncodingCoreProcessor.mergeableBytePairRankCount + specialTokensEncoder.size;
-		if (expectedVocabularySize !== void 0) {
-			if (this.vocabularySize !== expectedVocabularySize) throw new Error("The number of mergeable tokens and special tokens must be equal to expectedVocabularySize.");
-			if (maxTokenValue !== expectedVocabularySize - 1) throw new Error(`The model encodings are invalid. The maximum token value must be equal to expectedVocabularySize - 1. Currently ${maxTokenValue}, expected ${expectedVocabularySize - 1}`);
-		}
-		this.encode = this.encode.bind(this);
-		this.decode = this.decode.bind(this);
-		this.encodeGenerator = this.encodeGenerator.bind(this);
-		this.decodeGenerator = this.decodeGenerator.bind(this);
-		this.decodeAsyncGenerator = this.decodeAsyncGenerator.bind(this);
-		this.decodeAsync = this.decodeAsync.bind(this);
-		this.isWithinTokenLimit = this.isWithinTokenLimit.bind(this);
-		this.encodeChat = this.encodeChat.bind(this);
-		this.encodeChatGenerator = this.encodeChatGenerator.bind(this);
-		this.countTokens = this.countTokens.bind(this);
-		this.setMergeCacheSize = this.setMergeCacheSize.bind(this);
-		this.clearMergeCache = this.clearMergeCache.bind(this);
-		this.estimateCost = this.estimateCost.bind(this);
-		this.modelName = modelName;
-		this.modelSpec = modelSpec;
-	}
-	static getEncodingApi(encodingName, getMergeableRanks) {
-		const modelParams = getEncodingParams(encodingName, getMergeableRanks);
-		return new GptEncoding(modelParams);
-	}
-	static getEncodingApiForModel(modelName, getMergeableRanks, modelSpec) {
-		const encodingName = modelToEncodingMap[modelName] ?? DEFAULT_ENCODING;
-		const modelParams = getEncodingParams(encodingName, getMergeableRanks);
-		return new GptEncoding({
-			...modelParams,
-			modelName,
-			modelSpec
-		});
-	}
-	processSpecialTokens({ allowedSpecial, disallowedSpecial } = {}) {
-		let regexPattern;
-		if (allowedSpecial === ALL_SPECIAL_TOKENS || allowedSpecial?.has(ALL_SPECIAL_TOKENS)) {
-			allowedSpecial = new Set(this.specialTokensSet);
-			const allowedSpecialSet = allowedSpecial;
-			if (disallowedSpecial === ALL_SPECIAL_TOKENS) throw new Error("allowedSpecial and disallowedSpecial cannot both be set to \"all\".");
-			if (typeof disallowedSpecial === "object") disallowedSpecial.forEach((val) => allowedSpecialSet.delete(val));
-			else disallowedSpecial = /* @__PURE__ */ new Set();
-		}
-		if (!disallowedSpecial || disallowedSpecial === ALL_SPECIAL_TOKENS || disallowedSpecial.has(ALL_SPECIAL_TOKENS)) {
-			disallowedSpecial = new Set(this.specialTokensSet);
-			const disallowedSpecialSet = disallowedSpecial;
-			if (allowedSpecial?.size) {
-				allowedSpecial.forEach((val) => disallowedSpecialSet.delete(val));
-				disallowedSpecial.forEach((val) => allowedSpecial.delete(val));
-				if (disallowedSpecial.size > 0) regexPattern = getSpecialTokenRegex(disallowedSpecial);
-			} else regexPattern = this.allSpecialTokenRegex;
-		}
-		return {
-			allowedSpecial,
-			regexPattern
-		};
-	}
-	encodeGenerator(lineToEncode, encodeOptions) {
-		const specialTokenConfig = encodeOptions ? this.processSpecialTokens(encodeOptions) : this.defaultSpecialTokenConfig;
-		if (specialTokenConfig.regexPattern) {
-			const match = lineToEncode.match(specialTokenConfig.regexPattern);
-			if (match !== null) throw new Error(`Disallowed special token found: ${match[0]}`);
-		}
-		return this.bytePairEncodingCoreProcessor.encodeNativeGenerator(lineToEncode, specialTokenConfig.allowedSpecial);
-	}
-	encode(lineToEncode, encodeOptions) {
-		const specialTokenConfig = encodeOptions ? this.processSpecialTokens(encodeOptions) : this.defaultSpecialTokenConfig;
-		if (specialTokenConfig.regexPattern) {
-			const match = lineToEncode.match(specialTokenConfig.regexPattern);
-			if (match !== null) throw new Error(`Disallowed special token found: ${match[0]}`);
-		}
-		return this.bytePairEncodingCoreProcessor.encodeNative(lineToEncode, specialTokenConfig.allowedSpecial);
-	}
-	/**
-	* Progressively tokenizes an OpenAI chat.
-	* Warning: gpt-3.5-turbo and gpt-4 chat format may change over time.
-	* Returns tokens assuming the 'gpt-3.5-turbo-0301' / 'gpt-4-0314' format.
-	* Based on OpenAI's guidelines: https://github.com/openai/openai-python/blob/main/chatml.md
-	* Also mentioned in section 6 of this document: https://github.com/openai/openai-cookbook/blob/main/examples/How_to_count_tokens_with_tiktoken.ipynb
-	*/
-	*encodeChatGenerator(chat, model = this.modelName) {
-		if (!model) throw new Error("Model name must be provided either during initialization or passed in to the method.");
-		const params = chatModelParams[model];
-		const chatStartToken = this.specialTokensEncoder.get(ImStart);
-		const chatEndToken = this.specialTokensEncoder.get(ImEnd);
-		if (!params || chatStartToken === void 0 || chatEndToken === void 0) throw new Error(`Model '${model}' does not support chat.`);
-		const allowedSpecial = new Set([ImSep]);
-		const { messageSeparator, roleSeparator } = params;
-		const encodedMessageSeparator = messageSeparator.length > 0 ? this.encode(messageSeparator) : [];
-		const encodedRoleSeparator = roleSeparator.length > 0 ? this.encode(roleSeparator, { allowedSpecial }) : [];
-		const nameCache = /* @__PURE__ */ new Map();
-		for (const { role = "system", name = role, content } of chat) {
-			if (content === void 0) throw new Error("Content must be defined for all messages.");
-			yield [chatStartToken];
-			const encodedName = nameCache.get(name) ?? this.encode(name);
-			nameCache.set(name, encodedName);
-			yield encodedName;
-			if (encodedRoleSeparator.length > 0) yield encodedRoleSeparator;
-			yield* this.encodeGenerator(content);
-			yield [chatEndToken];
-			yield encodedMessageSeparator;
-		}
-		yield [chatStartToken];
-		yield* this.encodeGenerator("assistant");
-		if (encodedRoleSeparator.length > 0) yield encodedRoleSeparator;
-	}
-	/**
-	* Encodes a chat into a single array of tokens.
-	* Warning: gpt-3.5-turbo and gpt-4 chat format may change over time.
-	* Returns tokens assuming the 'gpt-3.5-turbo-0301' / 'gpt-4-0314' format.
-	* Based on OpenAI's guidelines: https://github.com/openai/openai-python/blob/main/chatml.md
-	* Also mentioned in section 6 of this document: https://github.com/openai/openai-cookbook/blob/main/examples/How_to_count_tokens_with_tiktoken.ipynb
-	*/
-	encodeChat(chat, model = this.modelName) {
-		return [...this.encodeChatGenerator(chat, model)].flat();
-	}
-	/**
-	* @returns {false | number} false if token limit is exceeded, otherwise the number of tokens
-	*/
-	isWithinTokenLimit(input, tokenLimit) {
-		const tokenGenerator = typeof input === "string" ? this.encodeGenerator(input) : this.encodeChatGenerator(input);
-		let count = 0;
-		for (const tokens of tokenGenerator) {
-			count += tokens.length;
-			if (count > tokenLimit) return false;
-		}
-		return count;
-	}
-	/**
-	* Counts the number of tokens in the input.
-	* @returns {number} The number of tokens.
-	*/
-	countTokens(input, encodeOptions) {
-		if (typeof input === "string") {
-			const specialTokenConfig = encodeOptions ? this.processSpecialTokens(encodeOptions) : this.defaultSpecialTokenConfig;
-			if (specialTokenConfig.regexPattern) {
-				const match = input.match(specialTokenConfig.regexPattern);
-				if (match !== null) throw new Error(`Disallowed special token found: ${match[0]}`);
-			}
-			return this.bytePairEncodingCoreProcessor.countNative(input, specialTokenConfig.allowedSpecial);
-		}
-		const tokenGenerator = this.encodeChatGenerator(input);
-		let count = 0;
-		for (const tokens of tokenGenerator) count += tokens.length;
-		return count;
-	}
-	setMergeCacheSize(size) {
-		this.bytePairEncodingCoreProcessor.setMergeCacheSize(size);
-	}
-	clearMergeCache() {
-		this.bytePairEncodingCoreProcessor.clearMergeCache();
-	}
-	decode(inputTokensToDecode) {
-		return this.bytePairEncodingCoreProcessor.decodeNative(inputTokensToDecode);
-	}
-	*decodeGenerator(inputTokensToDecode) {
-		const decodedByteGenerator = this.bytePairEncodingCoreProcessor.decodeNativeGenerator(inputTokensToDecode);
-		let buffer = "";
-		for (const decodedPart of decodedByteGenerator) {
-			buffer += typeof decodedPart === "string" ? decodedPart : decoder.decode(decodedPart, { stream: true });
-			if (buffer.length === 0 || endsWithIncompleteUtfPairSurrogate(buffer)) continue;
-			else {
-				yield buffer;
-				buffer = "";
-			}
-		}
-		if (buffer.length > 0) yield buffer;
-	}
-	async *decodeAsyncGenerator(inputTokensToDecode) {
-		const decodedByteGenerator = this.bytePairEncodingCoreProcessor.decodeNativeAsyncIterable(inputTokensToDecode);
-		let buffer = "";
-		for await (const decodedPart of decodedByteGenerator) {
-			buffer += typeof decodedPart === "string" ? decodedPart : decoder.decode(decodedPart, { stream: true });
-			if (buffer.length === 0 || endsWithIncompleteUtfPairSurrogate(buffer)) continue;
-			else {
-				yield buffer;
-				buffer = "";
-			}
-		}
-		if (buffer.length > 0) yield buffer;
-	}
-	async decodeAsync(inputTokensToDecode) {
-		const decodedByteGenerator = this.bytePairEncodingCoreProcessor.decodeNativeAsyncIterable(inputTokensToDecode);
-		let buffer = "";
-		for await (const decodedPart of decodedByteGenerator) buffer += typeof decodedPart === "string" ? decodedPart : decoder.decode(decodedPart, { stream: true });
-		return buffer;
-	}
-	/**
-	* Estimates the cost of processing a given token count using the model's pricing.
-	*
-	* @param tokenCount - The number of tokens to estimate cost for
-	* @returns Cost estimate object with applicable price components (input, output, batchInput, batchOutput)
-	*/
-	estimateCost(tokenCount, modelSpec = this.modelSpec) {
-		if (!modelSpec) throw new Error("Model spec must be provided either during initialization or passed in to the method.");
-		if (!modelSpec.price_data) throw new Error(`No cost information available for model: ${modelSpec.name}`);
-		const priceDataPerMillion = modelSpec.price_data;
-		const result = {};
-		const millionTokens = tokenCount / 1e6;
-		if (priceDataPerMillion.main) result.main = {
-			input: priceDataPerMillion.main.input && priceDataPerMillion.main.input * millionTokens,
-			output: priceDataPerMillion.main.output && priceDataPerMillion.main.output * millionTokens,
-			cached_input: priceDataPerMillion.main.cached_input && priceDataPerMillion.main.cached_input * millionTokens,
-			cached_output: priceDataPerMillion.main.cached_output && priceDataPerMillion.main.cached_output * millionTokens
-		};
-		if (priceDataPerMillion.batch) result.batch = {
-			input: priceDataPerMillion.batch.input && priceDataPerMillion.batch.input * millionTokens,
-			output: priceDataPerMillion.batch.output && priceDataPerMillion.batch.output * millionTokens,
-			cached_input: priceDataPerMillion.batch.cached_input && priceDataPerMillion.batch.cached_input * millionTokens,
-			cached_output: priceDataPerMillion.batch.cached_output && priceDataPerMillion.batch.cached_output * millionTokens
-		};
-		return result;
-	}
-};
-
-//#endregion
-export { ALL_SPECIAL_TOKENS, DEFAULT_MERGE_CACHE_SIZE, EndOfPrompt, EndOfText, FimMiddle, FimPrefix, FimSuffix, GptEncoding, ImEnd, ImSep, ImStart };
-//# sourceMappingURL=GptEncoding-TiCgsNrQ.js.map