@genai-fi/nanogpt 0.21.0 → 0.23.0

package/dist/Trainer-DBsyWJ4s.js

@@ -0,0 +1,228 @@
+import { packingSupported as e } from "./utilities/packed.js";
+import { t } from "./eventemitter3-D_qV3Lof.js";
+import n from "./training/PreTrainer.js";
+import { TokenStore as r } from "./training/tasks/TokenStore.js";
+import { tokensFromTasks as i } from "./training/tasks/Task.js";
+import { createTrainValidationDatasets as a, storeFromArray as o } from "./training/validation.js";
+import s from "./training/SFTTrainer.js";
+//#region node_modules/uuid/dist/stringify.js
+var c = [];
+for (let e = 0; e < 256; ++e) c.push((e + 256).toString(16).slice(1));
+function l(e, t = 0) {
+	return (c[e[t + 0]] + c[e[t + 1]] + c[e[t + 2]] + c[e[t + 3]] + "-" + c[e[t + 4]] + c[e[t + 5]] + "-" + c[e[t + 6]] + c[e[t + 7]] + "-" + c[e[t + 8]] + c[e[t + 9]] + "-" + c[e[t + 10]] + c[e[t + 11]] + c[e[t + 12]] + c[e[t + 13]] + c[e[t + 14]] + c[e[t + 15]]).toLowerCase();
+}
+//#endregion
+//#region node_modules/uuid/dist/rng.js
+var u = new Uint8Array(16);
+function d() {
+	return crypto.getRandomValues(u);
+}
+//#endregion
+//#region node_modules/uuid/dist/v4.js
+function f(e, t, n) {
+	return !t && !e && crypto.randomUUID ? crypto.randomUUID() : p(e, t, n);
+}
+function p(e, t, n) {
+	e ||= {};
+	let r = e.random ?? e.rng?.() ?? d();
+	if (r.length < 16) throw Error("Random bytes length must be >= 16");
+	if (r[6] = r[6] & 15 | 64, r[8] = r[8] & 63 | 128, t) {
+		if (n ||= 0, n < 0 || n + 16 > t.length) throw RangeError(`UUID byte range ${n}:${n + 15} is out of buffer bounds`);
+		for (let e = 0; e < 16; ++e) t[n + e] = r[e];
+		return t;
+	}
+	return l(r);
+}
+//#endregion
+//#region lib/Trainer.ts
+var m = class c extends t {
+	trainer;
+	trainingType = "pretraining";
+	hasTrained = !1;
+	trainDataset;
+	validationDataset;
+	totalTokens = 0;
+	tokensProcessed = 0;
+	log = [];
+	progress = null;
+	options = {
+		batchSize: 32,
+		sftMode: "full",
+		logInterval: 10
+	};
+	tokenizer;
+	constructor(t, r, i, a, o) {
+		if (super(), t instanceof c) {
+			let e = r || t.options, a = t.options, o = !1;
+			if (t.trainingType === "sft" && e.sftMode !== a.sftMode && (o = !0), t.trainer instanceof s && t.trainer.loraName && e.loraName !== t.trainer.loraName && (o = !0), i !== void 0 && i !== t.trainingType && (o = !0), o) {
+				if (t.trainingType === "sft") {
+					let n = new s(t.model, t.tokenizer, e);
+					this.trainer = n, n.loraName = e.loraName;
+				} else this.trainer = new n(t.model, t.tokenizer, e);
+				this.trainingType = i || t.trainingType, this.options = e, this.tokenizer = t.tokenizer;
+			} else this.trainer = t.trainer, this.trainingType = i || t.trainingType, this.options = e, this.trainer.updateOptimizer(this.options), this.log = t.log, this.progress = t.progress, this.totalTokens = t.totalTokens, this.tokenizer = t.tokenizer, e.batchSize === a.batchSize && (this.trainDataset = t.trainDataset, this.validationDataset = t.validationDataset);
+			return;
+		}
+		if (!r) throw Error("Tokeniser must be provided when initializing Trainer with a model");
+		if (!t) throw Error("Model must be provided when initializing Trainer");
+		this.options = a || {
+			batchSize: 32,
+			sftMode: "full"
+		};
+		let l = this.options.mixedPrecision && e();
+		if (this.options.lossScaling = l ? t.lossScaling : 1, i === "sft") {
+			let e = new s(t, r, this.options, o);
+			this.trainer = e, e.loraName = a?.loraName;
+		} else this.trainer = new n(t, r, this.options, o);
+		this.trainingType = i || "pretraining", this.tokenizer = r;
+	}
+	get model() {
+		return this.trainer.model;
+	}
+	get optimizer() {
+		return this.trainer.optimizer;
+	}
+	get isTraining() {
+		return this.trainer.isRunning;
+	}
+	stop() {
+		this.trainer.stop();
+	}
+	reset() {
+		this.hasTrained = !1, this.log = [], this.trainer.reset();
+	}
+	dispose() {
+		this.trainer.dispose(), this.removeAllListeners();
+	}
+	getTotalTokens() {
+		return this.totalTokens;
+	}
+	setOptions(e) {
+		let t = new Set(Object.keys(e).filter((t) => e[t] !== this.options[t]));
+		if (this.trainer.isRunning) {
+			if (t.has("batchSize")) throw Error("Cannot change batch size during training");
+			if (t.has("sftMode")) throw Error("Cannot change SFT mode during training");
+			if (t.has("loraConfig")) throw Error("Cannot change LoRA configuration during training");
+			if (t.has("validationSplit")) throw Error("Cannot change validation split during training");
+			if (t.has("trainableWeights")) throw Error("Cannot change trainable weights during training");
+			if (t.has("mixedPrecision")) throw Error("Cannot change mixed precision setting during training");
+			if (t.has("gradientCheckpointing")) throw Error("Cannot change gradient checkpointing setting during training");
+		}
+		this.options = {
+			...this.options,
+			...e
+		}, this.trainer.updateOptimizer(this.options), t.has("metrics") && this.trainer.setMetrics(e.metrics || []);
+	}
+	async prepare(e = [], t, n) {
+		let c = this.options, l = c.loraName || c.loraConfig;
+		if (n && l) throw Error("Cannot specify datasets when using LoRA fine-tuning");
+		if (!n && !l) throw Error("Must specify datasets for non-LoRA training");
+		if (n) {
+			let e = this.model.metaData.pretrainingData || [], t = [...e], r = !1;
+			for (let i of n) e.some((e) => e.id === i.id) || t.push({
+				id: i.id,
+				name: i.name,
+				conversational: i.conversational
+			}), i.conversational && (r = !0);
+			this.model.metaData.pretrainingData = t, r ? this.model.metaData.mode = "conversational" : this.model.metaData.mode !== "conversational" && (this.model.metaData.mode = "completion");
+		} else this.model.metaData.mode !== "conversational" && (this.model.metaData.mode = "completion");
+		let u = c.maskedLoss ?? this.trainingType === "sft";
+		if (this.trainingType === "sft" && this.trainer instanceof s && e instanceof Uint16Array) throw Error("SFT training requires Task[] input");
+		let d, f = t;
+		if (Array.isArray(e)) if (e[0] instanceof Uint16Array) d = e;
+		else {
+			let n = await i(e, this.trainer.tokenizer, {
+				masking: u,
+				validationSplit: c.validationSplit
+			});
+			d = n.trainingTokens, t || (f = n.validationTokens);
+		}
+		else d = e;
+		let p = d instanceof r ? d.getTokenCount() : d.reduce((e, t) => e + t.length, 0);
+		if (f) {
+			let { trainDataset: e, validationDataset: t } = await a(d, f, this.trainer.tokenizer, this.trainer.datasetBuilder, c?.batchSize || 32);
+			this.trainDataset = e, this.validationDataset = t;
+		} else {
+			let e = d instanceof r ? d : await o(d, this.trainer.tokenizer);
+			this.trainDataset = (await this.trainer.datasetBuilder.createTextDataset(e, c)).dataset;
+		}
+		this.totalTokens = p, this.options.epochSteps = Math.ceil(this.totalTokens / ((c?.batchSize || 32) * this.model.config.blockSize)), this.trainer.updateOptimizer(this.options);
+	}
+	configureModel(e) {
+		let t = e?.sftMode || "full";
+		if (this.trainingType === "pretraining" && (this.trainer.model.hasLoRA() && this.trainer.model.detachLoRA(), this.trainer.model.weightStore.setTrainable(["*"])), this.trainingType === "sft") {
+			if (t === "lora") {
+				let t = this.trainer.model;
+				if (e?.loraName) {
+					if (!t.hasLoRA(e.loraName)) if (e.loraConfig) t.createLoRA(e.loraName, e.loraConfig), t.attachLoRA(e.loraName);
+					else throw Error(`LoRA configuration must be provided to create LoRA with name ${e.loraName}`);
+					else if (t.attachLoRA(e.loraName), e.loraConfig) {
+						let n = t.lora;
+						(n.alpha !== e.loraConfig.alpha || n.rank !== e.loraConfig.rank) && (t.detachLoRA(), t.deleteLoRA(e.loraName), t.createLoRA(e.loraName, e.loraConfig), t.attachLoRA(e.loraName), console.warn("Resetting LoRA with new configuration."));
+					}
+				} else if (e?.loraConfig) if (t.hasLoRA()) {
+					let n = t.lora;
+					if (n.alpha !== e.loraConfig.alpha || n.rank !== e.loraConfig.rank) {
+						t.detachLoRA();
+						let n = e.loraName || f();
+						t.createLoRA(n, e.loraConfig), t.attachLoRA(n);
+					}
+				} else {
+					let n = e.loraName || f();
+					t.createLoRA(n, e.loraConfig), t.attachLoRA(n);
+				}
+				else if (!t.hasLoRA()) throw Error("LoRA configuration must be provided for lora SFT mode");
+			} else this.trainer.model.hasLoRA() && this.trainer.model.detachLoRA();
+			t === "last-layer" ? this.trainer.model.weightStore.setTrainable([`block_${this.trainer.model.config.nLayer - 1}_*`, "token_embedding"]) : t === "full" && this.trainer.model.weightStore.setTrainable(["*"]);
+		}
+		e?.trainableWeights && this.trainer.model.weightStore.setTrainable(e.trainableWeights);
+	}
+	async train() {
+		let e = this.options;
+		if (!this.trainDataset) throw Error("Dataset not prepared");
+		this.hasTrained || this.trainer.setLearningRate(e?.learningRate || .001), this.hasTrained = !0, this.emit("start"), this.model.metaData.pretrainingSettings = e;
+		let t = Date.now();
+		this.log.length > 0 && this.trainer.resumeFromLog(this.log[this.log.length - 1]), this.trainer.setGradientCheckpointing(e?.gradientCheckpointing || !1), this.trainer.setMixedPrecision(e?.mixedPrecision || !1), this.trainer.setLabelSmoothing(e?.labelSmoothing || 0), this.trainer.setDropout(e?.dropout || 0), this.trainer.setLayerDrop(e?.layerDrop || 0), this.configureModel(e), await this.trainer.trainOnDataset(this.trainDataset, {
+			...e,
+			onStep: async (e) => {
+				this.log.push(e), this.progress = {
+					lastLog: e,
+					progress: e.totalTokens / this.totalTokens,
+					remaining: Math.max(0, (this.totalTokens - e.totalTokens) / e.totalTokens * e.duration)
+				}, this.tokensProcessed = e.totalTokens;
+				let t = this.listeners("log");
+				for (let n of t) await n(e, this.progress);
+			}
+		}, this.validationDataset), this.model.metaData.actionLog = this.model.metaData.actionLog || [];
+		let n = Date.now();
+		this.model.metaData.actionLog.push({
+			action: "pretrain",
+			timestamp: n,
+			duration: n - t,
+			tokensProcessed: this.tokensProcessed,
+			options: e
+		}), this.emit("stop");
+	}
+	async step(e) {
+		if (!this.trainDataset) throw Error("Dataset not prepared");
+		this.hasTrained || this.trainer.setLearningRate(e?.learningRate || .001), this.hasTrained = !0, this.emit("start");
+		let { log: t } = await this.trainer.stepDataset(this.trainDataset, e || {}, this.validationDataset), n = this.listeners("log");
+		for (let e of n) await e(t, {
+			lastLog: t,
+			progress: t.totalTokens / this.totalTokens,
+			remaining: Math.max(0, (this.totalTokens - t.totalTokens) / t.totalTokens * t.duration)
+		});
+		this.emit("stop");
+	}
+	getLog() {
+		return this.log;
+	}
+	getProgress() {
+		return this.progress;
+	}
+	isPrepared() {
+		return this.trainDataset !== void 0 && this.validationDataset !== void 0;
+	}
+};
+//#endregion
+export { m as t };

package/dist/Trainer.d.ts

@@ -5,6 +5,7 @@ import { Task } from './training/tasks/Task';
 import { TrainingOptions, TrainingLogEntry } from './training/types';
 import { AdamWOptimizer } from './training/AdamW';
 import { DatasetMetadata } from './loader/types';
+import { TokenStore } from './training/tasks/TokenStore';
 interface TrainingProgress {
     lastLog: TrainingLogEntry;
     progress: number;
@@ -33,7 +34,7 @@ export default class Trainer extends EE<'start' | 'stop' | 'log'> {
     dispose(): void;
     getTotalTokens(): number;
     setOptions(options: TrainingOptions): void;
-    prepare(tasks?: Task[] | Uint16Array, datasets?: DatasetMetadata[]): Promise<void>;
+    prepare(tasks?: Task[] | Uint16Array[] | TokenStore, validation?: Uint16Array[] | TokenStore, datasets?: DatasetMetadata[]): Promise<void>;
     private configureModel;
     train(): Promise<void>;
     step(options?: TrainingOptions): Promise<void>;

package/dist/Trainer.js

@@ -1,2 +1,2 @@
-import { a as e } from "./main-DdsoTJAW.js";
+import { t as e } from "./Trainer-DBsyWJ4s.js";
 export { e as default };

package/dist/data/stream.d.ts

@@ -0,0 +1,24 @@
+import { Conversation } from '../../tokeniser/type';
+export interface ConversationCursor {
+    next(): Promise<Conversation[] | null>;
+}
+export interface ConversationStream {
+    cursor(): ConversationCursor;
+}
+export declare class MemoryConversationStream implements ConversationStream {
+    private conversations;
+    constructor(conversations: Conversation[][]);
+    cursor(): ConversationCursor;
+}
+declare class JSONLFromReadableStream implements ConversationStream {
+    private sourceFactory;
+    constructor(sourceFactory: () => Promise<ReadableStream<Uint8Array>>);
+    cursor(): ConversationCursor;
+}
+export declare class JSONLConversationStream extends JSONLFromReadableStream {
+    constructor(file: File);
+}
+export declare class ZipJSONLConversationStream extends JSONLFromReadableStream {
+    constructor(file: File, preferredEntryName?: string);
+}
+export {};

package/dist/data/stream.js

@@ -0,0 +1,2 @@
+import { n as e, r as t, t as n } from "../stream-BjdpSNqB.js";
+export { n as JSONLConversationStream, e as MemoryConversationStream, t as ZipJSONLConversationStream };

package/dist/data/textLoader.d.ts

@@ -1,7 +1,7 @@
-import { Conversation } from '../tokeniser/type';
+import { ConversationStream } from './stream';
 export interface DataOptions {
     maxSize?: number;
     column?: string;
     hasHeader?: boolean;
 }
-export default function loadTextData(file: File, options?: DataOptions, cb?: (progress: number) => void): Promise<Conversation[][]>;
+export default function loadTextData(file: File, options?: DataOptions): Promise<ConversationStream>;

package/dist/data/textLoader.js

@@ -1,9 +1,10 @@
 import { i as e, t } from "../chunk-CWhphoD1.js";
 import { loadPDF as n } from "./pdf.js";
-import { loadDOCX as r, t as i } from "./docx.js";
+import { loadDOCX as r } from "./docx.js";
+import { n as i, r as a, t as o } from "../stream-BjdpSNqB.js";
 //#endregion
 //#region lib/data/textLoader.ts
-var a = /* @__PURE__ */ e((/* @__PURE__ */ t(((e, t) => {
+var s = /* @__PURE__ */ e((/* @__PURE__ */ t(((e, t) => {
 	((n, r) => {
 		typeof define == "function" && define.amd ? define([], r) : typeof t == "object" && e !== void 0 ? t.exports = r() : n.Papa = r();
 	})(e, function e() {
@@ -496,19 +497,19 @@ var a = /* @__PURE__ */ e((/* @__PURE__ */ t(((e, t) => {
 			});
 		}), (l.prototype = Object.create(c.prototype)).constructor = l, (u.prototype = Object.create(c.prototype)).constructor = u, (d.prototype = Object.create(d.prototype)).constructor = d, (f.prototype = Object.create(c.prototype)).constructor = f, s;
 	});
-})))(), 1), o = /* @__PURE__ */ e(i(), 1);
-function s(e, t) {
+})))(), 1);
+function c(e, t) {
 	let n = e.findIndex((e) => e.toLowerCase() === t.toLowerCase());
 	return n === -1 ? 0 : n;
 }
-function c(e) {
+function l(e) {
 	return e.every((e) => e.length < 64);
 }
-function l(e) {
+function u(e) {
 	return e.split(".").pop() || "";
 }
-function u(e) {
-	switch (l(e)) {
+function d(e) {
+	switch (u(e)) {
 		case "json": return "application/json";
 		case "jsonl": return "application/jsonl";
 		case "parquet": return "application/parquet";
@@ -520,19 +521,14 @@ function u(e) {
 		default: return "unknown";
 	}
 }
-function d(e) {
-	if (!Array.isArray(e)) return !1;
-	let t = e[0];
-	return typeof t == "object" && !!t && "role" in t && "content" in t && typeof t.role == "string" && typeof t.content == "string";
-}
-async function f(e, t, i) {
-	let l = e.type === "" ? u(e.name) : e.type;
-	if (l === "application/parquet") throw Error("Parquet loading is not currently supported in the browser. Please convert your data to JSONL format.");
-	if (l === "application/pdf") return n(e, t?.maxSize);
-	if (l === "application/vnd.openxmlformats-officedocument.wordprocessingml.document") return r(e);
-	if (l === "application/json") {
+async function f(e, t) {
+	let u = e.type === "" ? d(e.name) : e.type;
+	if (u === "application/parquet") throw Error("Parquet loading is not currently supported in the browser. Please convert your data to JSONL format.");
+	if (u === "application/pdf") return new i(await n(e, t?.maxSize));
+	if (u === "application/vnd.openxmlformats-officedocument.wordprocessingml.document") return new i(await r(e));
+	if (u === "application/json") {
 		let t = await e.text(), n = JSON.parse(t);
-		if (Array.isArray(n)) return n.map((e) => [typeof e == "string" ? {
+		if (Array.isArray(n)) return new i(n.map((e) => [typeof e == "string" ? {
 			role: "text",
 			content: e
 		} : "text" in e ? {
@@ -541,61 +537,26 @@ async function f(e, t, i) {
 		} : {
 			role: "text",
 			content: JSON.stringify(e)
-		}]);
+		}]));
 		throw Error("Expected JSON array");
 	}
-	if (l === "application/jsonl") {
-		let t = await e.text();
-		return i && i(.1), t.split("\n").filter((e) => e.trim() !== "").map((e, t, n) => {
-			i && t % 1e3 == 0 && i(.1 + t / n.length * .9);
-			try {
-				let t = JSON.parse(e);
-				return d(t) ? t : [typeof t == "string" ? {
-					role: "text",
-					content: t
-				} : "text" in t ? {
-					role: "text",
-					content: t.text
-				} : {
-					role: "text",
-					content: JSON.stringify(t)
-				}];
-			} catch {
-				return [{
-					role: "text",
-					content: e
-				}];
-			}
-		});
-	}
-	if (l === "application/zip") {
-		let n = await o.default.loadAsync(e), r = [], a = Object.keys(n.files);
-		for (let e = 0; e < a.length; e++) {
-			let o = a[e], s = n.file(o);
-			if (s) {
-				let n = await s.async("blob", (t) => {
-					i && i(.1 + (t.percent / 100 * .9 / a.length + e / a.length * .9));
-				}), c = await f(new File([n], o), t);
-				i && i(.1 + (e + 1) / a.length * .9), r = r.concat(c);
-			}
-		}
-		return r;
-	}
-	if (l === "text/csv") {
+	if (u === "application/jsonl") return new o(e);
+	if (u === "application/zip") return new a(e);
+	if (u === "text/csv") {
 		let n = await e.text();
-		return i && i(.1), new Promise((e, r) => {
-			a.default.parse(n, {
+		return new Promise((e, r) => {
+			s.default.parse(n, {
 				header: !1,
 				skipEmptyLines: !0,
 				delimiter: ",",
 				complete: (n) => {
 					if (n.errors.length > 0) console.error(n.errors), r(/* @__PURE__ */ Error("Error parsing file"));
 					else {
-						let r = s(n.data[0], t?.column || "text");
-						e((t?.hasHeader ?? c(n.data[0]) ? n.data.slice(1) : n.data).map((e) => [{
+						let r = c(n.data[0], t?.column || "text");
+						e(new i((t?.hasHeader ?? l(n.data[0]) ? n.data.slice(1) : n.data).map((e) => [{
 							role: "text",
 							content: e[r]
-						}]));
+						}])));
 					}
 				},
 				error: (e) => {
@@ -603,11 +564,11 @@ async function f(e, t, i) {
 				}
 			});
 		});
-	} else if (l === "text/plain") return [[{
+	} else if (u === "text/plain") return new i([[{
 		role: "text",
 		content: await e.text()
-	}]];
-	throw Error(`Unsupported file type: ${l}`);
+	}]]);
+	throw Error(`Unsupported file type: ${u}`);
 }
 //#endregion
 export { f as default };

package/dist/{BaseTokeniser-C9TSv4th.js → eventemitter3-D_qV3Lof.js}

@@ -86,136 +86,6 @@ var n = (/* @__PURE__ */ e((/* @__PURE__ */ t(((e, t) => {
 		var t;
 		return e ? (t = r ? r + e : e, this._events[t] && s(this, t)) : (this._events = new i(), this._eventsCount = 0), this;
 	}, c.prototype.off = c.prototype.removeListener, c.prototype.addListener = c.prototype.on, c.prefixed = r, c.EventEmitter = c, t !== void 0 && (t.exports = c);
-})))(), 1)).default, r = [
-	"<eos>",
-	"<bos>",
-	"",
-	"<pad>",
-	"<|user_start|>",
-	"<|user_end|>",
-	"<|assistant_start|>",
-	"<|assistant_end|>",
-	"<|system_start|>",
-	"<|system_end|>"
-], i = class extends n {
-	id = "untrained";
-	datasetID;
-	specialTokens = /* @__PURE__ */ new Map();
-	specialTokenSet = /* @__PURE__ */ new Set();
-	isSpecialToken(e) {
-		return this.specialTokenSet.has(e);
-	}
-	addSpecialTokens() {
-		r.forEach((e, t) => {
-			this.addToken(e, t), this.specialTokens.set(e, t), this.specialTokenSet.add(t);
-		});
-	}
-	addSpecialToken(e, t) {
-		this.specialTokens.set(e, t), this.specialTokenSet.add(t);
-	}
-	generateID() {
-		let e = this.getVocab(), t = 2166136261, n = 2654435769;
-		if (e.length === 0) {
-			this.id = "untrained";
-			return;
-		}
-		for (let r = 0; r < e.length; r++) {
-			let i = e[r];
-			t ^= i.length, t = Math.imul(t, 16777619), n ^= r, n = Math.imul(n, 2246822507);
-			for (let e = 0; e < i.length; e++) {
-				let r = i.charCodeAt(e);
-				t ^= r, t = Math.imul(t, 16777619), n ^= r, n = Math.imul(n, 3266489909);
-			}
-		}
-		let r = (t >>> 0).toString(36), i = (n >>> 0).toString(36);
-		this.id = "tokeniser_" + r + "_" + i;
-	}
-	encodeSequence(e) {
-		let t = this.encode(e);
-		return [
-			this.bosToken,
-			...t,
-			this.eosToken
-		];
-	}
-	encodeAsSequence(e, t) {
-		let n = e.flatMap((e) => this.encode(e.content));
-		return t ? [
-			this.bosToken,
-			...n,
-			this.eosToken,
-			this.bosToken
-		] : [
-			this.bosToken,
-			...n,
-			this.eosToken
-		];
-	}
-	encodeConversation(e, t, n) {
-		let r = [[this.bosToken]], i;
-		n && (i = [[!1]]);
-		let a = [
-			this.getSpecialTokenIndex("<|user_start|>"),
-			this.getSpecialTokenIndex("<|assistant_start|>"),
-			this.getSpecialTokenIndex("<|system_start|>")
-		], o = [
-			this.getSpecialTokenIndex("<|user_end|>"),
-			this.getSpecialTokenIndex("<|assistant_end|>"),
-			this.getSpecialTokenIndex("<|system_end|>")
-		];
-		for (let t of e) {
-			let e = !1, s = this.encode(t.content);
-			switch (t.role) {
-				case "user":
-					r.push([a[0]]), e = !0;
-					break;
-				case "assistant":
-					r.push([a[1]]);
-					break;
-				case "system":
-					r.push([a[2]]), e = !0;
-					break;
-			}
-			switch (r.push(s), t.role) {
-				case "user":
-					r.push([o[0]]);
-					break;
-				case "assistant":
-					r.push([o[1]]);
-					break;
-				case "system":
-					r.push([o[2]]);
-					break;
-			}
-			n && i && e ? (i.push([!1]), i.push(s.map(() => !1)), i.push([!1])) : n && i && (i.push([!1]), i.push(s.map(() => !0)), i.push([!0]));
-		}
-		let s = r.flat();
-		return t ? (s.push(a[1]), n && i && i.push([!1])) : (s.push(this.eosToken), n && i && i.push([!0])), n && i ? {
-			tokens: s,
-			mask: i.flat()
-		} : s;
-	}
-	decodeConversation(e) {
-		let t = [], n = 0;
-		for (; n < e.length;) {
-			let r = e[n], i = null;
-			if (r === this.getSpecialTokenIndex("<|user_start|>") ? i = "user" : r === this.getSpecialTokenIndex("<|assistant_start|>") ? i = "assistant" : r === this.getSpecialTokenIndex("<|system_start|>") ? i = "system" : r === this.bosToken || (r === this.eosToken ? i = null : (i = "text", n--)), i) {
-				n++;
-				let r = [];
-				for (; n < e.length && e[n] !== this.getSpecialTokenIndex(`<|${i}_end|>`) && e[n] !== this.eosToken;) r.push(e[n]), n++;
-				let a = this.decode(r);
-				t.push({
-					role: i,
-					content: a
-				});
-			}
-			n++;
-		}
-		return t;
-	}
-	getSpecialTokenIndex(e) {
-		return this.specialTokens.get(e);
-	}
-};
+})))(), 1)).default;
 //#endregion
-export { r as n, n as r, i as t };
+export { n as t };

package/dist/loader/load.js

@@ -1,2 +1,2 @@
-import { d as e, u as t } from "../main-DdsoTJAW.js";
-export { t as VERSION, e as loadModel };
+import { c as e, l as t } from "../main-BSaDGH7I.js";
+export { e as VERSION, t as loadModel };

package/dist/loader/loadHF.js

@@ -1,2 +1,2 @@
-import { f as e } from "../main-DdsoTJAW.js";
+import { u as e } from "../main-BSaDGH7I.js";
 export { e as default };

package/dist/loader/loadTransformers.js

@@ -1,2 +1,2 @@
-import { h as e, m as t } from "../main-DdsoTJAW.js";
-export { t as default, e as mapTransformersConfigToGPTConfig };
+import { f as e, p as t } from "../main-BSaDGH7I.js";
+export { e as default, t as mapTransformersConfigToGPTConfig };

package/dist/loader/newZipLoad.js

@@ -1,2 +1,2 @@
-import { p as e } from "../main-DdsoTJAW.js";
+import { d as e } from "../main-BSaDGH7I.js";
 export { e as default };

package/dist/loader/oldZipLoad.js

@@ -1,2 +1,2 @@
-import { g as e } from "../main-DdsoTJAW.js";
+import { m as e } from "../main-BSaDGH7I.js";
 export { e as default };

package/dist/loader/save.js

@@ -1,2 +1,2 @@
-import { l as e } from "../main-DdsoTJAW.js";
+import { s as e } from "../main-BSaDGH7I.js";
 export { e as saveModel };