@genai-fi/nanogpt 0.22.0 → 0.24.0

package/dist/{DatasetBuilder-Ctb425Id.js → DatasetBuilder-DU1G1OKX.js}

@@ -1,12 +1,11 @@
 import { i as e } from "./chunk-CWhphoD1.js";
 import { Mi as t, Ms as n, Ps as r, V as i, Zr as a, bi as o, di as s, fi as c, ji as l, mt as u, ni as d, wi as f, yi as p } from "./dist-Da20xy8E.js";
-import { sliceUint16Shards as m, sliceUint8Shards as h } from "./utilities/tokens.js";
 //#region node_modules/@tensorflow/tfjs-data/dist/util/deep_map.js
-var g = /* @__PURE__ */ e(u());
-function ee(e, t) {
-	return _(e, t);
+var m = /* @__PURE__ */ e(u());
+function h(e, t) {
+	return g(e, t);
 }
-function _(e, t, n = /* @__PURE__ */ new Map(), r = /* @__PURE__ */ new Set()) {
+function g(e, t, n = /* @__PURE__ */ new Map(), r = /* @__PURE__ */ new Set()) {
 	if (e == null) return null;
 	if (typeof Blob == "function" && e instanceof Blob) return e.slice();
 	if (r.has(e)) throw Error("Circular references are not supported.");
@@ -14,34 +13,34 @@ function _(e, t, n = /* @__PURE__ */ new Map(), r = /* @__PURE__ */ new Set()) {
 	let i = t(e);
 	if (i.recurse && i.value !== null) throw Error("A deep map function may not return both a value and recurse=true.");
 	if (!i.recurse) return n.set(e, i.value), i.value;
-	if (x(e)) {
+	if (b(e)) {
 		let i = Array.isArray(e) ? [] : {};
 		r.add(e);
 		for (let a in e) {
 			let o = e[a];
-			i[a] = _(o, t, n, r);
+			i[a] = g(o, t, n, r);
 		}
 		return r.delete(e), e.__proto__ && (i.__proto__ = e.__proto__), i;
 	} else throw Error(`Can't recurse into non-iterable type: ${e}`);
 }
-function v(e, t = b) {
-	return y(e, t);
+function _(e, t = y) {
+	return v(e, t);
 }
-function y(e, t, n = /* @__PURE__ */ new Set()) {
+function v(e, t, n = /* @__PURE__ */ new Set()) {
 	let r = e[0];
 	if (n.has(r)) throw Error("Circular references are not supported.");
 	let i = t(e);
 	if (i.recurse && i.value !== null) throw Error("A deep zip function may not return both a value and recurse=true.");
 	if (!i.recurse) return i.value;
-	if (x(r)) {
+	if (b(r)) {
 		let i = Array.isArray(r) ? [] : {};
 		n.add(r);
-		for (let a in r) i[a] = y(e.map((e) => e[a]), t, n);
+		for (let a in r) i[a] = v(e.map((e) => e[a]), t, n);
 		return n.delete(r), i;
 	} else throw Error(`Can't recurse into non-iterable type: ${r}`);
 }
-function b(e) {
-	return e === null ? null : x(e[0]) ? {
+function y(e) {
+	return e === null ? null : b(e[0]) ? {
 		value: null,
 		recurse: !0
 	} : {
@@ -49,7 +48,7 @@ function b(e) {
 		recurse: !1
 	};
 }
-function x(e) {
+function b(e) {
 	let t = !1;
 	if (n().get("IS_BROWSER")) t = e instanceof TextDecoder;
 	else {
@@ -58,22 +57,22 @@ function x(e) {
 	}
 	return e != null && !ArrayBuffer.isView(e) && (Array.isArray(e) || typeof e == "object" && !(e instanceof f) && !(e instanceof Promise) && !t);
 }
-function S(e) {
-	return e == null || C(e) || Array.isArray(e) || typeof e == "object" && e instanceof f || l(e);
+function x(e) {
+	return e == null || ee(e) || Array.isArray(e) || typeof e == "object" && e instanceof f || l(e);
 }
-function C(e) {
+function ee(e) {
 	return e === null || typeof e != "object" && typeof e != "function";
 }
 //#endregion
 //#region node_modules/@tensorflow/tfjs-data/dist/util/deep_clone.js
-function w(e) {
-	return ee(e, T);
+function S(e) {
+	return h(e, C);
 }
-function T(e) {
+function C(e) {
 	return e instanceof f ? {
 		value: e.clone(),
 		recurse: !1
-	} : x(e) ? {
+	} : b(e) ? {
 		value: null,
 		recurse: !0
 	} : {
@@ -83,7 +82,7 @@ function T(e) {
 }
 //#endregion
 //#region node_modules/@tensorflow/tfjs-data/dist/util/ring_buffer.js
-var E = class {
+var w = class {
 	constructor(e) {
 		if (this.capacity = e, this.begin = 0, this.end = 0, e == null) throw RangeError("Can't create a ring buffer of unknown capacity.");
 		if (e < 1) throw RangeError("Can't create ring buffer of capacity < 1.");
@@ -138,7 +137,7 @@ var E = class {
 		let t = this.wrap(this.begin + e), n = this.get(t);
 		return this.set(t, this.pop()), n;
 	}
-}, D = class e extends E {
+}, T = class e extends w {
 	constructor() {
 		super(e.INITIAL_CAPACITY);
 	}
@@ -157,19 +156,19 @@ var E = class {
 		this.data = t, this.capacity = e, this.doubledCapacity = 2 * this.capacity, this.begin = 0, this.end = n;
 	}
 };
-D.INITIAL_CAPACITY = 32;
+T.INITIAL_CAPACITY = 32;
 //#endregion
 //#region node_modules/@tensorflow/tfjs-data/dist/iterators/lazy_iterator.js
-function O(e) {
-	return new M(e);
+function E(e) {
+	return new k(e);
 }
-function k(e) {
-	return new N(e);
+function D(e) {
+	return new A(e);
 }
-function A(e, t) {
-	return new W(e, t);
+function te(e, t) {
+	return new V(e, t);
 }
-var j = class {
+var O = class {
 	async toArray() {
 		let e = [], t = await this.next();
 		for (; !t.done;) e.push(t.value), t = await this.next();
@@ -189,22 +188,22 @@ var j = class {
 		for (; !t.done && n;) t = await this.next(), n = e(t.value);
 	}
 	handleErrors(e) {
-		return new B(this, e);
+		return new L(this, e);
 	}
 	filter(e) {
-		return new R(this, e);
+		return new F(this, e);
 	}
 	map(e) {
-		return new z(this, e);
+		return new I(this, e);
 	}
 	mapAsync(e) {
-		return new V(this, e);
+		return new R(this, e);
 	}
 	serialMapAsync(e) {
-		return new V(this, e).serial();
+		return new R(this, e).serial();
 	}
 	flatmap(e) {
-		return new U(this, e);
+		return new B(this, e);
 	}
 	async forEachAsync(e) {
 		return this.map(e).resolveFully();
@@ -213,30 +212,30 @@ var j = class {
 		return this.serialMapAsync(e).resolveWhile((e) => e === !0);
 	}
 	rowMajorBatch(e, t = !0) {
-		return new L(this, e, t);
+		return new P(this, e, t);
 	}
-	columnMajorBatch(e, t = !0, n = b) {
-		return this.rowMajorBatch(e, t).map((e) => v(e, n));
+	columnMajorBatch(e, t = !0, n = y) {
+		return this.rowMajorBatch(e, t).map((e) => _(e, n));
 	}
 	concatenate(e, t) {
-		return new W(O([this, e]), t);
+		return new V(E([this, e]), t);
 	}
 	take(e) {
-		return e < 0 || e == null ? this : new I(this, e);
+		return e < 0 || e == null ? this : new N(this, e);
 	}
 	skip(e) {
-		return e < 0 || e == null ? this : new F(this, e);
+		return e < 0 || e == null ? this : new M(this, e);
 	}
 	prefetch(e) {
-		return new K(this, e);
+		return new U(this, e);
 	}
 	shuffle(e, t) {
-		return new q(this, e, t);
+		return new W(this, e, t);
 	}
 	serial() {
-		return new P(this);
+		return new j(this);
 	}
-}, M = class extends j {
+}, k = class extends O {
 	constructor(e) {
 		super(), this.items = e, this.trav = 0;
 	}
@@ -250,11 +249,11 @@ var j = class {
 		};
 		let e = this.items[this.trav];
 		return this.trav++, {
-			value: w(e),
+			value: S(e),
 			done: !1
 		};
 	}
-}, N = class extends j {
+}, A = class extends O {
 	constructor(e) {
 		super(), this.nextFn = e;
 	}
@@ -268,7 +267,7 @@ var j = class {
 			throw e.message = `Error thrown while iterating through a dataset: ${e.message}`, e;
 		}
 	}
-}, P = class extends j {
+}, j = class extends O {
 	constructor(e) {
 		super(), this.upstream = e, this.lastRead = Promise.resolve({
 			value: null,
@@ -284,7 +283,7 @@ var j = class {
 	async serialNext() {
 		return this.upstream.next();
 	}
-}, F = class extends j {
+}, M = class extends O {
 	constructor(e, t) {
 		super(), this.upstream = e, this.maxCount = t, this.count = 0, this.lastRead = Promise.resolve({
 			value: null,
@@ -305,7 +304,7 @@ var j = class {
 		}
 		return this.upstream.next();
 	}
-}, I = class extends j {
+}, N = class extends O {
 	constructor(e, t) {
 		super(), this.upstream = e, this.maxCount = t, this.count = 0;
 	}
@@ -318,7 +317,7 @@ var j = class {
 			done: !0
 		} : this.upstream.next();
 	}
-}, L = class extends j {
+}, P = class extends O {
 	constructor(e, t, n = !0) {
 		super(), this.upstream = e, this.batchSize = t, this.enableSmallLastBatch = n, this.lastRead = Promise.resolve({
 			value: null,
@@ -349,7 +348,7 @@ var j = class {
 			done: !1
 		};
 	}
-}, R = class extends j {
+}, F = class extends O {
 	constructor(e, t) {
 		super(), this.upstream = e, this.predicate = t, this.lastRead = Promise.resolve({
 			value: null,
@@ -369,7 +368,7 @@ var j = class {
 			d(e.value);
 		}
 	}
-}, z = class extends j {
+}, I = class extends O {
 	constructor(e, t) {
 		super(), this.upstream = e, this.transform = t;
 	}
@@ -389,7 +388,7 @@ var j = class {
 			done: !1
 		};
 	}
-}, B = class extends j {
+}, L = class extends O {
 	constructor(e, t) {
 		super(), this.upstream = e, this.handler = t, this.count = 0, this.lastRead = Promise.resolve({
 			value: null,
@@ -412,7 +411,7 @@ var j = class {
 			};
 		}
 	}
-}, V = class extends j {
+}, R = class extends O {
 	constructor(e, t) {
 		super(), this.upstream = e, this.transform = t;
 	}
@@ -432,9 +431,9 @@ var j = class {
 			done: !1
 		};
 	}
-}, H = class extends j {
+}, z = class extends O {
 	constructor() {
-		super(), this.outputQueue = new D(), this.lastRead = Promise.resolve({
+		super(), this.outputQueue = new T(), this.lastRead = Promise.resolve({
 			value: null,
 			done: !1
 		});
@@ -452,7 +451,7 @@ var j = class {
 			done: !1
 		};
 	}
-}, U = class extends H {
+}, B = class extends z {
 	constructor(e, t) {
 		super(), this.upstream = e, this.transform = t;
 	}
@@ -467,7 +466,7 @@ var j = class {
 		for (let e of t) o(e, r) || e.dispose();
 		return !0;
 	}
-}, W = class extends j {
+}, V = class extends O {
 	constructor(e, t) {
 		super(), this.baseErrorHandler = t, this.lastRead = null, this.iterator = null, this.moreIterators = e;
 	}
@@ -489,13 +488,13 @@ var j = class {
 		let t = await this.iterator.next();
 		return t.done ? (this.iterator = null, this.readFromChain(e)) : t;
 	}
-}, G;
+}, H;
 (function(e) {
 	e[e.FAIL = 0] = "FAIL", e[e.SHORTEST = 1] = "SHORTEST", e[e.LONGEST = 2] = "LONGEST";
-})(G ||= {});
-var K = class extends j {
+})(H ||= {});
+var U = class extends O {
 	constructor(e, t) {
-		super(), this.upstream = e, this.bufferSize = t, this.buffer = new E(t);
+		super(), this.upstream = e, this.bufferSize = t, this.buffer = new w(t);
 	}
 	summary() {
 		return `${this.upstream.summary()} -> Prefetch`;
@@ -509,9 +508,9 @@ var K = class extends j {
 	next() {
 		return this.refill(), this.buffer.shift();
 	}
-}, q = class extends K {
+}, W = class extends U {
 	constructor(e, n, r) {
-		super(e, n), this.upstream = e, this.windowSize = n, this.upstreamExhausted = !1, this.random = g.alea(r || t().toString()), this.lastRead = Promise.resolve({
+		super(e, n), this.upstream = e, this.windowSize = n, this.upstreamExhausted = !1, this.random = m.alea(r || t().toString()), this.lastRead = Promise.resolve({
 			value: null,
 			done: !1
 		});
@@ -536,7 +535,7 @@ var K = class extends j {
 			done: !0
 		};
 	}
-}, J = class {
+}, G = class {
 	constructor() {
 		this.size = null;
 	}
@@ -545,54 +544,54 @@ var K = class extends j {
 		r(e > 0, () => `batchSize needs to be positive, but it is
       ${e}`);
 		let i;
-		return i = this.size === Infinity || this.size == null ? this.size : t ? Math.ceil(this.size / e) : Math.floor(this.size / e), Y(async () => (await n.iterator()).columnMajorBatch(e, t, X), i);
+		return i = this.size === Infinity || this.size == null ? this.size : t ? Math.ceil(this.size / e) : Math.floor(this.size / e), K(async () => (await n.iterator()).columnMajorBatch(e, t, q), i);
 	}
 	concatenate(e) {
 		let t = this, n;
-		return n = this.size === Infinity || e.size === Infinity ? Infinity : this.size != null && e.size != null ? this.size + e.size : null, Y(async () => (await t.iterator()).concatenate(await e.iterator()), n);
+		return n = this.size === Infinity || e.size === Infinity ? Infinity : this.size != null && e.size != null ? this.size + e.size : null, K(async () => (await t.iterator()).concatenate(await e.iterator()), n);
 	}
 	filter(e) {
 		let t = this, n;
-		return n = this.size === Infinity ? Infinity : null, Y(async () => (await t.iterator()).filter((t) => s(() => e(t))), n);
+		return n = this.size === Infinity ? Infinity : null, K(async () => (await t.iterator()).filter((t) => s(() => e(t))), n);
 	}
 	async forEachAsync(e) {
 		return (await this.iterator()).forEachAsync(e);
 	}
 	map(e) {
 		let t = this;
-		return Y(async () => (await t.iterator()).map((t) => s(() => e(t))), this.size);
+		return K(async () => (await t.iterator()).map((t) => s(() => e(t))), this.size);
 	}
 	mapAsync(e) {
 		let t = this;
-		return Y(async () => (await t.iterator()).mapAsync(e), this.size);
+		return K(async () => (await t.iterator()).mapAsync(e), this.size);
 	}
 	prefetch(e) {
 		if (e == null) throw RangeError("`Dataset.prefetch()` requires bufferSize to be specified.");
 		let t = this;
-		return Y(async () => (await t.iterator()).prefetch(e), this.size);
+		return K(async () => (await t.iterator()).prefetch(e), this.size);
 	}
 	repeat(e) {
 		let t = this, n;
-		return n = this.size != null && e > 0 ? this.size * e : e === 0 ? 0 : this.size != null && (e === void 0 || e < 0) ? Infinity : null, Y(async () => A(k(async () => ({
+		return n = this.size != null && e > 0 ? this.size * e : e === 0 ? 0 : this.size != null && (e === void 0 || e < 0) ? Infinity : null, K(async () => te(D(async () => ({
 			value: await t.iterator(),
 			done: !1
 		})).take(e)), n);
 	}
 	skip(e) {
 		let t = this, n;
-		return n = this.size != null && e >= 0 && this.size >= e ? this.size - e : this.size != null && (this.size < e || e === void 0 || e < 0) ? 0 : null, Y(async () => (await t.iterator()).skip(e), n);
+		return n = this.size != null && e >= 0 && this.size >= e ? this.size - e : this.size != null && (this.size < e || e === void 0 || e < 0) ? 0 : null, K(async () => (await t.iterator()).skip(e), n);
 	}
 	shuffle(e, n, r = !0) {
 		if (e == null || e < 0) throw this.size == null ? RangeError("`Dataset.shuffle()` requires bufferSize to be specified.") : RangeError(`\`Dataset.shuffle()\` requires bufferSize to be specified.  If your data fits in main memory (for regular JS objects), and/or GPU memory (for \`tf.Tensor\`s), consider setting bufferSize to the dataset size (${this.size} elements)`);
-		let i = this, a = g.alea(n || t().toString());
-		return Y(async () => {
+		let i = this, a = m.alea(n || t().toString());
+		return K(async () => {
 			let t = a.int32();
 			return r && (t += a.int32()), (await i.iterator()).shuffle(e, t.toString());
 		}, this.size);
 	}
 	take(e) {
 		let t = this, n;
-		return n = this.size != null && this.size > e ? e : this.size != null && this.size <= e ? this.size : null, Y(async () => (await t.iterator()).take(e), n);
+		return n = this.size != null && this.size > e ? e : this.size != null && this.size <= e ? this.size : null, K(async () => (await t.iterator()).take(e), n);
 	}
 	async toArray() {
 		if (this.size === Infinity) throw Error("Can not convert infinite data stream to array.");
@@ -603,9 +602,9 @@ var K = class extends j {
 		return (await this.iterator()).toArrayForTest();
 	}
 };
-J.MAX_BUFFER_SIZE = 1e4;
-function Y(e, t = null) {
-	return new class extends J {
+G.MAX_BUFFER_SIZE = 1e4;
+function K(e, t = null) {
+	return new class extends G {
 		constructor() {
 			super(...arguments), this.size = t;
 		}
@@ -614,84 +613,111 @@ function Y(e, t = null) {
 		}
 	}();
 }
-function X(e) {
+function q(e) {
 	if (e === null) return null;
 	let t = e[0];
-	return S(t) ? {
-		value: Z(e),
+	return x(t) ? {
+		value: J(e),
 		recurse: !1
 	} : {
 		value: null,
 		recurse: !0
 	};
 }
-function Z(e) {
+function J(e) {
 	if (e.length === 0) throw Error("Can't make a batch of zero elements.");
 	return e[0] instanceof f ? i(e) : c(e);
 }
 //#endregion
 //#region node_modules/@tensorflow/tfjs-data/dist/readers.js
-function Q(e) {
-	return Y(async () => {
+function Y(e) {
+	return K(async () => {
 		let t = await e();
-		return k(() => t.next());
+		return D(() => t.next());
 	});
 }
 //#endregion
 //#region lib/training/DatasetBuilder.ts
-function te(e, t) {
+function X(e, t) {
 	let n = e.map((e) => t.encodeConversation(e)).flat();
 	return new Uint16Array(n);
 }
-function ne(e, t) {
+function Z(e, t) {
 	let n = e.map((e) => t.encodeConversation(e, !1, !0)), r = n.map((e) => e.tokens).flat(), i = n.map((e) => e.mask).flat();
 	return {
 		tokens: new Uint16Array(r),
 		mask: new Uint8Array(i.map((e) => +!!e))
 	};
 }
-function $(e) {
+function Q(e) {
 	for (let t = e.length - 1; t > 0; t--) {
 		let n = Math.floor(Math.random() * (t + 1));
 		[e[t], e[n]] = [e[n], e[t]];
 	}
 	return e;
 }
-var re = class {
+async function $(e, t, n) {
+	e.step += 1;
+	let r = e.shardIndex === e.shuffledShards.length - 1 ? e.lastShardIndexes : e.shuffledIndexes;
+	if (e.step >= r.length) {
+		e.step = 0, e.shardIndex += 1, e.shardIndex >= e.shuffledShards.length && (e.shardIndex = 0, n || (Q(e.shuffledShards), Q(e.shuffledIndexes), Q(e.lastShardIndexes))), e.nextShard ? (e.currentShard = e.nextShard, e.nextShard = null) : e.currentShard = await t.getShard(e.shuffledShards[e.shardIndex]);
+		let r = (e.shardIndex + 1) % e.shuffledShards.length;
+		if (t.getShard(e.shuffledShards[r]).then((t) => {
+			e.nextShard = t;
+		}), t.hasMask()) {
+			e.nextMask ? (e.currentMask = e.nextMask, e.nextMask = null) : e.currentMask = await t.getMask(e.shuffledShards[e.shardIndex]) ?? null;
+			let n = (e.shardIndex + 1) % e.shuffledShards.length;
+			t.getMask(e.shuffledShards[n]).then((t) => {
+				e.nextMask = t ?? null;
+			});
+		}
+	}
+}
+var ne = class {
 	tokenizer;
 	blockSize;
 	constructor(e, t = 128) {
 		this.tokenizer = e, this.blockSize = t;
 	}
-	async createTextDataset(e, t = 32, n, r, i = 65535) {
-		let a = e.reduce((e, t) => e + t.length, 0);
+	async createTextDataset(e, t) {
+		let { batchSize: n = 32, noShuffle: r = !1, ignoreIndex: i = 65535 } = t || {}, a = e.getTokenCount();
 		if (a < this.blockSize + 1) throw Error(`Not enough tokens (${a}) for block size ${this.blockSize}`);
-		let o = Math.ceil(a / this.blockSize), c = {
-			shuffledIndexes: new Uint32Array(o),
+		let o = Math.ceil(e.shardSize / this.blockSize), c = {
+			shuffledShards: new Uint16Array(e.getShardCount()),
+			shuffledIndexes: new Uint16Array(o),
+			lastShardIndexes: new Uint16Array(Math.ceil(e.getShardLength(e.getShardCount() - 1) / this.blockSize)),
+			currentMask: null,
+			nextMask: null,
+			currentShard: null,
+			nextShard: null,
+			shardIndex: 0,
 			step: 0
 		};
-		if (n) c.shuffledIndexes = n;
-		else {
-			c.shuffledIndexes = new Uint32Array(o);
-			for (let e = 0; e < o; e++) c.shuffledIndexes[e] = e;
-		}
-		return {
-			dataset: Q(function* () {
+		for (let e = 0; e < c.shuffledShards.length; e++) c.shuffledShards[e] = e;
+		for (let e = 0; e < c.shuffledIndexes.length; e++) c.shuffledIndexes[e] = e;
+		for (let e = 0; e < c.lastShardIndexes.length; e++) c.lastShardIndexes[e] = e;
+		return t?.shuffleFirst && (Q(c.shuffledShards), Q(c.shuffledIndexes), Q(c.lastShardIndexes)), c.currentShard = await e.getShard(c.shuffledShards[c.shardIndex]), c.shardIndex + 1 < c.shuffledShards.length && e.getShard(c.shuffledShards[c.shardIndex + 1]).then((e) => {
+			c.nextShard = e;
+		}), e.hasMask() && (c.currentMask = await e.getMask(c.shuffledShards[c.shardIndex]) ?? null, c.shardIndex + 1 < c.shuffledShards.length && e.getMask(c.shuffledShards[c.shardIndex + 1]).then((e) => {
+			c.nextMask = e ?? null;
+		})), {
+			dataset: Y(async function* () {
 				for (;;) {
-					let t = c.shuffledIndexes[c.step++] * this.blockSize;
-					if (c.step >= c.shuffledIndexes.length && (c.step = 0, $(c.shuffledIndexes)), t + this.blockSize + 1 > a) continue;
-					let n = new Int32Array(m(e, t, t + this.blockSize)), o = new Int32Array(m(e, t + 1, t + this.blockSize + 1));
-					if (r) {
-						let e = 0, n = h(r, t + 1, t + this.blockSize + 1);
-						for (let t = 0; t < o.length; t++) n[t] === 0 && (o[t] = i, e++);
-						if (e === o.length) continue;
+					let t = (c.shardIndex === c.shuffledShards.length - 1 ? c.lastShardIndexes : c.shuffledIndexes)[c.step] * this.blockSize, n = c.currentShard, a = c.currentMask, o = $(c, e, r);
+					if (!n) break;
+					t + this.blockSize + 1 > n.length && (t = n.length - this.blockSize - 1);
+					let s = new Int32Array(n.slice(t, t + this.blockSize)), l = new Int32Array(n.slice(t + 1, t + this.blockSize + 1));
+					if (a) {
+						let e = 0, n = a.slice(t + 1, t + this.blockSize + 1);
+						for (let t = 0; t < l.length; t++) n[t] === 0 && (l[t] = i, e++);
+						if (e === l.length) continue;
 					}
 					yield {
-						xs: n,
-						ys: o
-					};
+						xs: s,
+						ys: l
+					}, await o;
 				}
-			}.bind(this)).batch(t).map((e) => {
+			}.bind(this)).batch(n).map((e) => {
 				let t = e;
 				return s(() => ({
 					xs: t.xs.cast("int32"),
@@ -703,4 +729,4 @@ var re = class {
 	}
 };
 //#endregion
-export { $ as i, te as n, ne as r, re as t };
+export { Q as a, $ as i, X as n, Z as r, ne as t };

package/dist/Generator.js

@@ -1,2 +1,2 @@
-import { c as e, s as t } from "./main-Bgc7_9kb.js";
+import { a as e, i as t } from "./main-Dz72vadm.js";
 export { t as default, e as isConversation };

package/dist/TeachableLLM.d.ts

@@ -7,7 +7,6 @@ import { default as Trainer, TrainingType } from './Trainer';
 import { ConversationStream } from './main';
 import { default as MemoryProfiler } from './utilities/profile';
 import { default as Model, ModelForwardAttributes } from './models/model';
-import { Task } from './training/tasks/Task';
 import { TrainingLogEntry, TrainingOptions } from './training/types';
 import { ModelMode, TransformersMetadata } from './loader/types';
 type TeachableLLMStatus = 'warmup' | 'awaitingTokens' | 'ready' | 'training' | 'loading' | 'busy' | 'error';
@@ -51,7 +50,7 @@ export default class TeachableLLM {
     set enableProfiler(value: boolean);
     getNumParams(): number;
     trainer(trainingType?: TrainingType, options?: TrainingOptions): Trainer;
-    train(text: Task[], options?: TrainingOptions, trainingType?: TrainingType): Promise<void>;
+    train(text: ConversationStream[], options?: TrainingOptions, trainingType?: TrainingType): Promise<void>;
     trainTokeniser(text: ConversationStream[]): Promise<number>;
     generator(): IGenerator;
     generateText(prompt: Conversation[], options?: IGenerateOptions): Promise<Conversation[]>;

package/dist/TeachableLLM.js

@@ -1,2 +1,2 @@
-import { i as e } from "./main-Bgc7_9kb.js";
+import { r as e } from "./main-Dz72vadm.js";
 export { e as default };