@genai-fi/nanogpt 0.20.7 → 0.22.0

package/dist/{BaseTokeniser-DSg9zcYq.js → BaseTokeniser-C9TSv4th.js}

@@ -1,4 +1,4 @@
-import { a as e, t } from "./chunk-BPntVaq0.js";
+import { i as e, t } from "./chunk-CWhphoD1.js";
 var n = (/* @__PURE__ */ e((/* @__PURE__ */ t(((e, t) => {
 	var n = Object.prototype.hasOwnProperty, r = "~";
 	function i() {}

package/dist/{DatasetBuilder-DgURD85T.js → DatasetBuilder-Ctb425Id.js}

@@ -1,12 +1,12 @@
-import { a as e } from "./chunk-BPntVaq0.js";
-import { Mi as t, Ms as n, Ps as r, V as i, Zr as a, bi as o, di as s, fi as c, ji as l, mt as ee, ni as u, wi as d, yi as f } from "./dist-BewPQWjc.js";
-import { n as p } from "./dist-DXwIvKxl.js";
+import { i as e } from "./chunk-CWhphoD1.js";
+import { Mi as t, Ms as n, Ps as r, V as i, Zr as a, bi as o, di as s, fi as c, ji as l, mt as u, ni as d, wi as f, yi as p } from "./dist-Da20xy8E.js";
+import { sliceUint16Shards as m, sliceUint8Shards as h } from "./utilities/tokens.js";
 //#region node_modules/@tensorflow/tfjs-data/dist/util/deep_map.js
-var m = /* @__PURE__ */ e(ee());
-function h(e, t) {
-	return g(e, t);
+var g = /* @__PURE__ */ e(u());
+function ee(e, t) {
+	return _(e, t);
 }
-function g(e, t, n = /* @__PURE__ */ new Map(), r = /* @__PURE__ */ new Set()) {
+function _(e, t, n = /* @__PURE__ */ new Map(), r = /* @__PURE__ */ new Set()) {
 	if (e == null) return null;
 	if (typeof Blob == "function" && e instanceof Blob) return e.slice();
 	if (r.has(e)) throw Error("Circular references are not supported.");
@@ -14,34 +14,34 @@ function g(e, t, n = /* @__PURE__ */ new Map(), r = /* @__PURE__ */ new Set()) {
 	let i = t(e);
 	if (i.recurse && i.value !== null) throw Error("A deep map function may not return both a value and recurse=true.");
 	if (!i.recurse) return n.set(e, i.value), i.value;
-	if (b(e)) {
+	if (x(e)) {
 		let i = Array.isArray(e) ? [] : {};
 		r.add(e);
 		for (let a in e) {
 			let o = e[a];
-			i[a] = g(o, t, n, r);
+			i[a] = _(o, t, n, r);
 		}
 		return r.delete(e), e.__proto__ && (i.__proto__ = e.__proto__), i;
 	} else throw Error(`Can't recurse into non-iterable type: ${e}`);
 }
-function _(e, t = y) {
-	return v(e, t);
+function v(e, t = b) {
+	return y(e, t);
 }
-function v(e, t, n = /* @__PURE__ */ new Set()) {
+function y(e, t, n = /* @__PURE__ */ new Set()) {
 	let r = e[0];
 	if (n.has(r)) throw Error("Circular references are not supported.");
 	let i = t(e);
 	if (i.recurse && i.value !== null) throw Error("A deep zip function may not return both a value and recurse=true.");
 	if (!i.recurse) return i.value;
-	if (b(r)) {
+	if (x(r)) {
 		let i = Array.isArray(r) ? [] : {};
 		n.add(r);
-		for (let a in r) i[a] = v(e.map((e) => e[a]), t, n);
+		for (let a in r) i[a] = y(e.map((e) => e[a]), t, n);
 		return n.delete(r), i;
 	} else throw Error(`Can't recurse into non-iterable type: ${r}`);
 }
-function y(e) {
-	return e === null ? null : b(e[0]) ? {
+function b(e) {
+	return e === null ? null : x(e[0]) ? {
 		value: null,
 		recurse: !0
 	} : {
@@ -49,31 +49,31 @@ function y(e) {
 		recurse: !1
 	};
 }
-function b(e) {
+function x(e) {
 	let t = !1;
 	if (n().get("IS_BROWSER")) t = e instanceof TextDecoder;
 	else {
 		let { StringDecoder: n } = a();
 		t = e instanceof n;
 	}
-	return e != null && !ArrayBuffer.isView(e) && (Array.isArray(e) || typeof e == "object" && !(e instanceof d) && !(e instanceof Promise) && !t);
-}
-function x(e) {
-	return e == null || S(e) || Array.isArray(e) || typeof e == "object" && e instanceof d || l(e);
+	return e != null && !ArrayBuffer.isView(e) && (Array.isArray(e) || typeof e == "object" && !(e instanceof f) && !(e instanceof Promise) && !t);
 }
 function S(e) {
+	return e == null || C(e) || Array.isArray(e) || typeof e == "object" && e instanceof f || l(e);
+}
+function C(e) {
 	return e === null || typeof e != "object" && typeof e != "function";
 }
 //#endregion
 //#region node_modules/@tensorflow/tfjs-data/dist/util/deep_clone.js
-function C(e) {
-	return h(e, w);
-}
 function w(e) {
-	return e instanceof d ? {
+	return ee(e, T);
+}
+function T(e) {
+	return e instanceof f ? {
 		value: e.clone(),
 		recurse: !1
-	} : b(e) ? {
+	} : x(e) ? {
 		value: null,
 		recurse: !0
 	} : {
@@ -83,7 +83,7 @@ function w(e) {
 }
 //#endregion
 //#region node_modules/@tensorflow/tfjs-data/dist/util/ring_buffer.js
-var T = class {
+var E = class {
 	constructor(e) {
 		if (this.capacity = e, this.begin = 0, this.end = 0, e == null) throw RangeError("Can't create a ring buffer of unknown capacity.");
 		if (e < 1) throw RangeError("Can't create ring buffer of capacity < 1.");
@@ -138,7 +138,7 @@ var T = class {
 		let t = this.wrap(this.begin + e), n = this.get(t);
 		return this.set(t, this.pop()), n;
 	}
-}, E = class e extends T {
+}, D = class e extends E {
 	constructor() {
 		super(e.INITIAL_CAPACITY);
 	}
@@ -157,19 +157,19 @@ var T = class {
 		this.data = t, this.capacity = e, this.doubledCapacity = 2 * this.capacity, this.begin = 0, this.end = n;
 	}
 };
-E.INITIAL_CAPACITY = 32;
+D.INITIAL_CAPACITY = 32;
 //#endregion
 //#region node_modules/@tensorflow/tfjs-data/dist/iterators/lazy_iterator.js
-function D(e) {
-	return new j(e);
-}
 function O(e) {
 	return new M(e);
 }
-function k(e, t) {
-	return new U(e, t);
+function k(e) {
+	return new N(e);
+}
+function A(e, t) {
+	return new W(e, t);
 }
-var A = class {
+var j = class {
 	async toArray() {
 		let e = [], t = await this.next();
 		for (; !t.done;) e.push(t.value), t = await this.next();
@@ -189,22 +189,22 @@ var A = class {
 		for (; !t.done && n;) t = await this.next(), n = e(t.value);
 	}
 	handleErrors(e) {
-		return new z(this, e);
+		return new B(this, e);
 	}
 	filter(e) {
-		return new L(this, e);
+		return new R(this, e);
 	}
 	map(e) {
-		return new R(this, e);
+		return new z(this, e);
 	}
 	mapAsync(e) {
-		return new B(this, e);
+		return new V(this, e);
 	}
 	serialMapAsync(e) {
-		return new B(this, e).serial();
+		return new V(this, e).serial();
 	}
 	flatmap(e) {
-		return new H(this, e);
+		return new U(this, e);
 	}
 	async forEachAsync(e) {
 		return this.map(e).resolveFully();
@@ -213,30 +213,30 @@ var A = class {
 		return this.serialMapAsync(e).resolveWhile((e) => e === !0);
 	}
 	rowMajorBatch(e, t = !0) {
-		return new I(this, e, t);
+		return new L(this, e, t);
 	}
-	columnMajorBatch(e, t = !0, n = y) {
-		return this.rowMajorBatch(e, t).map((e) => _(e, n));
+	columnMajorBatch(e, t = !0, n = b) {
+		return this.rowMajorBatch(e, t).map((e) => v(e, n));
 	}
 	concatenate(e, t) {
-		return new U(D([this, e]), t);
+		return new W(O([this, e]), t);
 	}
 	take(e) {
-		return e < 0 || e == null ? this : new F(this, e);
+		return e < 0 || e == null ? this : new I(this, e);
 	}
 	skip(e) {
-		return e < 0 || e == null ? this : new P(this, e);
+		return e < 0 || e == null ? this : new F(this, e);
 	}
 	prefetch(e) {
-		return new G(this, e);
+		return new K(this, e);
 	}
 	shuffle(e, t) {
-		return new K(this, e, t);
+		return new q(this, e, t);
 	}
 	serial() {
-		return new N(this);
+		return new P(this);
 	}
-}, j = class extends A {
+}, M = class extends j {
 	constructor(e) {
 		super(), this.items = e, this.trav = 0;
 	}
@@ -250,11 +250,11 @@ var A = class {
 		};
 		let e = this.items[this.trav];
 		return this.trav++, {
-			value: C(e),
+			value: w(e),
 			done: !1
 		};
 	}
-}, M = class extends A {
+}, N = class extends j {
 	constructor(e) {
 		super(), this.nextFn = e;
 	}
@@ -268,7 +268,7 @@ var A = class {
 			throw e.message = `Error thrown while iterating through a dataset: ${e.message}`, e;
 		}
 	}
-}, N = class extends A {
+}, P = class extends j {
 	constructor(e) {
 		super(), this.upstream = e, this.lastRead = Promise.resolve({
 			value: null,
@@ -284,7 +284,7 @@ var A = class {
 	async serialNext() {
 		return this.upstream.next();
 	}
-}, P = class extends A {
+}, F = class extends j {
 	constructor(e, t) {
 		super(), this.upstream = e, this.maxCount = t, this.count = 0, this.lastRead = Promise.resolve({
 			value: null,
@@ -301,11 +301,11 @@ var A = class {
 		for (; this.count++ < this.maxCount;) {
 			let e = await this.upstream.next();
 			if (e.done) return e;
-			u(e.value);
+			d(e.value);
 		}
 		return this.upstream.next();
 	}
-}, F = class extends A {
+}, I = class extends j {
 	constructor(e, t) {
 		super(), this.upstream = e, this.maxCount = t, this.count = 0;
 	}
@@ -318,7 +318,7 @@ var A = class {
 			done: !0
 		} : this.upstream.next();
 	}
-}, I = class extends A {
+}, L = class extends j {
 	constructor(e, t, n = !0) {
 		super(), this.upstream = e, this.batchSize = t, this.enableSmallLastBatch = n, this.lastRead = Promise.resolve({
 			value: null,
@@ -349,7 +349,7 @@ var A = class {
 			done: !1
 		};
 	}
-}, L = class extends A {
+}, R = class extends j {
 	constructor(e, t) {
 		super(), this.upstream = e, this.predicate = t, this.lastRead = Promise.resolve({
 			value: null,
@@ -366,10 +366,10 @@ var A = class {
 		for (;;) {
 			let e = await this.upstream.next();
 			if (e.done || this.predicate(e.value)) return e;
-			u(e.value);
+			d(e.value);
 		}
 	}
-}, R = class extends A {
+}, z = class extends j {
 	constructor(e, t) {
 		super(), this.upstream = e, this.transform = t;
 	}
@@ -382,14 +382,14 @@ var A = class {
 			value: null,
 			done: !0
 		};
-		let t = f(e.value), n = this.transform(e.value), r = f(n);
+		let t = p(e.value), n = this.transform(e.value), r = p(n);
 		for (let e of t) o(e, r) || e.dispose();
 		return {
 			value: n,
 			done: !1
 		};
 	}
-}, z = class extends A {
+}, B = class extends j {
 	constructor(e, t) {
 		super(), this.upstream = e, this.handler = t, this.count = 0, this.lastRead = Promise.resolve({
 			value: null,
@@ -412,7 +412,7 @@ var A = class {
 			};
 		}
 	}
-}, B = class extends A {
+}, V = class extends j {
 	constructor(e, t) {
 		super(), this.upstream = e, this.transform = t;
 	}
@@ -425,16 +425,16 @@ var A = class {
 			value: null,
 			done: !0
 		};
-		let t = f(e.value), n = await this.transform(e.value), r = f(n);
+		let t = p(e.value), n = await this.transform(e.value), r = p(n);
 		for (let e of t) o(e, r) || e.dispose();
 		return {
 			value: n,
 			done: !1
 		};
 	}
-}, V = class extends A {
+}, H = class extends j {
 	constructor() {
-		super(), this.outputQueue = new E(), this.lastRead = Promise.resolve({
+		super(), this.outputQueue = new D(), this.lastRead = Promise.resolve({
 			value: null,
 			done: !1
 		});
@@ -452,7 +452,7 @@ var A = class {
 			done: !1
 		};
 	}
-}, H = class extends V {
+}, U = class extends H {
 	constructor(e, t) {
 		super(), this.upstream = e, this.transform = t;
 	}
@@ -462,12 +462,12 @@ var A = class {
 	async pump() {
 		let e = await this.upstream.next();
 		if (e.done) return !1;
-		let t = f(e.value), n = this.transform(e.value), r = f(n);
+		let t = p(e.value), n = this.transform(e.value), r = p(n);
 		this.outputQueue.pushAll(n);
 		for (let e of t) o(e, r) || e.dispose();
 		return !0;
 	}
-}, U = class extends A {
+}, W = class extends j {
 	constructor(e, t) {
 		super(), this.baseErrorHandler = t, this.lastRead = null, this.iterator = null, this.moreIterators = e;
 	}
@@ -489,13 +489,13 @@ var A = class {
 		let t = await this.iterator.next();
 		return t.done ? (this.iterator = null, this.readFromChain(e)) : t;
 	}
-}, W;
+}, G;
 (function(e) {
 	e[e.FAIL = 0] = "FAIL", e[e.SHORTEST = 1] = "SHORTEST", e[e.LONGEST = 2] = "LONGEST";
-})(W ||= {});
-var G = class extends A {
+})(G ||= {});
+var K = class extends j {
 	constructor(e, t) {
-		super(), this.upstream = e, this.bufferSize = t, this.buffer = new T(t);
+		super(), this.upstream = e, this.bufferSize = t, this.buffer = new E(t);
 	}
 	summary() {
 		return `${this.upstream.summary()} -> Prefetch`;
@@ -509,9 +509,9 @@ var G = class extends A {
 	next() {
 		return this.refill(), this.buffer.shift();
 	}
-}, K = class extends G {
+}, q = class extends K {
 	constructor(e, n, r) {
-		super(e, n), this.upstream = e, this.windowSize = n, this.upstreamExhausted = !1, this.random = m.alea(r || t().toString()), this.lastRead = Promise.resolve({
+		super(e, n), this.upstream = e, this.windowSize = n, this.upstreamExhausted = !1, this.random = g.alea(r || t().toString()), this.lastRead = Promise.resolve({
 			value: null,
 			done: !1
 		});
@@ -536,7 +536,7 @@ var G = class extends A {
 			done: !0
 		};
 	}
-}, q = class {
+}, J = class {
 	constructor() {
 		this.size = null;
 	}
@@ -545,54 +545,54 @@ var G = class extends A {
 		r(e > 0, () => `batchSize needs to be positive, but it is
       ${e}`);
 		let i;
-		return i = this.size === Infinity || this.size == null ? this.size : t ? Math.ceil(this.size / e) : Math.floor(this.size / e), J(async () => (await n.iterator()).columnMajorBatch(e, t, Y), i);
+		return i = this.size === Infinity || this.size == null ? this.size : t ? Math.ceil(this.size / e) : Math.floor(this.size / e), Y(async () => (await n.iterator()).columnMajorBatch(e, t, X), i);
 	}
 	concatenate(e) {
 		let t = this, n;
-		return n = this.size === Infinity || e.size === Infinity ? Infinity : this.size != null && e.size != null ? this.size + e.size : null, J(async () => (await t.iterator()).concatenate(await e.iterator()), n);
+		return n = this.size === Infinity || e.size === Infinity ? Infinity : this.size != null && e.size != null ? this.size + e.size : null, Y(async () => (await t.iterator()).concatenate(await e.iterator()), n);
 	}
 	filter(e) {
 		let t = this, n;
-		return n = this.size === Infinity ? Infinity : null, J(async () => (await t.iterator()).filter((t) => s(() => e(t))), n);
+		return n = this.size === Infinity ? Infinity : null, Y(async () => (await t.iterator()).filter((t) => s(() => e(t))), n);
 	}
 	async forEachAsync(e) {
 		return (await this.iterator()).forEachAsync(e);
 	}
 	map(e) {
 		let t = this;
-		return J(async () => (await t.iterator()).map((t) => s(() => e(t))), this.size);
+		return Y(async () => (await t.iterator()).map((t) => s(() => e(t))), this.size);
 	}
 	mapAsync(e) {
 		let t = this;
-		return J(async () => (await t.iterator()).mapAsync(e), this.size);
+		return Y(async () => (await t.iterator()).mapAsync(e), this.size);
 	}
 	prefetch(e) {
 		if (e == null) throw RangeError("`Dataset.prefetch()` requires bufferSize to be specified.");
 		let t = this;
-		return J(async () => (await t.iterator()).prefetch(e), this.size);
+		return Y(async () => (await t.iterator()).prefetch(e), this.size);
 	}
 	repeat(e) {
 		let t = this, n;
-		return n = this.size != null && e > 0 ? this.size * e : e === 0 ? 0 : this.size != null && (e === void 0 || e < 0) ? Infinity : null, J(async () => k(O(async () => ({
+		return n = this.size != null && e > 0 ? this.size * e : e === 0 ? 0 : this.size != null && (e === void 0 || e < 0) ? Infinity : null, Y(async () => A(k(async () => ({
 			value: await t.iterator(),
 			done: !1
 		})).take(e)), n);
 	}
 	skip(e) {
 		let t = this, n;
-		return n = this.size != null && e >= 0 && this.size >= e ? this.size - e : this.size != null && (this.size < e || e === void 0 || e < 0) ? 0 : null, J(async () => (await t.iterator()).skip(e), n);
+		return n = this.size != null && e >= 0 && this.size >= e ? this.size - e : this.size != null && (this.size < e || e === void 0 || e < 0) ? 0 : null, Y(async () => (await t.iterator()).skip(e), n);
 	}
 	shuffle(e, n, r = !0) {
 		if (e == null || e < 0) throw this.size == null ? RangeError("`Dataset.shuffle()` requires bufferSize to be specified.") : RangeError(`\`Dataset.shuffle()\` requires bufferSize to be specified.  If your data fits in main memory (for regular JS objects), and/or GPU memory (for \`tf.Tensor\`s), consider setting bufferSize to the dataset size (${this.size} elements)`);
-		let i = this, a = m.alea(n || t().toString());
-		return J(async () => {
+		let i = this, a = g.alea(n || t().toString());
+		return Y(async () => {
 			let t = a.int32();
 			return r && (t += a.int32()), (await i.iterator()).shuffle(e, t.toString());
 		}, this.size);
 	}
 	take(e) {
 		let t = this, n;
-		return n = this.size != null && this.size > e ? e : this.size != null && this.size <= e ? this.size : null, J(async () => (await t.iterator()).take(e), n);
+		return n = this.size != null && this.size > e ? e : this.size != null && this.size <= e ? this.size : null, Y(async () => (await t.iterator()).take(e), n);
 	}
 	async toArray() {
 		if (this.size === Infinity) throw Error("Can not convert infinite data stream to array.");
@@ -603,9 +603,9 @@ var G = class extends A {
 		return (await this.iterator()).toArrayForTest();
 	}
 };
-q.MAX_BUFFER_SIZE = 1e4;
-function J(e, t = null) {
-	return new class extends q {
+J.MAX_BUFFER_SIZE = 1e4;
+function Y(e, t = null) {
+	return new class extends J {
 		constructor() {
 			super(...arguments), this.size = t;
 		}
@@ -614,43 +614,37 @@ function J(e, t = null) {
 		}
 	}();
 }
-function Y(e) {
+function X(e) {
 	if (e === null) return null;
 	let t = e[0];
-	return x(t) ? {
-		value: X(e),
+	return S(t) ? {
+		value: Z(e),
 		recurse: !1
 	} : {
 		value: null,
 		recurse: !0
 	};
 }
-function X(e) {
+function Z(e) {
 	if (e.length === 0) throw Error("Can't make a batch of zero elements.");
-	return e[0] instanceof d ? i(e) : c(e);
+	return e[0] instanceof f ? i(e) : c(e);
 }
 //#endregion
-//#region node_modules/@tensorflow/tfjs-data/dist/iterators/string_iterator.js
-p();
-//#endregion
 //#region node_modules/@tensorflow/tfjs-data/dist/readers.js
-function Z(e) {
-	return J(async () => {
+function Q(e) {
+	return Y(async () => {
 		let t = await e();
-		return O(() => t.next());
+		return k(() => t.next());
 	});
 }
 //#endregion
 //#region lib/training/DatasetBuilder.ts
-var Q = 8;
 function te(e, t) {
 	let n = e.map((e) => t.encodeConversation(e)).flat();
 	return new Uint16Array(n);
 }
 function ne(e, t) {
-	let n = e.map((e) => t.encodeConversation(e, !1, !0));
-	console.log("Tokenised Texts with Mask:", n);
-	let r = n.map((e) => e.tokens).flat(), i = n.map((e) => e.mask).flat();
+	let n = e.map((e) => t.encodeConversation(e, !1, !0)), r = n.map((e) => e.tokens).flat(), i = n.map((e) => e.mask).flat();
 	return {
 		tokens: new Uint16Array(r),
 		mask: new Uint8Array(i.map((e) => +!!e))
@@ -670,31 +664,31 @@ var re = class {
 		this.tokenizer = e, this.blockSize = t;
 	}
 	async createTextDataset(e, t = 32, n, r, i = 65535) {
-		if (e.length < this.blockSize + 1) throw Error(`Not enough tokens (${e.length}) for block size ${this.blockSize}`);
-		let a = {
-			shuffledIndexes: new Uint32Array(e.length),
+		let a = e.reduce((e, t) => e + t.length, 0);
+		if (a < this.blockSize + 1) throw Error(`Not enough tokens (${a}) for block size ${this.blockSize}`);
+		let o = Math.ceil(a / this.blockSize), c = {
+			shuffledIndexes: new Uint32Array(o),
 			step: 0
 		};
-		if (n) a.shuffledIndexes = n;
+		if (n) c.shuffledIndexes = n;
 		else {
-			a.shuffledIndexes = new Uint32Array(e.length);
-			for (let t = 0; t < e.length; t++) a.shuffledIndexes[t] = t;
-			$(a.shuffledIndexes);
+			c.shuffledIndexes = new Uint32Array(o);
+			for (let e = 0; e < o; e++) c.shuffledIndexes[e] = e;
 		}
 		return {
-			dataset: Z(function* () {
+			dataset: Q(function* () {
 				for (;;) {
-					let t = a.shuffledIndexes[a.step++];
-					if (a.step >= a.shuffledIndexes.length && (a.step = 0, $(a.shuffledIndexes)), t + this.blockSize + 1 > e.length) continue;
-					let n = new Int32Array(e.subarray(t, t + this.blockSize)), o = e.subarray(t + 1, t + this.blockSize + 1), s = new Int32Array(o);
+					let t = c.shuffledIndexes[c.step++] * this.blockSize;
+					if (c.step >= c.shuffledIndexes.length && (c.step = 0, $(c.shuffledIndexes)), t + this.blockSize + 1 > a) continue;
+					let n = new Int32Array(m(e, t, t + this.blockSize)), o = new Int32Array(m(e, t + 1, t + this.blockSize + 1));
 					if (r) {
-						let e = 0;
-						for (let n = 0; n < s.length; n++) r[t + 1 + n] === 0 && (s[n] = i, e++);
-						if (e === s.length) continue;
+						let e = 0, n = h(r, t + 1, t + this.blockSize + 1);
+						for (let t = 0; t < o.length; t++) n[t] === 0 && (o[t] = i, e++);
+						if (e === o.length) continue;
 					}
 					yield {
 						xs: n,
-						ys: s
+						ys: o
 					};
 				}
 			}.bind(this)).batch(t).map((e) => {
@@ -704,9 +698,9 @@ var re = class {
 					ys: t.ys.cast("int32")
 				}));
 			}).prefetch(2),
-			state: a
+			state: c
 		};
 	}
 };
 //#endregion
-export { $ as a, ne as i, Q as n, te as r, re as t };
+export { $ as i, te as n, ne as r, re as t };

package/dist/Generator.js

@@ -1,2 +1,2 @@
-import { c as e, s as t } from "./main-BTmbwF1h.js";
+import { c as e, s as t } from "./main-Bgc7_9kb.js";
 export { t as default, e as isConversation };

package/dist/{RealDiv-DBu0FQqT.js → RealDiv-CNsvC4AU.js}

@@ -1,9 +1,9 @@
-import { Dn as e, En as t, Io as n, Ks as r, Ms as i, Si as a, Tn as o, nc as s, oc as c, wn as l, xn as u } from "./dist-BewPQWjc.js";
-import { L as d } from "./backend_util-Cg-roD1p.js";
-import { o as f } from "./gpgpu_math-DvLcCH6u.js";
-import { J as p, b as m } from "./shared-DOgWaqvL.js";
-import { S as h, n as g } from "./kernel_funcs_utils-HiXOOx3f.js";
-import { t as _ } from "./Reshape-CABOPB9d.js";
+import { Dn as e, En as t, Io as n, Ks as r, Ms as i, Si as a, Tn as o, nc as s, oc as c, wn as l, xn as u } from "./dist-Da20xy8E.js";
+import { L as d } from "./backend_util-DAV0I34P.js";
+import { o as f } from "./gpgpu_math-DBYEAAdI.js";
+import { J as p, b as m } from "./shared-CfTzpULd.js";
+import { S as h, n as g } from "./kernel_funcs_utils-D-mATnGR.js";
+import { t as _ } from "./Reshape-dnm9bO3B.js";
 //#region node_modules/@tensorflow/tfjs-backend-webgl/dist/mean_gpu.js
 var v = class {
 	constructor(e, t) {

package/dist/{Reshape-DqO3r8BC.js → Reshape-CwxdaU8n.js}

@@ -1,4 +1,4 @@
-import { Bo as e, Gs as t, Ps as n, oc as r } from "./dist-BewPQWjc.js";
+import { Bo as e, Gs as t, Ps as n, oc as r } from "./dist-Da20xy8E.js";
 //#region node_modules/@tensorflow/tfjs-backend-webgpu/dist/kernels/Reshape.js
 function i(e) {
 	let { inputs: i, attrs: a } = e, { x: o } = i, { shape: s } = a, c = r(o.shape), l = t(s, c), u = r(l);

package/dist/{Reshape-CABOPB9d.js → Reshape-dnm9bO3B.js}

@@ -1,5 +1,5 @@
-import { Bo as e, Gs as t, Ps as n, oc as r } from "./dist-BewPQWjc.js";
-import { E as i, a, c as o, d as s, j as c, l, u, z as d } from "./gpgpu_math-DvLcCH6u.js";
+import { Bo as e, Gs as t, Ps as n, oc as r } from "./dist-Da20xy8E.js";
+import { E as i, a, c as o, d as s, j as c, l, u, z as d } from "./gpgpu_math-DBYEAAdI.js";
 //#region node_modules/@tensorflow/tfjs-backend-webgl/dist/reshape_packed_gpu.js
 var f = class {
 	constructor(e, t) {

package/dist/TeachableLLM.d.ts

@@ -4,6 +4,7 @@ import { SaveOptions } from './loader/save';
 import { LoadModelOptions } from './loader/load';
 import { IGenerateOptions, IGenerator } from './Generator';
 import { default as Trainer, TrainingType } from './Trainer';
+import { ConversationStream } from './main';
 import { default as MemoryProfiler } from './utilities/profile';
 import { default as Model, ModelForwardAttributes } from './models/model';
 import { Task } from './training/tasks/Task';
@@ -51,7 +52,7 @@ export default class TeachableLLM {
     getNumParams(): number;
     trainer(trainingType?: TrainingType, options?: TrainingOptions): Trainer;
     train(text: Task[], options?: TrainingOptions, trainingType?: TrainingType): Promise<void>;
-    trainTokeniser(text: Conversation[][]): Promise<number>;
+    trainTokeniser(text: ConversationStream[]): Promise<number>;
     generator(): IGenerator;
     generateText(prompt: Conversation[], options?: IGenerateOptions): Promise<Conversation[]>;
     generateText(options?: IGenerateOptions): Promise<Conversation[]>;

package/dist/TeachableLLM.js

@@ -1,2 +1,2 @@
-import { i as e } from "./main-BTmbwF1h.js";
+import { i as e } from "./main-Bgc7_9kb.js";
 export { e as default };

package/dist/Trainer.d.ts

@@ -33,7 +33,7 @@ export default class Trainer extends EE<'start' | 'stop' | 'log'> {
     dispose(): void;
     getTotalTokens(): number;
     setOptions(options: TrainingOptions): void;
-    prepare(tasks?: Task[] | Uint16Array, datasets?: DatasetMetadata[]): Promise<void>;
+    prepare(tasks?: Task[] | Uint16Array[], datasets?: DatasetMetadata[]): Promise<void>;
     private configureModel;
     train(): Promise<void>;
     step(options?: TrainingOptions): Promise<void>;

package/dist/Trainer.js

@@ -1,2 +1,2 @@
-import { a as e } from "./main-BTmbwF1h.js";
+import { a as e } from "./main-Bgc7_9kb.js";
 export { e as default };