@genai-fi/nanogpt 0.21.0 → 0.22.0

package/dist/{DatasetBuilder-B9UGNQT5.js → DatasetBuilder-Ctb425Id.js}

@@ -1,11 +1,12 @@
 import { i as e } from "./chunk-CWhphoD1.js";
 import { Mi as t, Ms as n, Ps as r, V as i, Zr as a, bi as o, di as s, fi as c, ji as l, mt as u, ni as d, wi as f, yi as p } from "./dist-Da20xy8E.js";
+import { sliceUint16Shards as m, sliceUint8Shards as h } from "./utilities/tokens.js";
 //#region node_modules/@tensorflow/tfjs-data/dist/util/deep_map.js
-var m = /* @__PURE__ */ e(u());
-function h(e, t) {
-	return g(e, t);
+var g = /* @__PURE__ */ e(u());
+function ee(e, t) {
+	return _(e, t);
 }
-function g(e, t, n = /* @__PURE__ */ new Map(), r = /* @__PURE__ */ new Set()) {
+function _(e, t, n = /* @__PURE__ */ new Map(), r = /* @__PURE__ */ new Set()) {
 	if (e == null) return null;
 	if (typeof Blob == "function" && e instanceof Blob) return e.slice();
 	if (r.has(e)) throw Error("Circular references are not supported.");
@@ -13,34 +14,34 @@ function g(e, t, n = /* @__PURE__ */ new Map(), r = /* @__PURE__ */ new Set()) {
 	let i = t(e);
 	if (i.recurse && i.value !== null) throw Error("A deep map function may not return both a value and recurse=true.");
 	if (!i.recurse) return n.set(e, i.value), i.value;
-	if (b(e)) {
+	if (x(e)) {
 		let i = Array.isArray(e) ? [] : {};
 		r.add(e);
 		for (let a in e) {
 			let o = e[a];
-			i[a] = g(o, t, n, r);
+			i[a] = _(o, t, n, r);
 		}
 		return r.delete(e), e.__proto__ && (i.__proto__ = e.__proto__), i;
 	} else throw Error(`Can't recurse into non-iterable type: ${e}`);
 }
-function _(e, t = y) {
-	return v(e, t);
+function v(e, t = b) {
+	return y(e, t);
 }
-function v(e, t, n = /* @__PURE__ */ new Set()) {
+function y(e, t, n = /* @__PURE__ */ new Set()) {
 	let r = e[0];
 	if (n.has(r)) throw Error("Circular references are not supported.");
 	let i = t(e);
 	if (i.recurse && i.value !== null) throw Error("A deep zip function may not return both a value and recurse=true.");
 	if (!i.recurse) return i.value;
-	if (b(r)) {
+	if (x(r)) {
 		let i = Array.isArray(r) ? [] : {};
 		n.add(r);
-		for (let a in r) i[a] = v(e.map((e) => e[a]), t, n);
+		for (let a in r) i[a] = y(e.map((e) => e[a]), t, n);
 		return n.delete(r), i;
 	} else throw Error(`Can't recurse into non-iterable type: ${r}`);
 }
-function y(e) {
-	return e === null ? null : b(e[0]) ? {
+function b(e) {
+	return e === null ? null : x(e[0]) ? {
 		value: null,
 		recurse: !0
 	} : {
@@ -48,7 +49,7 @@ function y(e) {
 		recurse: !1
 	};
 }
-function b(e) {
+function x(e) {
 	let t = !1;
 	if (n().get("IS_BROWSER")) t = e instanceof TextDecoder;
 	else {
@@ -57,22 +58,22 @@ function b(e) {
 	}
 	return e != null && !ArrayBuffer.isView(e) && (Array.isArray(e) || typeof e == "object" && !(e instanceof f) && !(e instanceof Promise) && !t);
 }
-function x(e) {
-	return e == null || S(e) || Array.isArray(e) || typeof e == "object" && e instanceof f || l(e);
-}
 function S(e) {
+	return e == null || C(e) || Array.isArray(e) || typeof e == "object" && e instanceof f || l(e);
+}
+function C(e) {
 	return e === null || typeof e != "object" && typeof e != "function";
 }
 //#endregion
 //#region node_modules/@tensorflow/tfjs-data/dist/util/deep_clone.js
-function C(e) {
-	return h(e, w);
-}
 function w(e) {
+	return ee(e, T);
+}
+function T(e) {
 	return e instanceof f ? {
 		value: e.clone(),
 		recurse: !1
-	} : b(e) ? {
+	} : x(e) ? {
 		value: null,
 		recurse: !0
 	} : {
@@ -82,7 +83,7 @@ function w(e) {
 }
 //#endregion
 //#region node_modules/@tensorflow/tfjs-data/dist/util/ring_buffer.js
-var T = class {
+var E = class {
 	constructor(e) {
 		if (this.capacity = e, this.begin = 0, this.end = 0, e == null) throw RangeError("Can't create a ring buffer of unknown capacity.");
 		if (e < 1) throw RangeError("Can't create ring buffer of capacity < 1.");
@@ -137,7 +138,7 @@ var T = class {
 		let t = this.wrap(this.begin + e), n = this.get(t);
 		return this.set(t, this.pop()), n;
 	}
-}, E = class e extends T {
+}, D = class e extends E {
 	constructor() {
 		super(e.INITIAL_CAPACITY);
 	}
@@ -156,19 +157,19 @@ var T = class {
 		this.data = t, this.capacity = e, this.doubledCapacity = 2 * this.capacity, this.begin = 0, this.end = n;
 	}
 };
-E.INITIAL_CAPACITY = 32;
+D.INITIAL_CAPACITY = 32;
 //#endregion
 //#region node_modules/@tensorflow/tfjs-data/dist/iterators/lazy_iterator.js
-function ee(e) {
-	return new A(e);
+function O(e) {
+	return new M(e);
 }
-function D(e) {
-	return new j(e);
+function k(e) {
+	return new N(e);
 }
-function O(e, t) {
-	return new H(e, t);
+function A(e, t) {
+	return new W(e, t);
 }
-var k = class {
+var j = class {
 	async toArray() {
 		let e = [], t = await this.next();
 		for (; !t.done;) e.push(t.value), t = await this.next();
@@ -188,22 +189,22 @@ var k = class {
 		for (; !t.done && n;) t = await this.next(), n = e(t.value);
 	}
 	handleErrors(e) {
-		return new R(this, e);
+		return new B(this, e);
 	}
 	filter(e) {
-		return new I(this, e);
+		return new R(this, e);
 	}
 	map(e) {
-		return new L(this, e);
+		return new z(this, e);
 	}
 	mapAsync(e) {
-		return new z(this, e);
+		return new V(this, e);
 	}
 	serialMapAsync(e) {
-		return new z(this, e).serial();
+		return new V(this, e).serial();
 	}
 	flatmap(e) {
-		return new V(this, e);
+		return new U(this, e);
 	}
 	async forEachAsync(e) {
 		return this.map(e).resolveFully();
@@ -212,30 +213,30 @@ var k = class {
 		return this.serialMapAsync(e).resolveWhile((e) => e === !0);
 	}
 	rowMajorBatch(e, t = !0) {
-		return new F(this, e, t);
+		return new L(this, e, t);
 	}
-	columnMajorBatch(e, t = !0, n = y) {
-		return this.rowMajorBatch(e, t).map((e) => _(e, n));
+	columnMajorBatch(e, t = !0, n = b) {
+		return this.rowMajorBatch(e, t).map((e) => v(e, n));
 	}
 	concatenate(e, t) {
-		return new H(ee([this, e]), t);
+		return new W(O([this, e]), t);
 	}
 	take(e) {
-		return e < 0 || e == null ? this : new P(this, e);
+		return e < 0 || e == null ? this : new I(this, e);
 	}
 	skip(e) {
-		return e < 0 || e == null ? this : new N(this, e);
+		return e < 0 || e == null ? this : new F(this, e);
 	}
 	prefetch(e) {
-		return new W(this, e);
+		return new K(this, e);
 	}
 	shuffle(e, t) {
-		return new G(this, e, t);
+		return new q(this, e, t);
 	}
 	serial() {
-		return new M(this);
+		return new P(this);
 	}
-}, A = class extends k {
+}, M = class extends j {
 	constructor(e) {
 		super(), this.items = e, this.trav = 0;
 	}
@@ -249,11 +250,11 @@ var k = class {
 		};
 		let e = this.items[this.trav];
 		return this.trav++, {
-			value: C(e),
+			value: w(e),
 			done: !1
 		};
 	}
-}, j = class extends k {
+}, N = class extends j {
 	constructor(e) {
 		super(), this.nextFn = e;
 	}
@@ -267,7 +268,7 @@ var k = class {
 			throw e.message = `Error thrown while iterating through a dataset: ${e.message}`, e;
 		}
 	}
-}, M = class extends k {
+}, P = class extends j {
 	constructor(e) {
 		super(), this.upstream = e, this.lastRead = Promise.resolve({
 			value: null,
@@ -283,7 +284,7 @@ var k = class {
 	async serialNext() {
 		return this.upstream.next();
 	}
-}, N = class extends k {
+}, F = class extends j {
 	constructor(e, t) {
 		super(), this.upstream = e, this.maxCount = t, this.count = 0, this.lastRead = Promise.resolve({
 			value: null,
@@ -304,7 +305,7 @@ var k = class {
 		}
 		return this.upstream.next();
 	}
-}, P = class extends k {
+}, I = class extends j {
 	constructor(e, t) {
 		super(), this.upstream = e, this.maxCount = t, this.count = 0;
 	}
@@ -317,7 +318,7 @@ var k = class {
 			done: !0
 		} : this.upstream.next();
 	}
-}, F = class extends k {
+}, L = class extends j {
 	constructor(e, t, n = !0) {
 		super(), this.upstream = e, this.batchSize = t, this.enableSmallLastBatch = n, this.lastRead = Promise.resolve({
 			value: null,
@@ -348,7 +349,7 @@ var k = class {
 			done: !1
 		};
 	}
-}, I = class extends k {
+}, R = class extends j {
 	constructor(e, t) {
 		super(), this.upstream = e, this.predicate = t, this.lastRead = Promise.resolve({
 			value: null,
@@ -368,7 +369,7 @@ var k = class {
 			d(e.value);
 		}
 	}
-}, L = class extends k {
+}, z = class extends j {
 	constructor(e, t) {
 		super(), this.upstream = e, this.transform = t;
 	}
@@ -388,7 +389,7 @@ var k = class {
 			done: !1
 		};
 	}
-}, R = class extends k {
+}, B = class extends j {
 	constructor(e, t) {
 		super(), this.upstream = e, this.handler = t, this.count = 0, this.lastRead = Promise.resolve({
 			value: null,
@@ -411,7 +412,7 @@ var k = class {
 			};
 		}
 	}
-}, z = class extends k {
+}, V = class extends j {
 	constructor(e, t) {
 		super(), this.upstream = e, this.transform = t;
 	}
@@ -431,9 +432,9 @@ var k = class {
 			done: !1
 		};
 	}
-}, B = class extends k {
+}, H = class extends j {
 	constructor() {
-		super(), this.outputQueue = new E(), this.lastRead = Promise.resolve({
+		super(), this.outputQueue = new D(), this.lastRead = Promise.resolve({
 			value: null,
 			done: !1
 		});
@@ -451,7 +452,7 @@ var k = class {
 			done: !1
 		};
 	}
-}, V = class extends B {
+}, U = class extends H {
 	constructor(e, t) {
 		super(), this.upstream = e, this.transform = t;
 	}
@@ -466,7 +467,7 @@ var k = class {
 		for (let e of t) o(e, r) || e.dispose();
 		return !0;
 	}
-}, H = class extends k {
+}, W = class extends j {
 	constructor(e, t) {
 		super(), this.baseErrorHandler = t, this.lastRead = null, this.iterator = null, this.moreIterators = e;
 	}
@@ -488,13 +489,13 @@ var k = class {
 		let t = await this.iterator.next();
 		return t.done ? (this.iterator = null, this.readFromChain(e)) : t;
 	}
-}, U;
+}, G;
 (function(e) {
 	e[e.FAIL = 0] = "FAIL", e[e.SHORTEST = 1] = "SHORTEST", e[e.LONGEST = 2] = "LONGEST";
-})(U ||= {});
-var W = class extends k {
+})(G ||= {});
+var K = class extends j {
 	constructor(e, t) {
-		super(), this.upstream = e, this.bufferSize = t, this.buffer = new T(t);
+		super(), this.upstream = e, this.bufferSize = t, this.buffer = new E(t);
 	}
 	summary() {
 		return `${this.upstream.summary()} -> Prefetch`;
@@ -508,9 +509,9 @@ var W = class extends k {
 	next() {
 		return this.refill(), this.buffer.shift();
 	}
-}, G = class extends W {
+}, q = class extends K {
 	constructor(e, n, r) {
-		super(e, n), this.upstream = e, this.windowSize = n, this.upstreamExhausted = !1, this.random = m.alea(r || t().toString()), this.lastRead = Promise.resolve({
+		super(e, n), this.upstream = e, this.windowSize = n, this.upstreamExhausted = !1, this.random = g.alea(r || t().toString()), this.lastRead = Promise.resolve({
 			value: null,
 			done: !1
 		});
@@ -535,7 +536,7 @@ var W = class extends k {
 			done: !0
 		};
 	}
-}, K = class {
+}, J = class {
 	constructor() {
 		this.size = null;
 	}
@@ -544,54 +545,54 @@ var W = class extends k {
 		r(e > 0, () => `batchSize needs to be positive, but it is
       ${e}`);
 		let i;
-		return i = this.size === Infinity || this.size == null ? this.size : t ? Math.ceil(this.size / e) : Math.floor(this.size / e), q(async () => (await n.iterator()).columnMajorBatch(e, t, J), i);
+		return i = this.size === Infinity || this.size == null ? this.size : t ? Math.ceil(this.size / e) : Math.floor(this.size / e), Y(async () => (await n.iterator()).columnMajorBatch(e, t, X), i);
 	}
 	concatenate(e) {
 		let t = this, n;
-		return n = this.size === Infinity || e.size === Infinity ? Infinity : this.size != null && e.size != null ? this.size + e.size : null, q(async () => (await t.iterator()).concatenate(await e.iterator()), n);
+		return n = this.size === Infinity || e.size === Infinity ? Infinity : this.size != null && e.size != null ? this.size + e.size : null, Y(async () => (await t.iterator()).concatenate(await e.iterator()), n);
 	}
 	filter(e) {
 		let t = this, n;
-		return n = this.size === Infinity ? Infinity : null, q(async () => (await t.iterator()).filter((t) => s(() => e(t))), n);
+		return n = this.size === Infinity ? Infinity : null, Y(async () => (await t.iterator()).filter((t) => s(() => e(t))), n);
 	}
 	async forEachAsync(e) {
 		return (await this.iterator()).forEachAsync(e);
 	}
 	map(e) {
 		let t = this;
-		return q(async () => (await t.iterator()).map((t) => s(() => e(t))), this.size);
+		return Y(async () => (await t.iterator()).map((t) => s(() => e(t))), this.size);
 	}
 	mapAsync(e) {
 		let t = this;
-		return q(async () => (await t.iterator()).mapAsync(e), this.size);
+		return Y(async () => (await t.iterator()).mapAsync(e), this.size);
 	}
 	prefetch(e) {
 		if (e == null) throw RangeError("`Dataset.prefetch()` requires bufferSize to be specified.");
 		let t = this;
-		return q(async () => (await t.iterator()).prefetch(e), this.size);
+		return Y(async () => (await t.iterator()).prefetch(e), this.size);
 	}
 	repeat(e) {
 		let t = this, n;
-		return n = this.size != null && e > 0 ? this.size * e : e === 0 ? 0 : this.size != null && (e === void 0 || e < 0) ? Infinity : null, q(async () => O(D(async () => ({
+		return n = this.size != null && e > 0 ? this.size * e : e === 0 ? 0 : this.size != null && (e === void 0 || e < 0) ? Infinity : null, Y(async () => A(k(async () => ({
 			value: await t.iterator(),
 			done: !1
 		})).take(e)), n);
 	}
 	skip(e) {
 		let t = this, n;
-		return n = this.size != null && e >= 0 && this.size >= e ? this.size - e : this.size != null && (this.size < e || e === void 0 || e < 0) ? 0 : null, q(async () => (await t.iterator()).skip(e), n);
+		return n = this.size != null && e >= 0 && this.size >= e ? this.size - e : this.size != null && (this.size < e || e === void 0 || e < 0) ? 0 : null, Y(async () => (await t.iterator()).skip(e), n);
 	}
 	shuffle(e, n, r = !0) {
 		if (e == null || e < 0) throw this.size == null ? RangeError("`Dataset.shuffle()` requires bufferSize to be specified.") : RangeError(`\`Dataset.shuffle()\` requires bufferSize to be specified.  If your data fits in main memory (for regular JS objects), and/or GPU memory (for \`tf.Tensor\`s), consider setting bufferSize to the dataset size (${this.size} elements)`);
-		let i = this, a = m.alea(n || t().toString());
-		return q(async () => {
+		let i = this, a = g.alea(n || t().toString());
+		return Y(async () => {
 			let t = a.int32();
 			return r && (t += a.int32()), (await i.iterator()).shuffle(e, t.toString());
 		}, this.size);
 	}
 	take(e) {
 		let t = this, n;
-		return n = this.size != null && this.size > e ? e : this.size != null && this.size <= e ? this.size : null, q(async () => (await t.iterator()).take(e), n);
+		return n = this.size != null && this.size > e ? e : this.size != null && this.size <= e ? this.size : null, Y(async () => (await t.iterator()).take(e), n);
 	}
 	async toArray() {
 		if (this.size === Infinity) throw Error("Can not convert infinite data stream to array.");
@@ -602,9 +603,9 @@ var W = class extends k {
 		return (await this.iterator()).toArrayForTest();
 	}
 };
-K.MAX_BUFFER_SIZE = 1e4;
-function q(e, t = null) {
-	return new class extends K {
+J.MAX_BUFFER_SIZE = 1e4;
+function Y(e, t = null) {
+	return new class extends J {
 		constructor() {
 			super(...arguments), this.size = t;
 		}
@@ -613,39 +614,37 @@ function q(e, t = null) {
 		}
 	}();
 }
-function J(e) {
+function X(e) {
 	if (e === null) return null;
 	let t = e[0];
-	return x(t) ? {
-		value: Y(e),
+	return S(t) ? {
+		value: Z(e),
 		recurse: !1
 	} : {
 		value: null,
 		recurse: !0
 	};
 }
-function Y(e) {
+function Z(e) {
 	if (e.length === 0) throw Error("Can't make a batch of zero elements.");
 	return e[0] instanceof f ? i(e) : c(e);
 }
 //#endregion
 //#region node_modules/@tensorflow/tfjs-data/dist/readers.js
-function X(e) {
-	return q(async () => {
+function Q(e) {
+	return Y(async () => {
 		let t = await e();
-		return D(() => t.next());
+		return k(() => t.next());
 	});
 }
 //#endregion
 //#region lib/training/DatasetBuilder.ts
-function Z(e, t) {
+function te(e, t) {
 	let n = e.map((e) => t.encodeConversation(e)).flat();
 	return new Uint16Array(n);
 }
-function Q(e, t) {
-	let n = e.map((e) => t.encodeConversation(e, !1, !0));
-	console.log("Tokenised Texts with Mask:", n);
-	let r = n.map((e) => e.tokens).flat(), i = n.map((e) => e.mask).flat();
+function ne(e, t) {
+	let n = e.map((e) => t.encodeConversation(e, !1, !0)), r = n.map((e) => e.tokens).flat(), i = n.map((e) => e.mask).flat();
 	return {
 		tokens: new Uint16Array(r),
 		mask: new Uint8Array(i.map((e) => +!!e))
@@ -658,37 +657,38 @@ function $(e) {
 	}
 	return e;
 }
-var te = class {
+var re = class {
 	tokenizer;
 	blockSize;
 	constructor(e, t = 128) {
 		this.tokenizer = e, this.blockSize = t;
 	}
 	async createTextDataset(e, t = 32, n, r, i = 65535) {
-		if (e.length < this.blockSize + 1) throw Error(`Not enough tokens (${e.length}) for block size ${this.blockSize}`);
-		let a = Math.ceil(e.length / this.blockSize), o = {
-			shuffledIndexes: new Uint32Array(a),
+		let a = e.reduce((e, t) => e + t.length, 0);
+		if (a < this.blockSize + 1) throw Error(`Not enough tokens (${a}) for block size ${this.blockSize}`);
+		let o = Math.ceil(a / this.blockSize), c = {
+			shuffledIndexes: new Uint32Array(o),
 			step: 0
 		};
-		if (n) o.shuffledIndexes = n;
+		if (n) c.shuffledIndexes = n;
 		else {
-			o.shuffledIndexes = new Uint32Array(a);
-			for (let e = 0; e < a; e++) o.shuffledIndexes[e] = e;
+			c.shuffledIndexes = new Uint32Array(o);
+			for (let e = 0; e < o; e++) c.shuffledIndexes[e] = e;
 		}
 		return {
-			dataset: X(function* () {
+			dataset: Q(function* () {
 				for (;;) {
-					let t = o.shuffledIndexes[o.step++] * this.blockSize;
-					if (o.step >= o.shuffledIndexes.length && (o.step = 0, $(o.shuffledIndexes)), t + this.blockSize + 1 > e.length) continue;
-					let n = new Int32Array(e.subarray(t, t + this.blockSize)), a = e.subarray(t + 1, t + this.blockSize + 1), s = new Int32Array(a);
+					let t = c.shuffledIndexes[c.step++] * this.blockSize;
+					if (c.step >= c.shuffledIndexes.length && (c.step = 0, $(c.shuffledIndexes)), t + this.blockSize + 1 > a) continue;
+					let n = new Int32Array(m(e, t, t + this.blockSize)), o = new Int32Array(m(e, t + 1, t + this.blockSize + 1));
 					if (r) {
-						let e = 0;
-						for (let n = 0; n < s.length; n++) r[t + 1 + n] === 0 && (s[n] = i, e++);
-						if (e === s.length) continue;
+						let e = 0, n = h(r, t + 1, t + this.blockSize + 1);
+						for (let t = 0; t < o.length; t++) n[t] === 0 && (o[t] = i, e++);
+						if (e === o.length) continue;
 					}
 					yield {
 						xs: n,
-						ys: s
+						ys: o
 					};
 				}
 			}.bind(this)).batch(t).map((e) => {
@@ -698,9 +698,9 @@ var te = class {
 					ys: t.ys.cast("int32")
 				}));
 			}).prefetch(2),
-			state: o
+			state: c
 		};
 	}
 };
 //#endregion
-export { $ as i, Z as n, Q as r, te as t };
+export { $ as i, te as n, ne as r, re as t };

package/dist/Generator.js

@@ -1,2 +1,2 @@
-import { c as e, s as t } from "./main-DdsoTJAW.js";
+import { c as e, s as t } from "./main-Bgc7_9kb.js";
 export { t as default, e as isConversation };

package/dist/TeachableLLM.d.ts

@@ -4,6 +4,7 @@ import { SaveOptions } from './loader/save';
 import { LoadModelOptions } from './loader/load';
 import { IGenerateOptions, IGenerator } from './Generator';
 import { default as Trainer, TrainingType } from './Trainer';
+import { ConversationStream } from './main';
 import { default as MemoryProfiler } from './utilities/profile';
 import { default as Model, ModelForwardAttributes } from './models/model';
 import { Task } from './training/tasks/Task';
@@ -51,7 +52,7 @@ export default class TeachableLLM {
     getNumParams(): number;
     trainer(trainingType?: TrainingType, options?: TrainingOptions): Trainer;
     train(text: Task[], options?: TrainingOptions, trainingType?: TrainingType): Promise<void>;
-    trainTokeniser(text: Conversation[][]): Promise<number>;
+    trainTokeniser(text: ConversationStream[]): Promise<number>;
     generator(): IGenerator;
     generateText(prompt: Conversation[], options?: IGenerateOptions): Promise<Conversation[]>;
     generateText(options?: IGenerateOptions): Promise<Conversation[]>;

package/dist/TeachableLLM.js

@@ -1,2 +1,2 @@
-import { i as e } from "./main-DdsoTJAW.js";
+import { i as e } from "./main-Bgc7_9kb.js";
 export { e as default };

package/dist/Trainer.d.ts

@@ -33,7 +33,7 @@ export default class Trainer extends EE<'start' | 'stop' | 'log'> {
     dispose(): void;
     getTotalTokens(): number;
     setOptions(options: TrainingOptions): void;
-    prepare(tasks?: Task[] | Uint16Array, datasets?: DatasetMetadata[]): Promise<void>;
+    prepare(tasks?: Task[] | Uint16Array[], datasets?: DatasetMetadata[]): Promise<void>;
     private configureModel;
     train(): Promise<void>;
     step(options?: TrainingOptions): Promise<void>;

package/dist/Trainer.js

@@ -1,2 +1,2 @@
-import { a as e } from "./main-DdsoTJAW.js";
+import { a as e } from "./main-Bgc7_9kb.js";
 export { e as default };

package/dist/data/stream.d.ts

@@ -0,0 +1,24 @@
+import { Conversation } from '../../tokeniser/type';
+export interface ConversationCursor {
+    next(): Promise<Conversation[] | null>;
+}
+export interface ConversationStream {
+    cursor(): ConversationCursor;
+}
+export declare class MemoryConversationStream implements ConversationStream {
+    private conversations;
+    constructor(conversations: Conversation[][]);
+    cursor(): ConversationCursor;
+}
+declare class JSONLFromReadableStream implements ConversationStream {
+    private sourceFactory;
+    constructor(sourceFactory: () => Promise<ReadableStream<Uint8Array>>);
+    cursor(): ConversationCursor;
+}
+export declare class JSONLConversationStream extends JSONLFromReadableStream {
+    constructor(file: File);
+}
+export declare class ZipJSONLConversationStream extends JSONLFromReadableStream {
+    constructor(file: File, preferredEntryName?: string);
+}
+export {};

package/dist/data/stream.js

@@ -0,0 +1,2 @@
+import { n as e, r as t, t as n } from "../stream-DKl3GTDL.js";
+export { n as JSONLConversationStream, e as MemoryConversationStream, t as ZipJSONLConversationStream };

package/dist/data/textLoader.d.ts

@@ -1,7 +1,7 @@
-import { Conversation } from '../tokeniser/type';
+import { ConversationStream } from './stream';
 export interface DataOptions {
     maxSize?: number;
     column?: string;
     hasHeader?: boolean;
 }
-export default function loadTextData(file: File, options?: DataOptions, cb?: (progress: number) => void): Promise<Conversation[][]>;
+export default function loadTextData(file: File, options?: DataOptions): Promise<ConversationStream>;