@genai-fi/nanogpt 0.21.0 → 0.23.0

package/dist/{DatasetBuilder-B9UGNQT5.js → DatasetBuilder-C0iJT29K.js}

@@ -66,9 +66,9 @@ function S(e) {
 //#endregion
 //#region node_modules/@tensorflow/tfjs-data/dist/util/deep_clone.js
 function C(e) {
-	return h(e, w);
+	return h(e, ee);
 }
-function w(e) {
+function ee(e) {
 	return e instanceof f ? {
 		value: e.clone(),
 		recurse: !1
@@ -82,7 +82,7 @@ function w(e) {
 }
 //#endregion
 //#region node_modules/@tensorflow/tfjs-data/dist/util/ring_buffer.js
-var T = class {
+var w = class {
 	constructor(e) {
 		if (this.capacity = e, this.begin = 0, this.end = 0, e == null) throw RangeError("Can't create a ring buffer of unknown capacity.");
 		if (e < 1) throw RangeError("Can't create ring buffer of capacity < 1.");
@@ -137,7 +137,7 @@ var T = class {
 		let t = this.wrap(this.begin + e), n = this.get(t);
 		return this.set(t, this.pop()), n;
 	}
-}, E = class e extends T {
+}, T = class e extends w {
 	constructor() {
 		super(e.INITIAL_CAPACITY);
 	}
@@ -156,19 +156,19 @@ var T = class {
 		this.data = t, this.capacity = e, this.doubledCapacity = 2 * this.capacity, this.begin = 0, this.end = n;
 	}
 };
-E.INITIAL_CAPACITY = 32;
+T.INITIAL_CAPACITY = 32;
 //#endregion
 //#region node_modules/@tensorflow/tfjs-data/dist/iterators/lazy_iterator.js
-function ee(e) {
-	return new A(e);
+function E(e) {
+	return new k(e);
 }
 function D(e) {
-	return new j(e);
+	return new A(e);
 }
-function O(e, t) {
-	return new H(e, t);
+function te(e, t) {
+	return new V(e, t);
 }
-var k = class {
+var O = class {
 	async toArray() {
 		let e = [], t = await this.next();
 		for (; !t.done;) e.push(t.value), t = await this.next();
@@ -188,22 +188,22 @@ var k = class {
 		for (; !t.done && n;) t = await this.next(), n = e(t.value);
 	}
 	handleErrors(e) {
-		return new R(this, e);
+		return new L(this, e);
 	}
 	filter(e) {
-		return new I(this, e);
+		return new F(this, e);
 	}
 	map(e) {
-		return new L(this, e);
+		return new I(this, e);
 	}
 	mapAsync(e) {
-		return new z(this, e);
+		return new R(this, e);
 	}
 	serialMapAsync(e) {
-		return new z(this, e).serial();
+		return new R(this, e).serial();
 	}
 	flatmap(e) {
-		return new V(this, e);
+		return new B(this, e);
 	}
 	async forEachAsync(e) {
 		return this.map(e).resolveFully();
@@ -212,30 +212,30 @@ var k = class {
 		return this.serialMapAsync(e).resolveWhile((e) => e === !0);
 	}
 	rowMajorBatch(e, t = !0) {
-		return new F(this, e, t);
+		return new P(this, e, t);
 	}
 	columnMajorBatch(e, t = !0, n = y) {
 		return this.rowMajorBatch(e, t).map((e) => _(e, n));
 	}
 	concatenate(e, t) {
-		return new H(ee([this, e]), t);
+		return new V(E([this, e]), t);
 	}
 	take(e) {
-		return e < 0 || e == null ? this : new P(this, e);
+		return e < 0 || e == null ? this : new N(this, e);
 	}
 	skip(e) {
-		return e < 0 || e == null ? this : new N(this, e);
+		return e < 0 || e == null ? this : new M(this, e);
 	}
 	prefetch(e) {
-		return new W(this, e);
+		return new U(this, e);
 	}
 	shuffle(e, t) {
-		return new G(this, e, t);
+		return new W(this, e, t);
 	}
 	serial() {
-		return new M(this);
+		return new j(this);
 	}
-}, A = class extends k {
+}, k = class extends O {
 	constructor(e) {
 		super(), this.items = e, this.trav = 0;
 	}
@@ -253,7 +253,7 @@ var k = class {
 			done: !1
 		};
 	}
-}, j = class extends k {
+}, A = class extends O {
 	constructor(e) {
 		super(), this.nextFn = e;
 	}
@@ -267,7 +267,7 @@ var k = class {
 			throw e.message = `Error thrown while iterating through a dataset: ${e.message}`, e;
 		}
 	}
-}, M = class extends k {
+}, j = class extends O {
 	constructor(e) {
 		super(), this.upstream = e, this.lastRead = Promise.resolve({
 			value: null,
@@ -283,7 +283,7 @@ var k = class {
 	async serialNext() {
 		return this.upstream.next();
 	}
-}, N = class extends k {
+}, M = class extends O {
 	constructor(e, t) {
 		super(), this.upstream = e, this.maxCount = t, this.count = 0, this.lastRead = Promise.resolve({
 			value: null,
@@ -304,7 +304,7 @@ var k = class {
 		}
 		return this.upstream.next();
 	}
-}, P = class extends k {
+}, N = class extends O {
 	constructor(e, t) {
 		super(), this.upstream = e, this.maxCount = t, this.count = 0;
 	}
@@ -317,7 +317,7 @@ var k = class {
 			done: !0
 		} : this.upstream.next();
 	}
-}, F = class extends k {
+}, P = class extends O {
 	constructor(e, t, n = !0) {
 		super(), this.upstream = e, this.batchSize = t, this.enableSmallLastBatch = n, this.lastRead = Promise.resolve({
 			value: null,
@@ -348,7 +348,7 @@ var k = class {
 			done: !1
 		};
 	}
-}, I = class extends k {
+}, F = class extends O {
 	constructor(e, t) {
 		super(), this.upstream = e, this.predicate = t, this.lastRead = Promise.resolve({
 			value: null,
@@ -368,7 +368,7 @@ var k = class {
 			d(e.value);
 		}
 	}
-}, L = class extends k {
+}, I = class extends O {
 	constructor(e, t) {
 		super(), this.upstream = e, this.transform = t;
 	}
@@ -388,7 +388,7 @@ var k = class {
 			done: !1
 		};
 	}
-}, R = class extends k {
+}, L = class extends O {
 	constructor(e, t) {
 		super(), this.upstream = e, this.handler = t, this.count = 0, this.lastRead = Promise.resolve({
 			value: null,
@@ -411,7 +411,7 @@ var k = class {
 			};
 		}
 	}
-}, z = class extends k {
+}, R = class extends O {
 	constructor(e, t) {
 		super(), this.upstream = e, this.transform = t;
 	}
@@ -431,9 +431,9 @@ var k = class {
 			done: !1
 		};
 	}
-}, B = class extends k {
+}, z = class extends O {
 	constructor() {
-		super(), this.outputQueue = new E(), this.lastRead = Promise.resolve({
+		super(), this.outputQueue = new T(), this.lastRead = Promise.resolve({
 			value: null,
 			done: !1
 		});
@@ -451,7 +451,7 @@ var k = class {
 			done: !1
 		};
 	}
-}, V = class extends B {
+}, B = class extends z {
 	constructor(e, t) {
 		super(), this.upstream = e, this.transform = t;
 	}
@@ -466,7 +466,7 @@ var k = class {
 		for (let e of t) o(e, r) || e.dispose();
 		return !0;
 	}
-}, H = class extends k {
+}, V = class extends O {
 	constructor(e, t) {
 		super(), this.baseErrorHandler = t, this.lastRead = null, this.iterator = null, this.moreIterators = e;
 	}
@@ -488,13 +488,13 @@ var k = class {
 		let t = await this.iterator.next();
 		return t.done ? (this.iterator = null, this.readFromChain(e)) : t;
 	}
-}, U;
+}, H;
 (function(e) {
 	e[e.FAIL = 0] = "FAIL", e[e.SHORTEST = 1] = "SHORTEST", e[e.LONGEST = 2] = "LONGEST";
-})(U ||= {});
-var W = class extends k {
+})(H ||= {});
+var U = class extends O {
 	constructor(e, t) {
-		super(), this.upstream = e, this.bufferSize = t, this.buffer = new T(t);
+		super(), this.upstream = e, this.bufferSize = t, this.buffer = new w(t);
 	}
 	summary() {
 		return `${this.upstream.summary()} -> Prefetch`;
@@ -508,7 +508,7 @@ var W = class extends k {
 	next() {
 		return this.refill(), this.buffer.shift();
 	}
-}, G = class extends W {
+}, W = class extends U {
 	constructor(e, n, r) {
 		super(e, n), this.upstream = e, this.windowSize = n, this.upstreamExhausted = !1, this.random = m.alea(r || t().toString()), this.lastRead = Promise.resolve({
 			value: null,
@@ -535,7 +535,7 @@ var W = class extends k {
 			done: !0
 		};
 	}
-}, K = class {
+}, G = class {
 	constructor() {
 		this.size = null;
 	}
@@ -544,54 +544,54 @@ var W = class extends k {
 		r(e > 0, () => `batchSize needs to be positive, but it is
       ${e}`);
 		let i;
-		return i = this.size === Infinity || this.size == null ? this.size : t ? Math.ceil(this.size / e) : Math.floor(this.size / e), q(async () => (await n.iterator()).columnMajorBatch(e, t, J), i);
+		return i = this.size === Infinity || this.size == null ? this.size : t ? Math.ceil(this.size / e) : Math.floor(this.size / e), K(async () => (await n.iterator()).columnMajorBatch(e, t, q), i);
 	}
 	concatenate(e) {
 		let t = this, n;
-		return n = this.size === Infinity || e.size === Infinity ? Infinity : this.size != null && e.size != null ? this.size + e.size : null, q(async () => (await t.iterator()).concatenate(await e.iterator()), n);
+		return n = this.size === Infinity || e.size === Infinity ? Infinity : this.size != null && e.size != null ? this.size + e.size : null, K(async () => (await t.iterator()).concatenate(await e.iterator()), n);
 	}
 	filter(e) {
 		let t = this, n;
-		return n = this.size === Infinity ? Infinity : null, q(async () => (await t.iterator()).filter((t) => s(() => e(t))), n);
+		return n = this.size === Infinity ? Infinity : null, K(async () => (await t.iterator()).filter((t) => s(() => e(t))), n);
 	}
 	async forEachAsync(e) {
 		return (await this.iterator()).forEachAsync(e);
 	}
 	map(e) {
 		let t = this;
-		return q(async () => (await t.iterator()).map((t) => s(() => e(t))), this.size);
+		return K(async () => (await t.iterator()).map((t) => s(() => e(t))), this.size);
 	}
 	mapAsync(e) {
 		let t = this;
-		return q(async () => (await t.iterator()).mapAsync(e), this.size);
+		return K(async () => (await t.iterator()).mapAsync(e), this.size);
 	}
 	prefetch(e) {
 		if (e == null) throw RangeError("`Dataset.prefetch()` requires bufferSize to be specified.");
 		let t = this;
-		return q(async () => (await t.iterator()).prefetch(e), this.size);
+		return K(async () => (await t.iterator()).prefetch(e), this.size);
 	}
 	repeat(e) {
 		let t = this, n;
-		return n = this.size != null && e > 0 ? this.size * e : e === 0 ? 0 : this.size != null && (e === void 0 || e < 0) ? Infinity : null, q(async () => O(D(async () => ({
+		return n = this.size != null && e > 0 ? this.size * e : e === 0 ? 0 : this.size != null && (e === void 0 || e < 0) ? Infinity : null, K(async () => te(D(async () => ({
 			value: await t.iterator(),
 			done: !1
 		})).take(e)), n);
 	}
 	skip(e) {
 		let t = this, n;
-		return n = this.size != null && e >= 0 && this.size >= e ? this.size - e : this.size != null && (this.size < e || e === void 0 || e < 0) ? 0 : null, q(async () => (await t.iterator()).skip(e), n);
+		return n = this.size != null && e >= 0 && this.size >= e ? this.size - e : this.size != null && (this.size < e || e === void 0 || e < 0) ? 0 : null, K(async () => (await t.iterator()).skip(e), n);
 	}
 	shuffle(e, n, r = !0) {
 		if (e == null || e < 0) throw this.size == null ? RangeError("`Dataset.shuffle()` requires bufferSize to be specified.") : RangeError(`\`Dataset.shuffle()\` requires bufferSize to be specified.  If your data fits in main memory (for regular JS objects), and/or GPU memory (for \`tf.Tensor\`s), consider setting bufferSize to the dataset size (${this.size} elements)`);
 		let i = this, a = m.alea(n || t().toString());
-		return q(async () => {
+		return K(async () => {
 			let t = a.int32();
 			return r && (t += a.int32()), (await i.iterator()).shuffle(e, t.toString());
 		}, this.size);
 	}
 	take(e) {
 		let t = this, n;
-		return n = this.size != null && this.size > e ? e : this.size != null && this.size <= e ? this.size : null, q(async () => (await t.iterator()).take(e), n);
+		return n = this.size != null && this.size > e ? e : this.size != null && this.size <= e ? this.size : null, K(async () => (await t.iterator()).take(e), n);
 	}
 	async toArray() {
 		if (this.size === Infinity) throw Error("Can not convert infinite data stream to array.");
@@ -602,9 +602,9 @@ var W = class extends k {
 		return (await this.iterator()).toArrayForTest();
 	}
 };
-K.MAX_BUFFER_SIZE = 1e4;
-function q(e, t = null) {
-	return new class extends K {
+G.MAX_BUFFER_SIZE = 1e4;
+function K(e, t = null) {
+	return new class extends G {
 		constructor() {
 			super(...arguments), this.size = t;
 		}
@@ -613,94 +613,123 @@ function q(e, t = null) {
 		}
 	}();
 }
-function J(e) {
+function q(e) {
 	if (e === null) return null;
 	let t = e[0];
 	return x(t) ? {
-		value: Y(e),
+		value: J(e),
 		recurse: !1
 	} : {
 		value: null,
 		recurse: !0
 	};
 }
-function Y(e) {
+function J(e) {
 	if (e.length === 0) throw Error("Can't make a batch of zero elements.");
 	return e[0] instanceof f ? i(e) : c(e);
 }
 //#endregion
 //#region node_modules/@tensorflow/tfjs-data/dist/readers.js
-function X(e) {
-	return q(async () => {
+function Y(e) {
+	return K(async () => {
 		let t = await e();
 		return D(() => t.next());
 	});
 }
 //#endregion
 //#region lib/training/DatasetBuilder.ts
-function Z(e, t) {
+function X(e, t) {
 	let n = e.map((e) => t.encodeConversation(e)).flat();
 	return new Uint16Array(n);
 }
-function Q(e, t) {
-	let n = e.map((e) => t.encodeConversation(e, !1, !0));
-	console.log("Tokenised Texts with Mask:", n);
-	let r = n.map((e) => e.tokens).flat(), i = n.map((e) => e.mask).flat();
+function Z(e, t) {
+	let n = e.map((e) => t.encodeConversation(e, !1, !0)), r = n.map((e) => e.tokens).flat(), i = n.map((e) => e.mask).flat();
 	return {
 		tokens: new Uint16Array(r),
 		mask: new Uint8Array(i.map((e) => +!!e))
 	};
 }
-function $(e) {
+function Q(e) {
 	for (let t = e.length - 1; t > 0; t--) {
 		let n = Math.floor(Math.random() * (t + 1));
 		[e[t], e[n]] = [e[n], e[t]];
 	}
 	return e;
 }
-var te = class {
+async function $(e, t, n) {
+	e.step += 1;
+	let r = e.shardIndex === e.shuffledShards.length - 1 ? e.lastShardIndexes : e.shuffledIndexes;
+	if (e.step >= r.length) {
+		e.step = 0, e.shardIndex += 1, e.shardIndex >= e.shuffledShards.length && (e.shardIndex = 0, n || (Q(e.shuffledShards), Q(e.shuffledIndexes), Q(e.lastShardIndexes))), e.nextShard ? (e.currentShard = e.nextShard, e.nextShard = null) : e.currentShard = await t.getShard(e.shuffledShards[e.shardIndex]);
+		let r = (e.shardIndex + 1) % e.shuffledShards.length;
+		if (t.getShard(e.shuffledShards[r]).then((t) => {
+			e.nextShard = t;
+		}), t.hasMask()) {
+			e.nextMask ? (e.currentMask = e.nextMask, e.nextMask = null) : e.currentMask = await t.getMask(e.shuffledShards[e.shardIndex]) ?? null;
+			let n = (e.shardIndex + 1) % e.shuffledShards.length;
+			t.getMask(e.shuffledShards[n]).then((t) => {
+				e.nextMask = t ?? null;
+			});
+		}
+	}
+}
+var ne = class {
 	tokenizer;
 	blockSize;
 	constructor(e, t = 128) {
 		this.tokenizer = e, this.blockSize = t;
 	}
-	async createTextDataset(e, t = 32, n, r, i = 65535) {
-		if (e.length < this.blockSize + 1) throw Error(`Not enough tokens (${e.length}) for block size ${this.blockSize}`);
-		let a = Math.ceil(e.length / this.blockSize), o = {
-			shuffledIndexes: new Uint32Array(a),
+	async createTextDataset(e, t) {
+		let { batchSize: n = 32, noShuffle: r = !1, ignoreIndex: i = 65535 } = t || {}, a = e.getTokenCount();
+		if (a < this.blockSize + 1) throw Error(`Not enough tokens (${a}) for block size ${this.blockSize}`);
+		let o = Math.ceil(e.shardSize / this.blockSize), c = {
+			shuffledShards: new Uint32Array(e.getShardCount()),
+			shuffledIndexes: new Uint32Array(o),
+			lastShardIndexes: new Uint32Array(Math.ceil(e.getShardLength(e.getShardCount() - 1) / this.blockSize)),
+			currentMask: null,
+			nextMask: null,
+			currentShard: null,
+			nextShard: null,
+			shardIndex: 0,
 			step: 0
 		};
-		if (n) o.shuffledIndexes = n;
-		else {
-			o.shuffledIndexes = new Uint32Array(a);
-			for (let e = 0; e < a; e++) o.shuffledIndexes[e] = e;
-		}
-		return {
-			dataset: X(function* () {
+		for (let e = 0; e < c.shuffledShards.length; e++) c.shuffledShards[e] = e;
+		for (let e = 0; e < c.shuffledIndexes.length; e++) c.shuffledIndexes[e] = e;
+		for (let e = 0; e < c.lastShardIndexes.length; e++) c.lastShardIndexes[e] = e;
+		return t?.shuffleFirst && (Q(c.shuffledShards), Q(c.shuffledIndexes), Q(c.lastShardIndexes)), c.currentShard = await e.getShard(c.shuffledShards[c.shardIndex]), c.shardIndex + 1 < c.shuffledShards.length && e.getShard(c.shuffledShards[c.shardIndex + 1]).then((e) => {
+			c.nextShard = e;
+		}), e.hasMask() && (c.currentMask = await e.getMask(c.shuffledShards[c.shardIndex]) ?? null, c.shardIndex + 1 < c.shuffledShards.length && e.getMask(c.shuffledShards[c.shardIndex + 1]).then((e) => {
+			c.nextMask = e ?? null;
+		})), {
+			dataset: Y(async function* () {
 				for (;;) {
-					let t = o.shuffledIndexes[o.step++] * this.blockSize;
-					if (o.step >= o.shuffledIndexes.length && (o.step = 0, $(o.shuffledIndexes)), t + this.blockSize + 1 > e.length) continue;
-					let n = new Int32Array(e.subarray(t, t + this.blockSize)), a = e.subarray(t + 1, t + this.blockSize + 1), s = new Int32Array(a);
-					if (r) {
-						let e = 0;
-						for (let n = 0; n < s.length; n++) r[t + 1 + n] === 0 && (s[n] = i, e++);
-						if (e === s.length) continue;
+					let t = (c.shardIndex === c.shuffledShards.length - 1 ? c.lastShardIndexes : c.shuffledIndexes)[c.step], n = t * this.blockSize, a = c.currentShard, o = c.currentMask, s = $(c, e, r);
+					if (!a) break;
+					if (n + this.blockSize + 1 > a.length) {
+						console.warn("Index out of bounds for current shard, moving to next shard", t, n, a.length);
+						continue;
+					}
+					let l = new Int32Array(a.slice(n, n + this.blockSize)), u = new Int32Array(a.slice(n + 1, n + this.blockSize + 1));
+					if (o) {
+						let e = 0, t = o.slice(n + 1, n + this.blockSize + 1);
+						for (let n = 0; n < u.length; n++) t[n] === 0 && (u[n] = i, e++);
+						if (e === u.length) continue;
 					}
 					yield {
-						xs: n,
-						ys: s
-					};
+						xs: l,
+						ys: u
+					}, await s;
 				}
-			}.bind(this)).batch(t).map((e) => {
+			}.bind(this)).batch(n).map((e) => {
 				let t = e;
 				return s(() => ({
 					xs: t.xs.cast("int32"),
 					ys: t.ys.cast("int32")
 				}));
 			}).prefetch(2),
-			state: o
+			state: c
 		};
 	}
 };
 //#endregion
-export { $ as i, Z as n, Q as r, te as t };
+export { Q as a, $ as i, X as n, Z as r, ne as t };

package/dist/Generator.js

@@ -1,2 +1,2 @@
-import { c as e, s as t } from "./main-DdsoTJAW.js";
-export { t as default, e as isConversation };
+import { a as e, o as t } from "./main-BSaDGH7I.js";
+export { e as default, t as isConversation };

package/dist/TeachableLLM.d.ts

@@ -4,6 +4,7 @@ import { SaveOptions } from './loader/save';
 import { LoadModelOptions } from './loader/load';
 import { IGenerateOptions, IGenerator } from './Generator';
 import { default as Trainer, TrainingType } from './Trainer';
+import { ConversationStream } from './main';
 import { default as MemoryProfiler } from './utilities/profile';
 import { default as Model, ModelForwardAttributes } from './models/model';
 import { Task } from './training/tasks/Task';
@@ -51,7 +52,7 @@ export default class TeachableLLM {
     getNumParams(): number;
     trainer(trainingType?: TrainingType, options?: TrainingOptions): Trainer;
     train(text: Task[], options?: TrainingOptions, trainingType?: TrainingType): Promise<void>;
-    trainTokeniser(text: Conversation[][]): Promise<number>;
+    trainTokeniser(text: ConversationStream[]): Promise<number>;
     generator(): IGenerator;
     generateText(prompt: Conversation[], options?: IGenerateOptions): Promise<Conversation[]>;
     generateText(options?: IGenerateOptions): Promise<Conversation[]>;

package/dist/TeachableLLM.js

@@ -1,2 +1,2 @@
-import { i as e } from "./main-DdsoTJAW.js";
+import { i as e } from "./main-BSaDGH7I.js";
 export { e as default };