@genai-fi/nanogpt 0.22.0 → 0.23.0

package/dist/main.d.ts

@@ -28,6 +28,7 @@ export type { GPTConfig } from './models/config';
 export { estimateParameterCount, estimateMemoryUsage, estimateTrainingMemoryUsage, estimateResources, validateConfig, } from './utilities/parameters';
 export { default as topP } from './utilities/topP';
 export { sliceUint16Shards, sliceUint8Shards } from './utilities/tokens';
+export { TokenStore, createTokenStore } from './training/tasks/TokenStore';
 export { Task, tokensFromTasks } from './training/tasks/Task';
 export declare const tasks: {
     PretrainingTask: typeof PretrainingTask;

package/dist/main.js

@@ -1,18 +1,20 @@
-import { a as e, b as t, i as n, n as r, r as i, s as a, t as o, y as s } from "./main-Bgc7_9kb.js";
-import c from "./tokeniser/CharTokeniser.js";
-import l from "./tokeniser/bpe.js";
-import { AdamWOptimizer as u } from "./training/AdamW.js";
-import d from "./utilities/topP.js";
-import f from "./training/Evaluator.js";
-import { sliceUint16Shards as p, sliceUint8Shards as m } from "./utilities/tokens.js";
-import h from "./utilities/waitForModel.js";
-import g from "./utilities/datasetID.js";
-import { n as _ } from "./stream-DKl3GTDL.js";
-import v from "./data/textLoader.js";
-import { estimateMemoryUsage as y, estimateParameterCount as b, estimateResources as x, estimateTrainingMemoryUsage as S, validateConfig as C } from "./utilities/parameters.js";
-import { Task as w, tokensFromTasks as T } from "./training/tasks/Task.js";
-import { selectBackend as E } from "./backend.js";
-import D from "./utilities/performance.js";
-import O from "./checks/index.js";
-import { sentenceEmbeddings as k, sentenceEmbeddingsTensor as A } from "./utilities/sentences.js";
-export { u as AdamWOptimizer, l as BPETokeniser, c as CharTokeniser, f as Evaluator, a as Generator, _ as MemoryConversationStream, t as Model, s as NanoGPT, w as Task, n as TeachableLLM, e as Trainer, O as checks, y as estimateMemoryUsage, b as estimateParameterCount, x as estimateResources, S as estimateTrainingMemoryUsage, g as generateDatasetID, o as layers, v as loadTextData, r as ops, D as performanceTest, E as selectBackend, k as sentenceEmbeddings, A as sentenceEmbeddingsTensor, p as sliceUint16Shards, m as sliceUint8Shards, i as tasks, T as tokensFromTasks, d as topP, C as validateConfig, h as waitForModel };
+import { _ as e, a as t, i as n, n as r, r as i, t as a, v as o } from "./main-BSaDGH7I.js";
+import s from "./tokeniser/CharTokeniser.js";
+import c from "./tokeniser/bpe.js";
+import { AdamWOptimizer as l } from "./training/AdamW.js";
+import u from "./utilities/topP.js";
+import d from "./training/Evaluator.js";
+import { TokenStore as f, createTokenStore as p } from "./training/tasks/TokenStore.js";
+import { Task as m, tokensFromTasks as h } from "./training/tasks/Task.js";
+import { t as g } from "./Trainer-DBsyWJ4s.js";
+import _ from "./utilities/waitForModel.js";
+import v from "./utilities/datasetID.js";
+import { n as y } from "./stream-BjdpSNqB.js";
+import b from "./data/textLoader.js";
+import { estimateMemoryUsage as x, estimateParameterCount as S, estimateResources as C, estimateTrainingMemoryUsage as w, validateConfig as T } from "./utilities/parameters.js";
+import { sliceUint16Shards as E, sliceUint8Shards as D } from "./utilities/tokens.js";
+import { selectBackend as O } from "./backend.js";
+import k from "./utilities/performance.js";
+import A from "./checks/index.js";
+import { sentenceEmbeddings as j, sentenceEmbeddingsTensor as M } from "./utilities/sentences.js";
+export { l as AdamWOptimizer, c as BPETokeniser, s as CharTokeniser, d as Evaluator, t as Generator, y as MemoryConversationStream, o as Model, e as NanoGPT, m as Task, n as TeachableLLM, f as TokenStore, g as Trainer, A as checks, p as createTokenStore, x as estimateMemoryUsage, S as estimateParameterCount, C as estimateResources, w as estimateTrainingMemoryUsage, v as generateDatasetID, a as layers, b as loadTextData, r as ops, k as performanceTest, O as selectBackend, j as sentenceEmbeddings, M as sentenceEmbeddingsTensor, E as sliceUint16Shards, D as sliceUint8Shards, i as tasks, h as tokensFromTasks, u as topP, T as validateConfig, _ as waitForModel };

package/dist/models/NanoGPTV1.js

@@ -1,2 +1,2 @@
-import { y as e } from "../main-Bgc7_9kb.js";
+import { _ as e } from "../main-BSaDGH7I.js";
 export { e as default };

package/dist/models/NanoGPTV2.js

@@ -1,2 +1,2 @@
-import { v as e } from "../main-Bgc7_9kb.js";
+import { g as e } from "../main-BSaDGH7I.js";
 export { e as default };

package/dist/models/factory.js

@@ -1,2 +1,2 @@
-import { _ as e } from "../main-Bgc7_9kb.js";
+import { h as e } from "../main-BSaDGH7I.js";
 export { e as default };

package/dist/models/model.js

@@ -1,2 +1,2 @@
-import { b as e } from "../main-Bgc7_9kb.js";
+import { v as e } from "../main-BSaDGH7I.js";
 export { e as default };

package/dist/{stream-DKl3GTDL.js → stream-BjdpSNqB.js}

@@ -2225,41 +2225,50 @@ function vi(e) {
 		}];
 	}
 }
-var yi = class {
+var yi = 100, bi = 1e3, xi = class {
 	sourceFactory;
 	constructor(e) {
 		this.sourceFactory = e;
 	}
 	cursor() {
-		let e = !1, t = null, n = new TextDecoder(), r = "", i = !1, a = [], o = async () => {
+		let e = !1, t = null, n = new TextDecoder(), r = "", i = !1, a = !1, o = [], s = async () => {
 			e ||= (t = (await this.sourceFactory()).getReader(), !0);
-		}, s = async () => {
-			for (await o(); a.length === 0 && !i;) {
-				let e = await t.read();
-				if (e.done) {
-					r += n.decode(), i = !0;
-					let e = r.trim();
-					e.length > 0 && a.push(vi(e)), r = "";
-					break;
-				}
-				r += n.decode(e.value, { stream: !0 });
-				let o = r.split("\n");
-				r = o.pop() ?? "";
-				for (let e of o) {
-					let t = e.trim();
-					t.length !== 0 && a.push(vi(t));
+		}, c = null, l = async () => {
+			if (!a) {
+				for (a = !0, await s(); o.length < bi && !i;) {
+					let e = await t.read();
+					if (c && o.length > 0 && (c(), c = null), e.done) {
+						r += n.decode(), i = !0;
+						let e = r.trim();
+						e.length > 0 && o.push(vi(e)), r = "";
+						break;
+					}
+					r += n.decode(e.value, { stream: !0 });
+					let a = r.split("\n");
+					r = a.pop() ?? "";
+					for (let e of a) {
+						let t = e.trim();
+						t.length !== 0 && o.push(vi(t));
+					}
 				}
+				a = !1, c && (o.length > 0 || i) && (c(), c = null);
 			}
-		};
+		}, u = null;
 		return { async next() {
-			return a.length > 0 || await s(), a.shift() ?? null;
+			o.length < yi && !i && !a && (u = l().then(() => {
+				u = null;
+			})), o.length === 0 && u && await new Promise((e) => {
+				c = e;
+			}), o.length === 0 && !i && console.warn("Queue is empty but not done");
+			let e = o.length > 0 ? o.shift() ?? null : null;
+			return e === null && !i && console.warn("Queue is empty and not done"), e;
 		} };
 	}
-}, bi = class extends yi {
+}, Si = class extends xi {
 	constructor(e) {
 		super(async () => e.stream());
 	}
-}, xi = class extends yi {
+}, Ci = class extends xi {
 	constructor(e, t) {
 		super(async () => {
 			let n = new wr(), r = e.stream().pipeThrough(n).getReader(), i = null;
@@ -2284,4 +2293,4 @@ var yi = class {
 	}
 };
 //#endregion
-export { gi as n, xi as r, bi as t };
+export { gi as n, Ci as r, Si as t };

package/dist/tokeniser/BaseTokeniser.js

@@ -1,2 +1,135 @@
-import { n as e, t } from "../BaseTokeniser-C9TSv4th.js";
-export { e as SPECIALS, t as default };
+import { t as e } from "../eventemitter3-D_qV3Lof.js";
+//#region lib/tokeniser/BaseTokeniser.ts
+var t = [
+	"<eos>",
+	"<bos>",
+	"",
+	"<pad>",
+	"<|user_start|>",
+	"<|user_end|>",
+	"<|assistant_start|>",
+	"<|assistant_end|>",
+	"<|system_start|>",
+	"<|system_end|>"
+], n = class extends e {
+	id = "untrained";
+	datasetID;
+	specialTokens = /* @__PURE__ */ new Map();
+	specialTokenSet = /* @__PURE__ */ new Set();
+	isSpecialToken(e) {
+		return this.specialTokenSet.has(e);
+	}
+	addSpecialTokens() {
+		t.forEach((e, t) => {
+			this.addToken(e, t), this.specialTokens.set(e, t), this.specialTokenSet.add(t);
+		});
+	}
+	addSpecialToken(e, t) {
+		this.specialTokens.set(e, t), this.specialTokenSet.add(t);
+	}
+	generateID() {
+		let e = this.getVocab(), t = 2166136261, n = 2654435769;
+		if (e.length === 0) {
+			this.id = "untrained";
+			return;
+		}
+		for (let r = 0; r < e.length; r++) {
+			let i = e[r];
+			t ^= i.length, t = Math.imul(t, 16777619), n ^= r, n = Math.imul(n, 2246822507);
+			for (let e = 0; e < i.length; e++) {
+				let r = i.charCodeAt(e);
+				t ^= r, t = Math.imul(t, 16777619), n ^= r, n = Math.imul(n, 3266489909);
+			}
+		}
+		let r = (t >>> 0).toString(36), i = (n >>> 0).toString(36);
+		this.id = "tokeniser_" + r + "_" + i;
+	}
+	encodeSequence(e) {
+		let t = this.encode(e);
+		return [
+			this.bosToken,
+			...t,
+			this.eosToken
+		];
+	}
+	encodeAsSequence(e, t) {
+		let n = e.flatMap((e) => this.encode(e.content));
+		return t ? [
+			this.bosToken,
+			...n,
+			this.eosToken,
+			this.bosToken
+		] : [
+			this.bosToken,
+			...n,
+			this.eosToken
+		];
+	}
+	encodeConversation(e, t, n) {
+		let r = [[this.bosToken]], i;
+		n && (i = [[!1]]);
+		let a = [
+			this.getSpecialTokenIndex("<|user_start|>"),
+			this.getSpecialTokenIndex("<|assistant_start|>"),
+			this.getSpecialTokenIndex("<|system_start|>")
+		], o = [
+			this.getSpecialTokenIndex("<|user_end|>"),
+			this.getSpecialTokenIndex("<|assistant_end|>"),
+			this.getSpecialTokenIndex("<|system_end|>")
+		];
+		for (let t of e) {
+			let e = !1, s = this.encode(t.content);
+			switch (t.role) {
+				case "user":
+					r.push([a[0]]), e = !0;
+					break;
+				case "assistant":
+					r.push([a[1]]);
+					break;
+				case "system":
+					r.push([a[2]]), e = !0;
+					break;
+			}
+			switch (r.push(s), t.role) {
+				case "user":
+					r.push([o[0]]);
+					break;
+				case "assistant":
+					r.push([o[1]]);
+					break;
+				case "system":
+					r.push([o[2]]);
+					break;
+			}
+			n && i && e ? (i.push([!1]), i.push(s.map(() => !1)), i.push([!1])) : n && i && (i.push([!1]), i.push(s.map(() => !0)), i.push([!0]));
+		}
+		let s = r.flat();
+		return t ? (s.push(a[1]), n && i && i.push([!1])) : (s.push(this.eosToken), n && i && i.push([!0])), n && i ? {
+			tokens: s,
+			mask: i.flat()
+		} : s;
+	}
+	decodeConversation(e) {
+		let t = [], n = 0;
+		for (; n < e.length;) {
+			let r = e[n], i = null;
+			if (r === this.getSpecialTokenIndex("<|user_start|>") ? i = "user" : r === this.getSpecialTokenIndex("<|assistant_start|>") ? i = "assistant" : r === this.getSpecialTokenIndex("<|system_start|>") ? i = "system" : r === this.bosToken || (r === this.eosToken ? i = null : (i = "text", n--)), i) {
+				n++;
+				let r = [];
+				for (; n < e.length && e[n] !== this.getSpecialTokenIndex(`<|${i}_end|>`) && e[n] !== this.eosToken;) r.push(e[n]), n++;
+				let a = this.decode(r);
+				t.push({
+					role: i,
+					content: a
+				});
+			}
+			n++;
+		}
+		return t;
+	}
+	getSpecialTokenIndex(e) {
+		return this.specialTokens.get(e);
+	}
+};
+//#endregion
+export { t as SPECIALS, n as default };

package/dist/tokeniser/CharTokeniser.js

@@ -1,7 +1,7 @@
 import { yieldIfNeeded as e } from "../utilities/yielder.js";
-import { n as t, t as n } from "../BaseTokeniser-C9TSv4th.js";
+import t, { SPECIALS as n } from "./BaseTokeniser.js";
 //#region lib/tokeniser/CharTokeniser.ts
-var r = ["<eos>", "<unk>"], i = class extends n {
+var r = ["<eos>", "<unk>"], i = class extends t {
 	vocabSize = 0;
 	eosToken = 0;
 	bosToken = 0;
@@ -11,7 +11,7 @@ var r = ["<eos>", "<unk>"], i = class extends n {
 	_trained = !1;
 	constructor(e) {
 		if (super(), Array.isArray(e)) {
-			if (this.vocab = e, this.vocab.length > 0) this.vocabSize = this.vocab.length, t.forEach((e) => {
+			if (this.vocab = e, this.vocab.length > 0) this.vocabSize = this.vocab.length, n.forEach((e) => {
 				let t = this.vocab.indexOf(e);
 				t !== -1 && this.addSpecialToken(e, t);
 			}), this.eosToken = this.getSpecialTokenIndex("<eos>"), this.bosToken = this.getSpecialTokenIndex("<bos>") ?? this.eosToken, this.unkToken = this.getSpecialTokenIndex("") ?? -1, this.unkToken === -1 && (this.unkToken = this.vocab.indexOf("<unk>")), this.unkToken === -1 && (this.unkToken = this.vocab.indexOf("<pad>")), this.unkToken === -1 && (this.unkToken = this.vocab.indexOf("_")), this.unkToken === -1 && (this.unkToken = this.vocab.indexOf(" ")), this.unkToken === -1 && (this.unkToken = this.eosToken), this.vocab = this.vocab.map((e) => e === "<pad>" ? "" : e), this.vocab.forEach((e, t) => {

package/dist/tokeniser/bpe.js

@@ -1,5 +1,5 @@
 import { yieldIfNeeded as e } from "../utilities/yielder.js";
-import { n as t, t as n } from "../BaseTokeniser-C9TSv4th.js";
+import t, { SPECIALS as n } from "./BaseTokeniser.js";
 import r from "../utilities/tokenParse.js";
 //#region lib/tokeniser/bpe.ts
 function i(e, t) {
@@ -58,16 +58,16 @@ function l(e, t) {
 		e.tokens[n] = i;
 	}), e.pairs.delete(i(t.a, t.b));
 }
-var u = class extends n {
+var u = class extends t {
 	targetSize;
 	vocab = /* @__PURE__ */ new Set();
 	vocabIndex = /* @__PURE__ */ new Map();
 	merges = [];
 	pretokenMap = /* @__PURE__ */ new Map();
-	constructor(e, n) {
+	constructor(e, t) {
 		super(), Array.isArray(e) ? (e.forEach((e, t) => {
 			this.vocab.add(e), this.vocabIndex.set(e, t);
-		}), n && (this.merges = n), this.targetSize = e.length, t.forEach((t) => {
+		}), t && (this.merges = t), this.targetSize = e.length, n.forEach((t) => {
 			let n = e.indexOf(t);
 			n !== -1 && this.addSpecialToken(t, n);
 		})) : (this.addSpecialTokens(), this.targetSize = e);
@@ -84,7 +84,7 @@ var u = class extends n {
 		this.vocab.clear(), this.vocabIndex.clear(), this.merges = [], this.pretokenMap.clear();
 	}
 	get trained() {
-		return this.vocab.size > t.length && this.vocab.size <= this.targetSize;
+		return this.vocab.size > n.length && this.vocab.size <= this.targetSize;
 	}
 	get vocabSize() {
 		return this.vocab.size;
@@ -101,18 +101,18 @@ var u = class extends n {
 	async train(t = [], n, i) {
 		this.datasetID = i;
 		let o = performance.now(), c = /* @__PURE__ */ new Set();
+		this.vocab = /* @__PURE__ */ new Set(), this.pretokenMap.clear(), this.merges = [], this.addSpecialTokens();
 		for (let i of t) {
 			let t = i.cursor(), a = await t.next();
 			for (; a !== null;) {
 				for (let e of a) {
 					let t = r(e.content);
-					for (let e of t) c.add(e);
+					for (let e of t) c.has(e) || (c.add(e), Array.from(e).forEach((e) => this.vocab.add(e)));
 				}
 				o = await e(o, n, this.vocab.size), a = await t.next();
 			}
 		}
-		this.vocab = /* @__PURE__ */ new Set(), this.pretokenMap.clear(), this.merges = [], this.addSpecialTokens();
-		let u = Array.from(c), d = u.map((e) => Array.from(e).map((e) => (this.vocab.add(e), e))), f = a(d);
+		let u = Array.from(c), d = u.map((e) => Array.from(e)), f = a(d);
 		if (o = await e(o, n, this.vocab.size), this.vocab.size >= this.targetSize) throw console.warn("Initial vocab size is greater than or equal to target size. No merges will be performed.", this.vocab.size, this.targetSize), Error("too_small_vocab");
 		for (; this.vocab.size < this.targetSize && this.merges.length < this.targetSize;) {
 			let t = s(f);

package/dist/training/DatasetBuilder.d.ts

@@ -1,6 +1,7 @@
 import { Tensor } from '@tensorflow/tfjs-core';
 import { Conversation, ITokeniser } from '../tokeniser/type';
 import { Dataset } from '@tensorflow/tfjs-data';
+import { TokenStore } from './tasks/TokenStore';
 export declare function flattenTokens(textData: Conversation[][], tokenizer: ITokeniser): Uint16Array;
 export declare function flattenTokensWithMask(textData: Conversation[][], tokenizer: ITokeniser): {
     tokens: Uint16Array;
@@ -8,14 +9,28 @@ export declare function flattenTokensWithMask(textData: Conversation[][], tokeni
 };
 export declare function shuffle(array: Uint32Array): Uint32Array;
 export interface DatasetState {
+    shuffledShards: Uint32Array;
     shuffledIndexes: Uint32Array;
+    lastShardIndexes: Uint32Array;
+    currentShard: Uint16Array | null;
+    nextShard: Uint16Array | null;
+    currentMask: Uint8Array | null;
+    nextMask: Uint8Array | null;
+    shardIndex: number;
     step: number;
 }
+export declare function moveToNext(state: DatasetState, store: TokenStore, noShuffle?: boolean): Promise<void>;
+interface DatasetOptions {
+    batchSize: number;
+    noShuffle?: boolean;
+    ignoreIndex?: number;
+    shuffleFirst?: boolean;
+}
 export declare class DatasetBuilder {
     tokenizer: ITokeniser;
     blockSize: number;
     constructor(tokenizer: ITokeniser, blockSize?: number);
-    createTextDataset(flatTokens: Uint16Array[], batchSize?: number, indexes?: Uint32Array, mask?: Uint8Array[], ignoreIndex?: number): Promise<{
+    createTextDataset(store: TokenStore, options?: DatasetOptions): Promise<{
         dataset: Dataset<{
             xs: Tensor;
             ys: Tensor;
@@ -23,3 +38,4 @@ export declare class DatasetBuilder {
         state: DatasetState;
     }>;
 }
+export {};

package/dist/training/DatasetBuilder.js

@@ -1,2 +1,2 @@
-import { i as e, n as t, r as n, t as r } from "../DatasetBuilder-Ctb425Id.js";
-export { r as DatasetBuilder, t as flattenTokens, n as flattenTokensWithMask, e as shuffle };
+import { a as e, i as t, n, r, t as i } from "../DatasetBuilder-C0iJT29K.js";
+export { i as DatasetBuilder, n as flattenTokens, r as flattenTokensWithMask, t as moveToNext, e as shuffle };

package/dist/training/PreTrainer.js

@@ -1,4 +1,4 @@
-import { t as e } from "../DatasetBuilder-Ctb425Id.js";
+import { t as e } from "../DatasetBuilder-C0iJT29K.js";
 import t from "./BasicTrainer.js";
 //#region lib/training/PreTrainer.ts
 var n = {

package/dist/training/SFTTrainer.js

@@ -1,4 +1,4 @@
-import { t as e } from "../DatasetBuilder-Ctb425Id.js";
+import { t as e } from "../DatasetBuilder-C0iJT29K.js";
 import t from "./BasicTrainer.js";
 //#region lib/training/SFTTrainer.ts
 var n = {

package/dist/training/tasks/Task.d.ts

@@ -1,4 +1,5 @@
 import { Conversation, ITokeniser } from '../../../main';
+import { TokenStore } from './TokenStore';
 export declare abstract class Task {
     abstract get length(): number;
     abstract hasMoreConversations(): boolean;
@@ -13,8 +14,16 @@ export declare abstract class Task {
         mask: boolean[];
     } | null>;
 }
-export declare function tokensFromTasks(tasks: Task[], tokenizer: ITokeniser, cb?: (tokens: number) => void): Promise<Uint16Array[]>;
-export declare function tokensFromTasks(tasks: Task[], tokenizer: ITokeniser, cb?: (tokens: number) => void, masking?: boolean): Promise<{
-    tokens: Uint16Array[];
-    mask: Uint8Array[];
+interface TokensFromTasksOptions {
+    masking?: boolean;
+    maxCachedShards?: number;
+    noOPFS?: boolean;
+    shardSize?: number;
+    validationSplit?: number;
+    cb?: (tokens: number) => void;
+}
+export declare function tokensFromTasks(tasks: Task[], tokenizer: ITokeniser, options?: TokensFromTasksOptions): Promise<{
+    trainingTokens: TokenStore;
+    validationTokens?: TokenStore;
 }>;
+export {};

package/dist/training/tasks/Task.js

@@ -1,7 +1,8 @@
 import { yieldIfNeeded as e } from "../../utilities/yielder.js";
+import { createTokenStore as t, deleteTokenStore as n } from "./TokenStore.js";
 //#region lib/training/tasks/Task.ts
-var t = class {};
-async function n(e, t, n, r, i, a) {
+var r = class {};
+async function i(e, t, n, r, i, a) {
 	for (let o of e) {
 		let e = await o.nextTokens(n, a ? !0 : void 0);
 		if (e) {
@@ -24,19 +25,26 @@ async function n(e, t, n, r, i, a) {
 		}
 	}
 }
-async function r(t, r, i, a) {
-	let o = 1e4 * 1024, s = [new Uint16Array(o)], c = a ? [new Uint8Array(o)] : null, l = {
+async function a(r, a, o) {
+	await n("training-tokens");
+	let s = await t("training-tokens", a.id, a.datasetID ?? "", o);
+	await n("validation-tokens");
+	let c = o?.validationSplit && o.validationSplit > 0 ? await t("validation-tokens", a.id, a.datasetID ?? "", o) : void 0, l = [new Uint16Array(s.shardSize)], u = o?.masking ? [new Uint8Array(s.shardSize)] : null, d = {
 		offset: 0,
 		total: 0
-	}, u = performance.now();
-	for (; await n(t, s, r, l, o, c || void 0), !t.every((e) => !e.hasMoreConversations());) u = await e(u, i, l.total);
-	return s.length === 1 ? c ? {
-		tokens: [s[0].subarray(0, l.offset)],
-		mask: [c[0].subarray(0, l.offset)]
-	} : [s[0].subarray(0, l.offset)] : (s[s.length - 1] = s[s.length - 1].subarray(0, l.offset), c ? (c[c.length - 1] = c[c.length - 1].subarray(0, l.offset), {
-		tokens: s,
-		mask: c
-	}) : s);
+	}, f = o?.validationSplit && o.validationSplit > 0 ? [new Uint16Array(c.shardSize)] : void 0, p = o?.masking && f ? [new Uint8Array(c.shardSize)] : null, m = {
+		offset: 0,
+		total: 0
+	}, h = performance.now();
+	for (;;) {
+		let t = o?.validationSplit && o.validationSplit > 0 && Math.random() < o.validationSplit, n = t ? f : l, g = t ? m : d, _ = t ? p : u, v = t ? c : s;
+		if (await i(r, n, a, g, v.shardSize, _ || void 0), n.length > 1 && (v.appendShard(n[0], _ ? _[0] : void 0), n.shift(), _ && _.shift()), r.every((e) => !e.hasMoreConversations())) break;
+		h = await e(h, o?.cb, d.total);
+	}
+	return l.length === 1 && (l[0] = l[0].subarray(0, d.offset), await s.appendShard(l[0], u ? u[0].subarray(0, d.offset) : void 0)), f && f.length === 1 && (f[0] = f[0].subarray(0, m.offset), await c.appendShard(f[0], p ? p[0].subarray(0, m.offset) : void 0)), {
+		trainingTokens: s,
+		validationTokens: f ? c : void 0
+	};
 }
 //#endregion
-export { t as Task, r as tokensFromTasks };
+export { r as Task, a as tokensFromTasks };

package/dist/training/tasks/TokenStore.d.ts

@@ -0,0 +1,46 @@
+export declare class TokenStore {
+    readonly name: string;
+    readonly tokeniserId: string;
+    readonly datasetId: string;
+    private shards;
+    private masks?;
+    private shardCount;
+    private lastShardLength;
+    private _shardSize;
+    private dirHandle;
+    private opfsAvailable;
+    private lru;
+    private maxCachedShards;
+    constructor(tokeniserId: string, datasetId: string, name?: string, maxCachedShards?: number, shardSize?: number);
+    get shardSize(): number;
+    hasMask(): boolean;
+    getShardLength(index: number): number;
+    slice(start: number, end: number): Promise<Uint16Array>;
+    init(): Promise<void>;
+    private writeManifest;
+    private readManifest;
+    private shardFilename;
+    private maskFilename;
+    private writeShardToOPFS;
+    private writeMaskToOPFS;
+    private readShardFromOPFS;
+    private readMaskFromOPFS;
+    private touchLRU;
+    private evictIfNeeded;
+    getShard(index: number): Promise<Uint16Array>;
+    getMask(index: number): Promise<Uint8Array | undefined>;
+    getShardCount(): number;
+    getTokenCount(): number;
+    appendShard(shard: Uint16Array, mask?: Uint8Array): Promise<void>;
+    dispose(): Promise<void>;
+    clear(): Promise<void>;
+}
+export declare function getTokenStore(name: string, tokeniserId: string, datasetId: string): TokenStore | null;
+export declare function deleteTokenStore(name: string): Promise<void>;
+interface TokenStoreOptions {
+    maxCachedShards?: number;
+    shardSize?: number;
+    noOPFS?: boolean;
+}
+export declare function createTokenStore(name: string, tokeniserId: string, datasetId: string, options?: TokenStoreOptions): Promise<TokenStore>;
+export {};