@genai-fi/nanogpt 0.23.0 → 0.24.0

package/dist/tokeniser/CharTokeniser.js

@@ -1,7 +1,6 @@
-import { yieldIfNeeded as e } from "../utilities/yielder.js";
-import t, { SPECIALS as n } from "./BaseTokeniser.js";
+import e, { SPECIALS as t } from "./BaseTokeniser.js";
 //#region lib/tokeniser/CharTokeniser.ts
-var r = ["<eos>", "<unk>"], i = class extends t {
+var n = ["<eos>", "<unk>"], r = class extends e {
 	vocabSize = 0;
 	eosToken = 0;
 	bosToken = 0;
@@ -11,7 +10,7 @@ var r = ["<eos>", "<unk>"], i = class extends t {
 	_trained = !1;
 	constructor(e) {
 		if (super(), Array.isArray(e)) {
-			if (this.vocab = e, this.vocab.length > 0) this.vocabSize = this.vocab.length, n.forEach((e) => {
+			if (this.vocab = e, this.vocab.length > 0) this.vocabSize = this.vocab.length, t.forEach((e) => {
 				let t = this.vocab.indexOf(e);
 				t !== -1 && this.addSpecialToken(e, t);
 			}), this.eosToken = this.getSpecialTokenIndex("<eos>"), this.bosToken = this.getSpecialTokenIndex("<bos>") ?? this.eosToken, this.unkToken = this.getSpecialTokenIndex("") ?? -1, this.unkToken === -1 && (this.unkToken = this.vocab.indexOf("<unk>")), this.unkToken === -1 && (this.unkToken = this.vocab.indexOf("<pad>")), this.unkToken === -1 && (this.unkToken = this.vocab.indexOf("_")), this.unkToken === -1 && (this.unkToken = this.vocab.indexOf(" ")), this.unkToken === -1 && (this.unkToken = this.eosToken), this.vocab = this.vocab.map((e) => e === "<pad>" ? "" : e), this.vocab.forEach((e, t) => {
@@ -35,22 +34,21 @@ var r = ["<eos>", "<unk>"], i = class extends t {
 	destroy() {
 		this.cache.clear(), this.vocab = [];
 	}
-	async train(t, n, i) {
-		this.datasetID = i;
-		let a = /* @__PURE__ */ new Set(), o = performance.now();
-		for (let r of t) {
-			let t = r.cursor(), i = await t.next();
-			for (; i !== null;) i.forEach((e) => {
-				for (let t of e.content) a.add(t);
-			}), o = await e(o, n, 0), i = await t.next();
-		}
-		let s = Array.from(a), c = this.vocab.indexOf("", this.unkToken + 1), l = this.vocabSize - r.length;
-		if (c === -1) return this.generateID(), this.vocabSize;
-		if (this._trained = !0, s.length > l) throw Error("too_small_vocab");
-		let u = c;
-		if (u !== -1) {
+	async train(e, t, r) {
+		this.datasetID = r;
+		let i = /* @__PURE__ */ new Set();
+		for (let n of e) await n.begin((e) => {
+			e.forEach((e) => {
+				for (let t of e.content) i.add(t);
+			});
+		}, t ? () => t(i.size) : void 0);
+		let a = Array.from(i), o = this.vocab.indexOf("", this.unkToken + 1), s = this.vocabSize - n.length;
+		if (o === -1) return this.generateID(), this.vocabSize;
+		if (this._trained = !0, a.length > s) throw Error("too_small_vocab");
+		let c = o;
+		if (c !== -1) {
 			let e = new Set(this.vocab);
-			for (let t of s) if (!e.has(t) && (this.vocab[u] = t, e.add(t), u = this.vocab.indexOf("", u + 1), u === -1)) break;
+			for (let t of a) if (!e.has(t) && (this.vocab[c] = t, e.add(t), c = this.vocab.indexOf("", c + 1), c === -1)) break;
 		}
 		return this.cache.clear(), this.vocab.forEach((e, t) => {
 			this.cache.set(e, t);
@@ -85,4 +83,4 @@ var r = ["<eos>", "<unk>"], i = class extends t {
 	}
 };
 //#endregion
-export { i as default };
+export { r as default };

package/dist/tokeniser/bpe.js

@@ -1,5 +1,5 @@
-import { yieldIfNeeded as e } from "../utilities/yielder.js";
-import t, { SPECIALS as n } from "./BaseTokeniser.js";
+import e, { SPECIALS as t } from "./BaseTokeniser.js";
+import { yieldIfNeeded as n } from "../utilities/yielder.js";
 import r from "../utilities/tokenParse.js";
 //#region lib/tokeniser/bpe.ts
 function i(e, t) {
@@ -58,16 +58,16 @@ function l(e, t) {
 		e.tokens[n] = i;
 	}), e.pairs.delete(i(t.a, t.b));
 }
-var u = class extends t {
+var u = class extends e {
 	targetSize;
 	vocab = /* @__PURE__ */ new Set();
 	vocabIndex = /* @__PURE__ */ new Map();
 	merges = [];
 	pretokenMap = /* @__PURE__ */ new Map();
-	constructor(e, t) {
+	constructor(e, n) {
 		super(), Array.isArray(e) ? (e.forEach((e, t) => {
 			this.vocab.add(e), this.vocabIndex.set(e, t);
-		}), t && (this.merges = t), this.targetSize = e.length, n.forEach((t) => {
+		}), n && (this.merges = n), this.targetSize = e.length, t.forEach((t) => {
 			let n = e.indexOf(t);
 			n !== -1 && this.addSpecialToken(t, n);
 		})) : (this.addSpecialTokens(), this.targetSize = e);
@@ -84,7 +84,7 @@ var u = class extends t {
 		this.vocab.clear(), this.vocabIndex.clear(), this.merges = [], this.pretokenMap.clear();
 	}
 	get trained() {
-		return this.vocab.size > n.length && this.vocab.size <= this.targetSize;
+		return this.vocab.size > t.length && this.vocab.size <= this.targetSize;
 	}
 	get vocabSize() {
 		return this.vocab.size;
@@ -98,26 +98,22 @@ var u = class extends t {
 	get unkToken() {
 		return this.vocabIndex.get("") ?? 1;
 	}
-	async train(t = [], n, i) {
+	async train(e = [], t, i) {
 		this.datasetID = i;
-		let o = performance.now(), c = /* @__PURE__ */ new Set();
+		let o = /* @__PURE__ */ new Set();
 		this.vocab = /* @__PURE__ */ new Set(), this.pretokenMap.clear(), this.merges = [], this.addSpecialTokens();
-		for (let i of t) {
-			let t = i.cursor(), a = await t.next();
-			for (; a !== null;) {
-				for (let e of a) {
-					let t = r(e.content);
-					for (let e of t) c.has(e) || (c.add(e), Array.from(e).forEach((e) => this.vocab.add(e)));
-				}
-				o = await e(o, n, this.vocab.size), a = await t.next();
+		for (let n of e) await n.begin((e) => {
+			for (let t of e) {
+				let e = r(t.content);
+				for (let t of e) o.has(t) || (o.add(t), Array.from(t).forEach((e) => this.vocab.add(e)));
 			}
-		}
-		let u = Array.from(c), d = u.map((e) => Array.from(e)), f = a(d);
-		if (o = await e(o, n, this.vocab.size), this.vocab.size >= this.targetSize) throw console.warn("Initial vocab size is greater than or equal to target size. No merges will be performed.", this.vocab.size, this.targetSize), Error("too_small_vocab");
+		}, t ? () => t(this.vocab.size) : void 0);
+		let c = performance.now(), u = Array.from(o), d = u.map((e) => Array.from(e)), f = a(d);
+		if (c = await n(c, t, this.vocab.size), this.vocab.size >= this.targetSize) throw console.warn("Initial vocab size is greater than or equal to target size. No merges will be performed.", this.vocab.size, this.targetSize), Error("too_small_vocab");
 		for (; this.vocab.size < this.targetSize && this.merges.length < this.targetSize;) {
-			let t = s(f);
-			if (!t) break;
-			this.merges.push([t.a, t.b]), this.vocab.add(t.a + t.b), l(f, t), o = await e(o, n, this.vocab.size);
+			let e = s(f);
+			if (!e) break;
+			this.merges.push([e.a, e.b]), this.vocab.add(e.a + e.b), l(f, e), c = await n(c, t, this.vocab.size);
 		}
 		u.forEach((e, t) => {
 			let n = d[t];

package/dist/training/DatasetBuilder.d.ts

@@ -7,11 +7,11 @@ export declare function flattenTokensWithMask(textData: Conversation[][], tokeni
     tokens: Uint16Array;
     mask: Uint8Array;
 };
-export declare function shuffle(array: Uint32Array): Uint32Array;
+export declare function shuffle(array: Uint32Array | Uint16Array): Uint32Array | Uint16Array;
 export interface DatasetState {
-    shuffledShards: Uint32Array;
-    shuffledIndexes: Uint32Array;
-    lastShardIndexes: Uint32Array;
+    shuffledShards: Uint16Array;
+    shuffledIndexes: Uint16Array;
+    lastShardIndexes: Uint16Array;
     currentShard: Uint16Array | null;
     nextShard: Uint16Array | null;
     currentMask: Uint8Array | null;

package/dist/training/DatasetBuilder.js

@@ -1,2 +1,2 @@
-import { a as e, i as t, n, r, t as i } from "../DatasetBuilder-C0iJT29K.js";
+import { a as e, i as t, n, r, t as i } from "../DatasetBuilder-DU1G1OKX.js";
 export { i as DatasetBuilder, n as flattenTokens, r as flattenTokensWithMask, t as moveToNext, e as shuffle };

package/dist/training/PreTrainer.js

@@ -1,4 +1,4 @@
-import { t as e } from "../DatasetBuilder-C0iJT29K.js";
+import { t as e } from "../DatasetBuilder-DU1G1OKX.js";
 import t from "./BasicTrainer.js";
 //#region lib/training/PreTrainer.ts
 var n = {

package/dist/training/SFTTrainer.js

@@ -1,4 +1,4 @@
-import { t as e } from "../DatasetBuilder-C0iJT29K.js";
+import { t as e } from "../DatasetBuilder-DU1G1OKX.js";
 import t from "./BasicTrainer.js";
 //#region lib/training/SFTTrainer.ts
 var n = {

package/dist/training/tasks/TokenStore.d.ts

@@ -31,7 +31,8 @@ export declare class TokenStore {
     getMask(index: number): Promise<Uint8Array | undefined>;
     getShardCount(): number;
     getTokenCount(): number;
-    appendShard(shard: Uint16Array, mask?: Uint8Array): Promise<void>;
+    finish(): Promise<void>;
+    appendShard(shard: Uint16Array, mask?: Uint8Array): void;
     dispose(): Promise<void>;
     clear(): Promise<void>;
 }

package/dist/training/tasks/TokenStore.js

@@ -7,12 +7,12 @@ var e = "llm-tokenstore", t = class {
 	masks;
 	shardCount = 0;
 	lastShardLength = -1;
-	_shardSize = 4e3 * 1024;
+	_shardSize = 8e3 * 1024;
 	dirHandle = null;
 	opfsAvailable = !1;
 	lru = /* @__PURE__ */ new Map();
 	maxCachedShards;
-	constructor(t, n, r, i = 8, a = 4e3 * 1024) {
+	constructor(t, n, r, i = 2, a = 8e3 * 1024) {
 		this.tokeniserId = t, this.datasetId = n, this.maxCachedShards = Math.max(1, Math.trunc(i)), this._shardSize = Math.max(1, Math.trunc(a)), this.name = r ?? e;
 	}
 	get shardSize() {
@@ -148,12 +148,15 @@ var e = "llm-tokenstore", t = class {
 	getTokenCount() {
 		return this.shardCount === 0 ? 0 : (this.shardCount - 1) * this._shardSize + this.lastShardLength;
 	}
-	async appendShard(e, t) {
+	async finish() {
+		await this.writeManifest();
+	}
+	appendShard(e, t) {
 		if (this.lastShardLength >= 0 && this.lastShardLength < this._shardSize) throw Error("Previous shard was not full");
 		let n = this.shards.length;
 		this.shards.push(e), !this.masks && t && (this.masks = Array(n).fill(null)), this.masks && this.masks.push(t ?? null), this.shardCount = this.shards.length, this.lastShardLength = e.length, this.touchLRU(n);
 		try {
-			this.opfsAvailable && this.dirHandle && (await this.writeShardToOPFS(n, e), t && await this.writeMaskToOPFS(n, t), await this.writeManifest());
+			this.opfsAvailable && this.dirHandle && (this.writeShardToOPFS(n, e), t && this.writeMaskToOPFS(n, t));
 		} catch (e) {
 			console.error(e), this.maxCachedShards = 2 ** 53 - 1;
 		}
@@ -208,7 +211,7 @@ async function a(e, r, i, a) {
 	let o = n.get(e);
 	if (o && o.tokeniserId === r && o.datasetId === i) return o;
 	o && (await o.dispose(), n.delete(e));
-	let s = new t(r, i, e, a?.maxCachedShards ?? 8, a?.shardSize ?? 4e3 * 1024);
+	let s = new t(r, i, e, a?.maxCachedShards, a?.shardSize);
 	return a?.noOPFS || await s.init(), n.set(e, s), s;
 }
 //#endregion

package/dist/training/tasks/tokenStream.d.ts

@@ -0,0 +1,16 @@
+import { ConversationStream, ITokeniser } from '../../../main';
+import { TokenStore } from './TokenStore';
+interface TokensFromTasksOptions {
+    masking?: boolean;
+    maxCachedShards?: number;
+    noOPFS?: boolean;
+    shardSize?: number;
+    validationSplit?: number;
+    validationSeed?: string | number;
+    cb?: (tokens: number) => void;
+}
+export declare function tokensFromStreams(tasks: ConversationStream[], tokenizer: ITokeniser, options?: TokensFromTasksOptions): Promise<{
+    trainingTokens: TokenStore;
+    validationTokens?: TokenStore;
+}>;
+export {};

package/dist/training/tasks/tokenStream.js

@@ -0,0 +1,46 @@
+import { createTokenStore as e, deleteTokenStore as t } from "./TokenStore.js";
+import { seededRng as n } from "../../utilities/random.js";
+//#region lib/training/tasks/tokenStream.ts
+function r(e, t, n, r, i, a) {
+	let o = n.encodeConversation(e, !1, !!a);
+	if (o) {
+		let e = Array.isArray(o) ? o : o.tokens;
+		r.total += e.length;
+		let n = t[t.length - 1], s = a ? a[a.length - 1] : null;
+		if (r.offset + e.length > n.length) {
+			let c = n.length - r.offset;
+			n.set(e.slice(0, c), r.offset);
+			let l = e.length - c;
+			if (l > i) throw Error(`Estimated tokens (${i}) is too small for the next batch of tokens (${l}).`);
+			let u = new Uint16Array(i);
+			if (u.set(e.slice(c), 0), t.push(u), a && s && !Array.isArray(o)) {
+				s.set(o.mask.slice(0, c).map((e) => +!!e), r.offset);
+				let e = new Uint8Array(u.length);
+				e.set(o.mask.slice(c).map((e) => +!!e), 0), a.push(e);
+			}
+			r.offset = e.length - c;
+		} else n.set(e, r.offset), s && !Array.isArray(o) && s.set(o.mask.map((e) => +!!e), r.offset), r.offset += e.length;
+	}
+}
+async function i(i, a, o) {
+	await t("training-tokens");
+	let s = await e("training-tokens", a.id, a.datasetID ?? "", o);
+	await t("validation-tokens");
+	let c = o?.validationSplit && o.validationSplit > 0 ? await e("validation-tokens", a.id, a.datasetID ?? "", o) : void 0, l = [new Uint16Array(s.shardSize)], u = o?.masking ? [new Uint8Array(s.shardSize)] : null, d = {
+		offset: 0,
+		total: 0
+	}, f = o?.validationSplit && o.validationSplit > 0 ? [new Uint16Array(c.shardSize)] : void 0, p = o?.masking && f ? [new Uint8Array(c.shardSize)] : null, m = {
+		offset: 0,
+		total: 0
+	}, h = 0, g = o?.cb, _ = o?.validationSeed === void 0 ? Math.random : n(o.validationSeed);
+	for (; h < i.length;) await i[h++].begin((e) => {
+		let t = o?.validationSplit && o.validationSplit > 0 && _() < o.validationSplit, n = t ? f : l, i = t ? m : d, h = t ? p : u, g = t ? c : s;
+		r(e, n, a, i, g.shardSize, h || void 0), n.length > 1 && (g.appendShard(n[0], h ? h[0] : void 0), n.shift(), h && h.shift());
+	}, g ? () => g(d.total) : void 0);
+	return l.length === 1 && (l[0] = l[0].subarray(0, d.offset), s.appendShard(l[0], u ? u[0].subarray(0, d.offset) : void 0)), f && f.length === 1 && (f[0] = f[0].subarray(0, m.offset), c.appendShard(f[0], p ? p[0].subarray(0, m.offset) : void 0)), await s.finish(), c && await c.finish(), {
+		trainingTokens: s,
+		validationTokens: f ? c : void 0
+	};
+}
+//#endregion
+export { i as tokensFromStreams };

package/dist/training/validation.js

@@ -1,8 +1,10 @@
 import { TokenStore as e, createTokenStore as t } from "./tasks/TokenStore.js";
 //#region lib/training/validation.ts
 async function n(e, n) {
-	let r = await t("training-tokens", n.id, n.datasetID ?? ""), i = e.map((e) => r.appendShard(e));
-	return await Promise.all(i), r;
+	let r = await t("training-tokens", n.id, n.datasetID ?? "");
+	return e.forEach((e) => {
+		r.appendShard(e);
+	}), await r.finish(), r;
 }
 async function r(t, r, i, a, o) {
 	let s = t instanceof e ? t : await n(t, i), c = r instanceof e ? r : await n(r, i), l = s.getTokenCount(), { dataset: u, state: d } = await a.createTextDataset(s, { batchSize: o }), { dataset: f, state: p } = await a.createTextDataset(c, {

package/dist/utilities/random.d.ts

@@ -0,0 +1 @@
+export declare function seededRng(seed: string | number): () => number;

package/dist/utilities/random.js

@@ -0,0 +1,19 @@
+//#region lib/utilities/random.ts
+function e(e) {
+	let t = 1779033703 ^ e.length;
+	for (let n = 0; n < e.length; n++) t = Math.imul(t ^ e.charCodeAt(n), 3432918353), t = t << 13 | t >>> 19;
+	return () => (t = Math.imul(t ^ t >>> 16, 2246822507), t = Math.imul(t ^ t >>> 13, 3266489909), (t ^= t >>> 16) >>> 0);
+}
+function t(e) {
+	let t = e >>> 0;
+	return () => {
+		t += 1831565813;
+		let e = Math.imul(t ^ t >>> 15, t | 1);
+		return e ^= e + Math.imul(e ^ e >>> 7, e | 61), ((e ^ e >>> 14) >>> 0) / 4294967296;
+	};
+}
+function n(n) {
+	return t(typeof n == "number" ? n >>> 0 : e(n)());
+}
+//#endregion
+export { n as seededRng };

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@genai-fi/nanogpt",
-    "version": "0.23.0",
+    "version": "0.24.0",
     "type": "module",
     "main": "dist/main.js",
     "types": "dist/main.d.ts",

package/dist/training/tasks/ConversationTask.d.ts

@@ -1,17 +0,0 @@
-import { Conversation, ConversationStream, ITokeniser } from '../../../main';
-import { Task } from './Task';
-export default class ConversationTask extends Task {
-    private streams;
-    private streamIndex;
-    private currentCursor;
-    get length(): number;
-    constructor(conversations: ConversationStream[]);
-    hasMoreConversations(): boolean;
-    nextConversation(): Promise<Conversation[] | null>;
-    nextTokens(tokeniser: ITokeniser): Promise<number[] | null>;
-    nextTokens(tokeniser: ITokeniser, masking: boolean): Promise<{
-        tokens: number[];
-        mask: boolean[];
-    } | null>;
-    estimateTokens(tokeniser: ITokeniser): Promise<number>;
-}

package/dist/training/tasks/ConversationTask.js

@@ -1,29 +0,0 @@
-import { Task as e } from "./Task.js";
-//#region lib/training/tasks/ConversationTask.ts
-var t = class extends e {
-	streams;
-	streamIndex = 0;
-	currentCursor = null;
-	get length() {
-		return this.streams.length;
-	}
-	constructor(e) {
-		super(), this.streams = e;
-	}
-	hasMoreConversations() {
-		return this.streamIndex < this.streams.length;
-	}
-	async nextConversation() {
-		return this.streamIndex < this.streams.length ? (this.currentCursor ||= this.streams[this.streamIndex].cursor(), await this.currentCursor.next() || (this.streamIndex++, this.currentCursor = null, this.nextConversation())) : null;
-	}
-	async nextTokens(e, t) {
-		let n = await this.nextConversation();
-		return n ? e.encodeConversation(n, !1, t) : null;
-	}
-	async estimateTokens(e) {
-		let t = await this.streams[0].cursor().next();
-		return t ? e.encodeConversation(t).length * this.length : 0;
-	}
-};
-//#endregion
-export { t as default };

package/dist/training/tasks/PretrainingTask.d.ts

@@ -1,17 +0,0 @@
-import { Conversation, ITokeniser } from '../../../main';
-import { Task } from './Task';
-export default class PretrainingTask extends Task {
-    private rawText;
-    private index;
-    get length(): number;
-    constructor(texts: string[]);
-    hasMoreConversations(): boolean;
-    nextConversation(): Promise<Conversation[] | null>;
-    nextTokens(tokeniser: ITokeniser): Promise<number[] | null>;
-    nextTokens(tokeniser: ITokeniser, masking: boolean): Promise<{
-        tokens: number[];
-        mask: boolean[];
-    } | null>;
-    shuffle(): void;
-    estimateTokens(tokeniser: ITokeniser): Promise<number>;
-}

package/dist/training/tasks/PretrainingTask.js

@@ -1,42 +0,0 @@
-import { Task as e } from "./Task.js";
-//#region lib/training/tasks/PretrainingTask.ts
-var t = class extends e {
-	rawText;
-	index = 0;
-	get length() {
-		return this.rawText.length;
-	}
-	constructor(e) {
-		super(), this.rawText = e;
-	}
-	hasMoreConversations() {
-		return this.index < this.rawText.length;
-	}
-	async nextConversation() {
-		if (this.index >= this.rawText.length) return null;
-		let e = {
-			role: "assistant",
-			content: this.rawText[this.index]
-		};
-		return this.index++, [e];
-	}
-	async nextTokens(e, t) {
-		if (this.index >= this.rawText.length) return null;
-		let n = e.encodeSequence(this.rawText[this.index]);
-		return this.index++, t ? {
-			tokens: n,
-			mask: Array(n.length).fill(!0)
-		} : n;
-	}
-	shuffle() {
-		this.index = 0;
-	}
-	async estimateTokens(e) {
-		return e.encodeConversation([{
-			role: "assistant",
-			content: this.rawText[0]
-		}]).length * this.length;
-	}
-};
-//#endregion
-export { t as default };

package/dist/training/tasks/StartSentenceTask.d.ts

@@ -1,18 +0,0 @@
-import { Conversation, ITokeniser } from '../../../main';
-import { Task } from './Task';
-export default class StartSentenceTask extends Task {
-    private rawText;
-    private index;
-    get length(): number;
-    constructor(texts: string[]);
-    hasMoreConversations(): boolean;
-    nextConversation(): Promise<Conversation[] | null>;
-    nextTokens(tokeniser: ITokeniser): Promise<number[] | null>;
-    nextTokens(tokeniser: ITokeniser, masking: boolean): Promise<{
-        tokens: number[];
-        mask: boolean[];
-    } | null>;
-    shuffle(): void;
-    private conversationFromString;
-    estimateTokens(tokeniser: ITokeniser): Promise<number>;
-}

package/dist/training/tasks/StartSentenceTask.js

@@ -1,45 +0,0 @@
-import { Task as e } from "./Task.js";
-//#region lib/training/tasks/StartSentenceTask.ts
-var t = class extends e {
-	rawText;
-	index = 0;
-	get length() {
-		return this.rawText.length;
-	}
-	constructor(e) {
-		super(), this.rawText = e;
-	}
-	hasMoreConversations() {
-		return this.index < this.rawText.length;
-	}
-	async nextConversation() {
-		if (this.index >= this.rawText.length) return null;
-		let e = this.rawText[this.index];
-		return this.index++, this.conversationFromString(e);
-	}
-	async nextTokens(e, t) {
-		let n = await this.nextConversation();
-		return n ? e.encodeConversation(n, !1, t) : null;
-	}
-	shuffle() {
-		this.index = 0;
-	}
-	conversationFromString(e) {
-		let t = e.indexOf(".");
-		return t === -1 ? [{
-			role: "assistant",
-			content: this.rawText[this.index]
-		}] : [{
-			role: "user",
-			content: e.slice(0, t + 1).trim()
-		}, {
-			role: "assistant",
-			content: e.slice(t + 1).trim()
-		}];
-	}
-	async estimateTokens(e) {
-		return (await e.encodeConversation(this.conversationFromString(this.rawText[0]))).length * this.length;
-	}
-};
-//#endregion
-export { t as default };

package/dist/training/tasks/Task.d.ts

@@ -1,29 +0,0 @@
-import { Conversation, ITokeniser } from '../../../main';
-import { TokenStore } from './TokenStore';
-export declare abstract class Task {
-    abstract get length(): number;
-    abstract hasMoreConversations(): boolean;
-    abstract nextConversation(): Promise<Conversation[] | null>;
-    abstract nextTokens(tokeniser: ITokeniser): Promise<number[] | null>;
-    abstract nextTokens(tokeniser: ITokeniser, masking: boolean): Promise<{
-        tokens: number[];
-        mask: boolean[];
-    } | null>;
-    abstract nextTokens(tokeniser: ITokeniser, masking?: boolean): Promise<number[] | {
-        tokens: number[];
-        mask: boolean[];
-    } | null>;
-}
-interface TokensFromTasksOptions {
-    masking?: boolean;
-    maxCachedShards?: number;
-    noOPFS?: boolean;
-    shardSize?: number;
-    validationSplit?: number;
-    cb?: (tokens: number) => void;
-}
-export declare function tokensFromTasks(tasks: Task[], tokenizer: ITokeniser, options?: TokensFromTasksOptions): Promise<{
-    trainingTokens: TokenStore;
-    validationTokens?: TokenStore;
-}>;
-export {};

package/dist/training/tasks/Task.js

@@ -1,50 +0,0 @@
-import { yieldIfNeeded as e } from "../../utilities/yielder.js";
-import { createTokenStore as t, deleteTokenStore as n } from "./TokenStore.js";
-//#region lib/training/tasks/Task.ts
-var r = class {};
-async function i(e, t, n, r, i, a) {
-	for (let o of e) {
-		let e = await o.nextTokens(n, a ? !0 : void 0);
-		if (e) {
-			let n = Array.isArray(e) ? e : e.tokens;
-			r.total += n.length;
-			let o = t[t.length - 1], s = a ? a[a.length - 1] : null;
-			if (r.offset + n.length > o.length) {
-				let c = o.length - r.offset;
-				o.set(n.slice(0, c), r.offset);
-				let l = n.length - c;
-				if (l > i) throw Error(`Estimated tokens (${i}) is too small for the next batch of tokens (${l}).`);
-				let u = new Uint16Array(i);
-				if (u.set(n.slice(c), 0), t.push(u), a && s && !Array.isArray(e)) {
-					s.set(e.mask.slice(0, c).map((e) => +!!e), r.offset);
-					let t = new Uint8Array(u.length);
-					t.set(e.mask.slice(c).map((e) => +!!e), 0), a.push(t);
-				}
-				r.offset = n.length - c;
-			} else o.set(n, r.offset), s && !Array.isArray(e) && s.set(e.mask.map((e) => +!!e), r.offset), r.offset += n.length;
-		}
-	}
-}
-async function a(r, a, o) {
-	await n("training-tokens");
-	let s = await t("training-tokens", a.id, a.datasetID ?? "", o);
-	await n("validation-tokens");
-	let c = o?.validationSplit && o.validationSplit > 0 ? await t("validation-tokens", a.id, a.datasetID ?? "", o) : void 0, l = [new Uint16Array(s.shardSize)], u = o?.masking ? [new Uint8Array(s.shardSize)] : null, d = {
-		offset: 0,
-		total: 0
-	}, f = o?.validationSplit && o.validationSplit > 0 ? [new Uint16Array(c.shardSize)] : void 0, p = o?.masking && f ? [new Uint8Array(c.shardSize)] : null, m = {
-		offset: 0,
-		total: 0
-	}, h = performance.now();
-	for (;;) {
-		let t = o?.validationSplit && o.validationSplit > 0 && Math.random() < o.validationSplit, n = t ? f : l, g = t ? m : d, _ = t ? p : u, v = t ? c : s;
-		if (await i(r, n, a, g, v.shardSize, _ || void 0), n.length > 1 && (v.appendShard(n[0], _ ? _[0] : void 0), n.shift(), _ && _.shift()), r.every((e) => !e.hasMoreConversations())) break;
-		h = await e(h, o?.cb, d.total);
-	}
-	return l.length === 1 && (l[0] = l[0].subarray(0, d.offset), await s.appendShard(l[0], u ? u[0].subarray(0, d.offset) : void 0)), f && f.length === 1 && (f[0] = f[0].subarray(0, m.offset), await c.appendShard(f[0], p ? p[0].subarray(0, m.offset) : void 0)), {
-		trainingTokens: s,
-		validationTokens: f ? c : void 0
-	};
-}
-//#endregion
-export { r as Task, a as tokensFromTasks };

package/dist/training/tasks/splitter.d.ts

@@ -1,5 +0,0 @@
-import { Task } from './Task';
-export default function splitValidation(tasks: Task[], validationSplit: number): Promise<{
-    training: Task;
-    validation: Task;
-}>;

package/dist/training/tasks/splitter.js

@@ -1,18 +0,0 @@
-import { n as e } from "../../stream-BjdpSNqB.js";
-import t from "./ConversationTask.js";
-//#region lib/training/tasks/splitter.ts
-async function n(n, r) {
-	if (r <= 0 || r >= 1) throw Error("validationSplit must be between 0 and 1");
-	let i = [], a = [];
-	for (let e of n) for (; e.hasMoreConversations();) {
-		let t = await e.nextConversation();
-		if (!t) break;
-		Math.random() < r ? a.push(t) : i.push(t);
-	}
-	return {
-		training: new t([new e(i)]),
-		validation: new t([new e(a)])
-	};
-}
-//#endregion
-export { n as default };