@genai-fi/nanogpt 0.22.0 → 0.24.0

package/dist/training/validation.js

@@ -1,2 +1,23 @@
-import { o as e } from "../main-Bgc7_9kb.js";
-export { e as createTrainValidationSplit };
+import { TokenStore as e, createTokenStore as t } from "./tasks/TokenStore.js";
+//#region lib/training/validation.ts
+async function n(e, n) {
+	let r = await t("training-tokens", n.id, n.datasetID ?? "");
+	return e.forEach((e) => {
+		r.appendShard(e);
+	}), await r.finish(), r;
+}
+async function r(t, r, i, a, o) {
+	let s = t instanceof e ? t : await n(t, i), c = r instanceof e ? r : await n(r, i), l = s.getTokenCount(), { dataset: u, state: d } = await a.createTextDataset(s, { batchSize: o }), { dataset: f, state: p } = await a.createTextDataset(c, {
+		batchSize: o,
+		shuffleFirst: !0
+	});
+	return {
+		trainDataset: u,
+		validationDataset: f,
+		size: l,
+		validationState: p,
+		trainState: d
+	};
+}
+//#endregion
+export { r as createTrainValidationDatasets, n as storeFromArray };

package/dist/utilities/random.d.ts

@@ -0,0 +1 @@
+export declare function seededRng(seed: string | number): () => number;

package/dist/utilities/random.js

@@ -0,0 +1,19 @@
+//#region lib/utilities/random.ts
+function e(e) {
+	let t = 1779033703 ^ e.length;
+	for (let n = 0; n < e.length; n++) t = Math.imul(t ^ e.charCodeAt(n), 3432918353), t = t << 13 | t >>> 19;
+	return () => (t = Math.imul(t ^ t >>> 16, 2246822507), t = Math.imul(t ^ t >>> 13, 3266489909), (t ^= t >>> 16) >>> 0);
+}
+function t(e) {
+	let t = e >>> 0;
+	return () => {
+		t += 1831565813;
+		let e = Math.imul(t ^ t >>> 15, t | 1);
+		return e ^= e + Math.imul(e ^ e >>> 7, e | 61), ((e ^ e >>> 14) >>> 0) / 4294967296;
+	};
+}
+function n(n) {
+	return t(typeof n == "number" ? n >>> 0 : e(n)());
+}
+//#endregion
+export { n as seededRng };

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@genai-fi/nanogpt",
-    "version": "0.22.0",
+    "version": "0.24.0",
     "type": "module",
     "main": "dist/main.js",
     "types": "dist/main.d.ts",

package/dist/training/tasks/ConversationTask.d.ts

@@ -1,17 +0,0 @@
-import { Conversation, ConversationStream, ITokeniser } from '../../../main';
-import { Task } from './Task';
-export default class ConversationTask extends Task {
-    private streams;
-    private streamIndex;
-    private currentCursor;
-    get length(): number;
-    constructor(conversations: ConversationStream[]);
-    hasMoreConversations(): boolean;
-    nextConversation(): Promise<Conversation[] | null>;
-    nextTokens(tokeniser: ITokeniser): Promise<number[] | null>;
-    nextTokens(tokeniser: ITokeniser, masking: boolean): Promise<{
-        tokens: number[];
-        mask: boolean[];
-    } | null>;
-    estimateTokens(tokeniser: ITokeniser): Promise<number>;
-}

package/dist/training/tasks/ConversationTask.js

@@ -1,29 +0,0 @@
-import { Task as e } from "./Task.js";
-//#region lib/training/tasks/ConversationTask.ts
-var t = class extends e {
-	streams;
-	streamIndex = 0;
-	currentCursor = null;
-	get length() {
-		return this.streams.length;
-	}
-	constructor(e) {
-		super(), this.streams = e;
-	}
-	hasMoreConversations() {
-		return this.streamIndex < this.streams.length;
-	}
-	async nextConversation() {
-		return this.streamIndex < this.streams.length ? (this.currentCursor ||= this.streams[this.streamIndex].cursor(), await this.currentCursor.next() || (this.streamIndex++, this.currentCursor = null, this.nextConversation())) : null;
-	}
-	async nextTokens(e, t) {
-		let n = await this.nextConversation();
-		return n ? e.encodeConversation(n, !1, t) : null;
-	}
-	async estimateTokens(e) {
-		let t = await this.streams[0].cursor().next();
-		return t ? e.encodeConversation(t).length * this.length : 0;
-	}
-};
-//#endregion
-export { t as default };

package/dist/training/tasks/PretrainingTask.d.ts

@@ -1,17 +0,0 @@
-import { Conversation, ITokeniser } from '../../../main';
-import { Task } from './Task';
-export default class PretrainingTask extends Task {
-    private rawText;
-    private index;
-    get length(): number;
-    constructor(texts: string[]);
-    hasMoreConversations(): boolean;
-    nextConversation(): Promise<Conversation[] | null>;
-    nextTokens(tokeniser: ITokeniser): Promise<number[] | null>;
-    nextTokens(tokeniser: ITokeniser, masking: boolean): Promise<{
-        tokens: number[];
-        mask: boolean[];
-    } | null>;
-    shuffle(): void;
-    estimateTokens(tokeniser: ITokeniser): Promise<number>;
-}

package/dist/training/tasks/PretrainingTask.js

@@ -1,42 +0,0 @@
-import { Task as e } from "./Task.js";
-//#region lib/training/tasks/PretrainingTask.ts
-var t = class extends e {
-	rawText;
-	index = 0;
-	get length() {
-		return this.rawText.length;
-	}
-	constructor(e) {
-		super(), this.rawText = e;
-	}
-	hasMoreConversations() {
-		return this.index < this.rawText.length;
-	}
-	async nextConversation() {
-		if (this.index >= this.rawText.length) return null;
-		let e = {
-			role: "assistant",
-			content: this.rawText[this.index]
-		};
-		return this.index++, [e];
-	}
-	async nextTokens(e, t) {
-		if (this.index >= this.rawText.length) return null;
-		let n = e.encodeSequence(this.rawText[this.index]);
-		return this.index++, t ? {
-			tokens: n,
-			mask: Array(n.length).fill(!0)
-		} : n;
-	}
-	shuffle() {
-		this.index = 0;
-	}
-	async estimateTokens(e) {
-		return e.encodeConversation([{
-			role: "assistant",
-			content: this.rawText[0]
-		}]).length * this.length;
-	}
-};
-//#endregion
-export { t as default };

package/dist/training/tasks/StartSentenceTask.d.ts

@@ -1,18 +0,0 @@
-import { Conversation, ITokeniser } from '../../../main';
-import { Task } from './Task';
-export default class StartSentenceTask extends Task {
-    private rawText;
-    private index;
-    get length(): number;
-    constructor(texts: string[]);
-    hasMoreConversations(): boolean;
-    nextConversation(): Promise<Conversation[] | null>;
-    nextTokens(tokeniser: ITokeniser): Promise<number[] | null>;
-    nextTokens(tokeniser: ITokeniser, masking: boolean): Promise<{
-        tokens: number[];
-        mask: boolean[];
-    } | null>;
-    shuffle(): void;
-    private conversationFromString;
-    estimateTokens(tokeniser: ITokeniser): Promise<number>;
-}

package/dist/training/tasks/StartSentenceTask.js

@@ -1,45 +0,0 @@
-import { Task as e } from "./Task.js";
-//#region lib/training/tasks/StartSentenceTask.ts
-var t = class extends e {
-	rawText;
-	index = 0;
-	get length() {
-		return this.rawText.length;
-	}
-	constructor(e) {
-		super(), this.rawText = e;
-	}
-	hasMoreConversations() {
-		return this.index < this.rawText.length;
-	}
-	async nextConversation() {
-		if (this.index >= this.rawText.length) return null;
-		let e = this.rawText[this.index];
-		return this.index++, this.conversationFromString(e);
-	}
-	async nextTokens(e, t) {
-		let n = await this.nextConversation();
-		return n ? e.encodeConversation(n, !1, t) : null;
-	}
-	shuffle() {
-		this.index = 0;
-	}
-	conversationFromString(e) {
-		let t = e.indexOf(".");
-		return t === -1 ? [{
-			role: "assistant",
-			content: this.rawText[this.index]
-		}] : [{
-			role: "user",
-			content: e.slice(0, t + 1).trim()
-		}, {
-			role: "assistant",
-			content: e.slice(t + 1).trim()
-		}];
-	}
-	async estimateTokens(e) {
-		return (await e.encodeConversation(this.conversationFromString(this.rawText[0]))).length * this.length;
-	}
-};
-//#endregion
-export { t as default };

package/dist/training/tasks/Task.d.ts

@@ -1,20 +0,0 @@
-import { Conversation, ITokeniser } from '../../../main';
-export declare abstract class Task {
-    abstract get length(): number;
-    abstract hasMoreConversations(): boolean;
-    abstract nextConversation(): Promise<Conversation[] | null>;
-    abstract nextTokens(tokeniser: ITokeniser): Promise<number[] | null>;
-    abstract nextTokens(tokeniser: ITokeniser, masking: boolean): Promise<{
-        tokens: number[];
-        mask: boolean[];
-    } | null>;
-    abstract nextTokens(tokeniser: ITokeniser, masking?: boolean): Promise<number[] | {
-        tokens: number[];
-        mask: boolean[];
-    } | null>;
-}
-export declare function tokensFromTasks(tasks: Task[], tokenizer: ITokeniser, cb?: (tokens: number) => void): Promise<Uint16Array[]>;
-export declare function tokensFromTasks(tasks: Task[], tokenizer: ITokeniser, cb?: (tokens: number) => void, masking?: boolean): Promise<{
-    tokens: Uint16Array[];
-    mask: Uint8Array[];
-}>;

package/dist/training/tasks/Task.js

@@ -1,42 +0,0 @@
-import { yieldIfNeeded as e } from "../../utilities/yielder.js";
-//#region lib/training/tasks/Task.ts
-var t = class {};
-async function n(e, t, n, r, i, a) {
-	for (let o of e) {
-		let e = await o.nextTokens(n, a ? !0 : void 0);
-		if (e) {
-			let n = Array.isArray(e) ? e : e.tokens;
-			r.total += n.length;
-			let o = t[t.length - 1], s = a ? a[a.length - 1] : null;
-			if (r.offset + n.length > o.length) {
-				let c = o.length - r.offset;
-				o.set(n.slice(0, c), r.offset);
-				let l = n.length - c;
-				if (l > i) throw Error(`Estimated tokens (${i}) is too small for the next batch of tokens (${l}).`);
-				let u = new Uint16Array(i);
-				if (u.set(n.slice(c), 0), t.push(u), a && s && !Array.isArray(e)) {
-					s.set(e.mask.slice(0, c).map((e) => +!!e), r.offset);
-					let t = new Uint8Array(u.length);
-					t.set(e.mask.slice(c).map((e) => +!!e), 0), a.push(t);
-				}
-				r.offset = n.length - c;
-			} else o.set(n, r.offset), s && !Array.isArray(e) && s.set(e.mask.map((e) => +!!e), r.offset), r.offset += n.length;
-		}
-	}
-}
-async function r(t, r, i, a) {
-	let o = 1e4 * 1024, s = [new Uint16Array(o)], c = a ? [new Uint8Array(o)] : null, l = {
-		offset: 0,
-		total: 0
-	}, u = performance.now();
-	for (; await n(t, s, r, l, o, c || void 0), !t.every((e) => !e.hasMoreConversations());) u = await e(u, i, l.total);
-	return s.length === 1 ? c ? {
-		tokens: [s[0].subarray(0, l.offset)],
-		mask: [c[0].subarray(0, l.offset)]
-	} : [s[0].subarray(0, l.offset)] : (s[s.length - 1] = s[s.length - 1].subarray(0, l.offset), c ? (c[c.length - 1] = c[c.length - 1].subarray(0, l.offset), {
-		tokens: s,
-		mask: c
-	}) : s);
-}
-//#endregion
-export { t as Task, r as tokensFromTasks };

package/dist/training/tasks/splitter.d.ts

@@ -1,5 +0,0 @@
-import { Task } from './Task';
-export default function splitValidation(tasks: Task[], validationSplit: number): Promise<{
-    training: Task;
-    validation: Task;
-}>;

package/dist/training/tasks/splitter.js

@@ -1,18 +0,0 @@
-import { n as e } from "../../stream-DKl3GTDL.js";
-import t from "./ConversationTask.js";
-//#region lib/training/tasks/splitter.ts
-async function n(n, r) {
-	if (r <= 0 || r >= 1) throw Error("validationSplit must be between 0 and 1");
-	let i = [], a = [];
-	for (let e of n) for (; e.hasMoreConversations();) {
-		let t = await e.nextConversation();
-		if (!t) break;
-		Math.random() < r ? a.push(t) : i.push(t);
-	}
-	return {
-		training: new t([new e(i)]),
-		validation: new t([new e(a)])
-	};
-}
-//#endregion
-export { n as default };