@genai-fi/nanogpt 0.23.0 → 1.0.0

package/dist/training/tasks/ConversationTask.d.ts

@@ -1,17 +0,0 @@
-import { Conversation, ConversationStream, ITokeniser } from '../../../main';
-import { Task } from './Task';
-export default class ConversationTask extends Task {
-    private streams;
-    private streamIndex;
-    private currentCursor;
-    get length(): number;
-    constructor(conversations: ConversationStream[]);
-    hasMoreConversations(): boolean;
-    nextConversation(): Promise<Conversation[] | null>;
-    nextTokens(tokeniser: ITokeniser): Promise<number[] | null>;
-    nextTokens(tokeniser: ITokeniser, masking: boolean): Promise<{
-        tokens: number[];
-        mask: boolean[];
-    } | null>;
-    estimateTokens(tokeniser: ITokeniser): Promise<number>;
-}

package/dist/training/tasks/ConversationTask.js

@@ -1,29 +0,0 @@
-import { Task as e } from "./Task.js";
-//#region lib/training/tasks/ConversationTask.ts
-var t = class extends e {
-	streams;
-	streamIndex = 0;
-	currentCursor = null;
-	get length() {
-		return this.streams.length;
-	}
-	constructor(e) {
-		super(), this.streams = e;
-	}
-	hasMoreConversations() {
-		return this.streamIndex < this.streams.length;
-	}
-	async nextConversation() {
-		return this.streamIndex < this.streams.length ? (this.currentCursor ||= this.streams[this.streamIndex].cursor(), await this.currentCursor.next() || (this.streamIndex++, this.currentCursor = null, this.nextConversation())) : null;
-	}
-	async nextTokens(e, t) {
-		let n = await this.nextConversation();
-		return n ? e.encodeConversation(n, !1, t) : null;
-	}
-	async estimateTokens(e) {
-		let t = await this.streams[0].cursor().next();
-		return t ? e.encodeConversation(t).length * this.length : 0;
-	}
-};
-//#endregion
-export { t as default };

package/dist/training/tasks/PretrainingTask.d.ts

@@ -1,17 +0,0 @@
-import { Conversation, ITokeniser } from '../../../main';
-import { Task } from './Task';
-export default class PretrainingTask extends Task {
-    private rawText;
-    private index;
-    get length(): number;
-    constructor(texts: string[]);
-    hasMoreConversations(): boolean;
-    nextConversation(): Promise<Conversation[] | null>;
-    nextTokens(tokeniser: ITokeniser): Promise<number[] | null>;
-    nextTokens(tokeniser: ITokeniser, masking: boolean): Promise<{
-        tokens: number[];
-        mask: boolean[];
-    } | null>;
-    shuffle(): void;
-    estimateTokens(tokeniser: ITokeniser): Promise<number>;
-}

package/dist/training/tasks/PretrainingTask.js

@@ -1,42 +0,0 @@
-import { Task as e } from "./Task.js";
-//#region lib/training/tasks/PretrainingTask.ts
-var t = class extends e {
-	rawText;
-	index = 0;
-	get length() {
-		return this.rawText.length;
-	}
-	constructor(e) {
-		super(), this.rawText = e;
-	}
-	hasMoreConversations() {
-		return this.index < this.rawText.length;
-	}
-	async nextConversation() {
-		if (this.index >= this.rawText.length) return null;
-		let e = {
-			role: "assistant",
-			content: this.rawText[this.index]
-		};
-		return this.index++, [e];
-	}
-	async nextTokens(e, t) {
-		if (this.index >= this.rawText.length) return null;
-		let n = e.encodeSequence(this.rawText[this.index]);
-		return this.index++, t ? {
-			tokens: n,
-			mask: Array(n.length).fill(!0)
-		} : n;
-	}
-	shuffle() {
-		this.index = 0;
-	}
-	async estimateTokens(e) {
-		return e.encodeConversation([{
-			role: "assistant",
-			content: this.rawText[0]
-		}]).length * this.length;
-	}
-};
-//#endregion
-export { t as default };

package/dist/training/tasks/StartSentenceTask.d.ts

@@ -1,18 +0,0 @@
-import { Conversation, ITokeniser } from '../../../main';
-import { Task } from './Task';
-export default class StartSentenceTask extends Task {
-    private rawText;
-    private index;
-    get length(): number;
-    constructor(texts: string[]);
-    hasMoreConversations(): boolean;
-    nextConversation(): Promise<Conversation[] | null>;
-    nextTokens(tokeniser: ITokeniser): Promise<number[] | null>;
-    nextTokens(tokeniser: ITokeniser, masking: boolean): Promise<{
-        tokens: number[];
-        mask: boolean[];
-    } | null>;
-    shuffle(): void;
-    private conversationFromString;
-    estimateTokens(tokeniser: ITokeniser): Promise<number>;
-}

package/dist/training/tasks/StartSentenceTask.js

@@ -1,45 +0,0 @@
-import { Task as e } from "./Task.js";
-//#region lib/training/tasks/StartSentenceTask.ts
-var t = class extends e {
-	rawText;
-	index = 0;
-	get length() {
-		return this.rawText.length;
-	}
-	constructor(e) {
-		super(), this.rawText = e;
-	}
-	hasMoreConversations() {
-		return this.index < this.rawText.length;
-	}
-	async nextConversation() {
-		if (this.index >= this.rawText.length) return null;
-		let e = this.rawText[this.index];
-		return this.index++, this.conversationFromString(e);
-	}
-	async nextTokens(e, t) {
-		let n = await this.nextConversation();
-		return n ? e.encodeConversation(n, !1, t) : null;
-	}
-	shuffle() {
-		this.index = 0;
-	}
-	conversationFromString(e) {
-		let t = e.indexOf(".");
-		return t === -1 ? [{
-			role: "assistant",
-			content: this.rawText[this.index]
-		}] : [{
-			role: "user",
-			content: e.slice(0, t + 1).trim()
-		}, {
-			role: "assistant",
-			content: e.slice(t + 1).trim()
-		}];
-	}
-	async estimateTokens(e) {
-		return (await e.encodeConversation(this.conversationFromString(this.rawText[0]))).length * this.length;
-	}
-};
-//#endregion
-export { t as default };

package/dist/training/tasks/Task.d.ts

@@ -1,29 +0,0 @@
-import { Conversation, ITokeniser } from '../../../main';
-import { TokenStore } from './TokenStore';
-export declare abstract class Task {
-    abstract get length(): number;
-    abstract hasMoreConversations(): boolean;
-    abstract nextConversation(): Promise<Conversation[] | null>;
-    abstract nextTokens(tokeniser: ITokeniser): Promise<number[] | null>;
-    abstract nextTokens(tokeniser: ITokeniser, masking: boolean): Promise<{
-        tokens: number[];
-        mask: boolean[];
-    } | null>;
-    abstract nextTokens(tokeniser: ITokeniser, masking?: boolean): Promise<number[] | {
-        tokens: number[];
-        mask: boolean[];
-    } | null>;
-}
-interface TokensFromTasksOptions {
-    masking?: boolean;
-    maxCachedShards?: number;
-    noOPFS?: boolean;
-    shardSize?: number;
-    validationSplit?: number;
-    cb?: (tokens: number) => void;
-}
-export declare function tokensFromTasks(tasks: Task[], tokenizer: ITokeniser, options?: TokensFromTasksOptions): Promise<{
-    trainingTokens: TokenStore;
-    validationTokens?: TokenStore;
-}>;
-export {};

package/dist/training/tasks/Task.js

@@ -1,50 +0,0 @@
-import { yieldIfNeeded as e } from "../../utilities/yielder.js";
-import { createTokenStore as t, deleteTokenStore as n } from "./TokenStore.js";
-//#region lib/training/tasks/Task.ts
-var r = class {};
-async function i(e, t, n, r, i, a) {
-	for (let o of e) {
-		let e = await o.nextTokens(n, a ? !0 : void 0);
-		if (e) {
-			let n = Array.isArray(e) ? e : e.tokens;
-			r.total += n.length;
-			let o = t[t.length - 1], s = a ? a[a.length - 1] : null;
-			if (r.offset + n.length > o.length) {
-				let c = o.length - r.offset;
-				o.set(n.slice(0, c), r.offset);
-				let l = n.length - c;
-				if (l > i) throw Error(`Estimated tokens (${i}) is too small for the next batch of tokens (${l}).`);
-				let u = new Uint16Array(i);
-				if (u.set(n.slice(c), 0), t.push(u), a && s && !Array.isArray(e)) {
-					s.set(e.mask.slice(0, c).map((e) => +!!e), r.offset);
-					let t = new Uint8Array(u.length);
-					t.set(e.mask.slice(c).map((e) => +!!e), 0), a.push(t);
-				}
-				r.offset = n.length - c;
-			} else o.set(n, r.offset), s && !Array.isArray(e) && s.set(e.mask.map((e) => +!!e), r.offset), r.offset += n.length;
-		}
-	}
-}
-async function a(r, a, o) {
-	await n("training-tokens");
-	let s = await t("training-tokens", a.id, a.datasetID ?? "", o);
-	await n("validation-tokens");
-	let c = o?.validationSplit && o.validationSplit > 0 ? await t("validation-tokens", a.id, a.datasetID ?? "", o) : void 0, l = [new Uint16Array(s.shardSize)], u = o?.masking ? [new Uint8Array(s.shardSize)] : null, d = {
-		offset: 0,
-		total: 0
-	}, f = o?.validationSplit && o.validationSplit > 0 ? [new Uint16Array(c.shardSize)] : void 0, p = o?.masking && f ? [new Uint8Array(c.shardSize)] : null, m = {
-		offset: 0,
-		total: 0
-	}, h = performance.now();
-	for (;;) {
-		let t = o?.validationSplit && o.validationSplit > 0 && Math.random() < o.validationSplit, n = t ? f : l, g = t ? m : d, _ = t ? p : u, v = t ? c : s;
-		if (await i(r, n, a, g, v.shardSize, _ || void 0), n.length > 1 && (v.appendShard(n[0], _ ? _[0] : void 0), n.shift(), _ && _.shift()), r.every((e) => !e.hasMoreConversations())) break;
-		h = await e(h, o?.cb, d.total);
-	}
-	return l.length === 1 && (l[0] = l[0].subarray(0, d.offset), await s.appendShard(l[0], u ? u[0].subarray(0, d.offset) : void 0)), f && f.length === 1 && (f[0] = f[0].subarray(0, m.offset), await c.appendShard(f[0], p ? p[0].subarray(0, m.offset) : void 0)), {
-		trainingTokens: s,
-		validationTokens: f ? c : void 0
-	};
-}
-//#endregion
-export { r as Task, a as tokensFromTasks };

package/dist/training/tasks/splitter.d.ts

@@ -1,5 +0,0 @@
-import { Task } from './Task';
-export default function splitValidation(tasks: Task[], validationSplit: number): Promise<{
-    training: Task;
-    validation: Task;
-}>;

package/dist/training/tasks/splitter.js

@@ -1,18 +0,0 @@
-import { n as e } from "../../stream-BjdpSNqB.js";
-import t from "./ConversationTask.js";
-//#region lib/training/tasks/splitter.ts
-async function n(n, r) {
-	if (r <= 0 || r >= 1) throw Error("validationSplit must be between 0 and 1");
-	let i = [], a = [];
-	for (let e of n) for (; e.hasMoreConversations();) {
-		let t = await e.nextConversation();
-		if (!t) break;
-		Math.random() < r ? a.push(t) : i.push(t);
-	}
-	return {
-		training: new t([new e(i)]),
-		validation: new t([new e(a)])
-	};
-}
-//#endregion
-export { n as default };