@genai-fi/nanogpt 0.21.0 → 0.22.0

package/dist/tokeniser/BaseTokeniser.d.ts

@@ -1,3 +1,4 @@
+import { ConversationStream } from '../../data/stream';
 import { Conversation, ITokeniser } from './type';
 import { default as EE } from 'eventemitter3';
 export declare const SPECIALS: string[];
@@ -15,7 +16,7 @@ export default abstract class BaseTokeniser extends EE<'trainStatus'> implements
     protected addSpecialTokens(): void;
     protected addSpecialToken(token: string, index: number): void;
     generateID(): void;
-    abstract train(text: Conversation[][], cb?: (vocab: number) => void, datasetID?: string): Promise<number>;
+    abstract train(text: ConversationStream[], cb?: (vocab: number) => void, datasetID?: string): Promise<number>;
     abstract getVocab(): string[];
     abstract getMerges(): [string, string][];
     abstract destroy(): void;

package/dist/tokeniser/CharTokeniser.d.ts

@@ -1,5 +1,5 @@
 import { default as BaseTokeniser } from './BaseTokeniser';
-import { Conversation } from './type';
+import { ConversationStream } from '../../data/stream';
 export default class CharTokeniser extends BaseTokeniser {
     vocabSize: number;
     eosToken: number;
@@ -12,7 +12,7 @@ export default class CharTokeniser extends BaseTokeniser {
     addToken(token: string, index?: number): number;
     get trained(): boolean;
     destroy(): void;
-    train(text: Conversation[][], cb?: (vocab: number) => void, datasetID?: string): Promise<number>;
+    train(text: ConversationStream[], cb?: (vocab: number) => void, datasetID?: string): Promise<number>;
     tokenise(text: string[], numeric: true): number[][];
     tokenise(text: string[]): string[][];
     detokenise(tokens: (number[] | Uint16Array)[]): string[];

package/dist/tokeniser/CharTokeniser.js

@@ -38,19 +38,15 @@ var r = ["<eos>", "<unk>"], i = class extends n {
 	async train(t, n, i) {
 		this.datasetID = i;
 		let a = /* @__PURE__ */ new Set(), o = performance.now();
-		for (let r of t) r.forEach((e) => {
-			for (let t of e.content) a.add(t);
-		}), o = await e(o, n, 0);
+		for (let r of t) {
+			let t = r.cursor(), i = await t.next();
+			for (; i !== null;) i.forEach((e) => {
+				for (let t of e.content) a.add(t);
+			}), o = await e(o, n, 0), i = await t.next();
+		}
 		let s = Array.from(a), c = this.vocab.indexOf("", this.unkToken + 1), l = this.vocabSize - r.length;
 		if (c === -1) return this.generateID(), this.vocabSize;
-		if (this._trained = !0, s.length > l) {
-			let e = /* @__PURE__ */ new Map();
-			t.forEach((t) => {
-				t.forEach((t) => {
-					for (let n of t.content) e.set(n, (e.get(n) || 0) + 1);
-				});
-			}), s.sort((t, n) => (e.get(t) || 0) - (e.get(n) || 0)), s.splice(0, s.length - l);
-		}
+		if (this._trained = !0, s.length > l) throw Error("too_small_vocab");
 		let u = c;
 		if (u !== -1) {
 			let e = new Set(this.vocab);

package/dist/tokeniser/bpe.d.ts

@@ -1,5 +1,5 @@
 import { default as BaseTokeniser } from './BaseTokeniser';
-import { Conversation } from './type';
+import { ConversationStream } from '../../data/stream';
 export default class BPETokeniser extends BaseTokeniser {
     private targetSize;
     private vocab;
@@ -15,7 +15,7 @@ export default class BPETokeniser extends BaseTokeniser {
     get eosToken(): number;
     get bosToken(): number;
     get unkToken(): number;
-    train(text?: Conversation[][], cb?: (vocab: number) => void, datasetID?: string): Promise<number>;
+    train(text?: ConversationStream[], cb?: (vocab: number) => void, datasetID?: string): Promise<number>;
     getVocab(): string[];
     getMerges(): [string, string][];
     private tokeniseWord;

package/dist/tokeniser/bpe.js

@@ -100,40 +100,31 @@ var u = class extends n {
 	}
 	async train(t = [], n, i) {
 		this.datasetID = i;
-		let o = performance.now(), c = Array(t.length);
-		for (let i = 0; i < t.length; i++) {
-			let a = t[i], s = Array(a.length);
-			for (let e = 0; e < a.length; e++) s[e] = r(a[e].content);
-			o = await e(o, n, this.vocab.size), c[i] = s;
+		let o = performance.now(), c = /* @__PURE__ */ new Set();
+		for (let i of t) {
+			let t = i.cursor(), a = await t.next();
+			for (; a !== null;) {
+				for (let e of a) {
+					let t = r(e.content);
+					for (let e of t) c.add(e);
+				}
+				o = await e(o, n, this.vocab.size), a = await t.next();
+			}
 		}
-		let u = c.flat(2), d = new Set(u);
 		this.vocab = /* @__PURE__ */ new Set(), this.pretokenMap.clear(), this.merges = [], this.addSpecialTokens();
-		let f = Array.from(d), p = f.map((e) => Array.from(e).map((e) => (this.vocab.add(e), e))), m = a(p);
-		if (o = await e(o, n, this.vocab.size), this.vocab.size >= this.targetSize) {
-			console.warn("Initial vocab size is greater than or equal to target size. No merges will be performed.");
-			let e = /* @__PURE__ */ new Map();
-			u.forEach((t) => {
-				Array.from(t).forEach((t) => {
-					e.set(t, (e.get(t) || 0) + 1);
-				});
-			});
-			let t = Array.from(e.entries()).sort((e, t) => t[1] - e[1]);
-			this.vocab = /* @__PURE__ */ new Set(), this.addSpecialTokens(), t.slice(0, this.targetSize - this.vocab.size).map(([e]) => e).forEach((e) => this.vocab.add(e)), this.vocabIndex.clear();
-			let n = 0;
-			for (let e of this.vocab.keys()) this.vocabIndex.set(e, n++);
-			return this.generateID(), this.emit("trainStatus", "trained"), this.vocab.size;
-		}
+		let u = Array.from(c), d = u.map((e) => Array.from(e).map((e) => (this.vocab.add(e), e))), f = a(d);
+		if (o = await e(o, n, this.vocab.size), this.vocab.size >= this.targetSize) throw console.warn("Initial vocab size is greater than or equal to target size. No merges will be performed.", this.vocab.size, this.targetSize), Error("too_small_vocab");
 		for (; this.vocab.size < this.targetSize && this.merges.length < this.targetSize;) {
-			let t = s(m);
+			let t = s(f);
 			if (!t) break;
-			this.merges.push([t.a, t.b]), this.vocab.add(t.a + t.b), l(m, t), o = await e(o, n, this.vocab.size);
+			this.merges.push([t.a, t.b]), this.vocab.add(t.a + t.b), l(f, t), o = await e(o, n, this.vocab.size);
 		}
-		f.forEach((e, t) => {
-			let n = p[t];
+		u.forEach((e, t) => {
+			let n = d[t];
 			this.pretokenMap.set(e, n);
 		}), this.vocabIndex.clear();
-		let h = 0;
-		for (let e of this.vocab.keys()) this.vocabIndex.set(e, h++);
+		let p = 0;
+		for (let e of this.vocab.keys()) this.vocabIndex.set(e, p++);
 		return this.generateID(), this.emit("trainStatus", "trained"), this.vocab.size;
 	}
 	getVocab() {

package/dist/tokeniser/type.d.ts

@@ -1,13 +1,14 @@
 import { default as EE } from 'eventemitter3';
+import { ConversationStream } from '../data/stream';
 export type Roles = 'user' | 'assistant' | 'system' | 'text';
 export interface Conversation {
     role: Roles;
     content: string;
 }
-export interface ITokeniser extends EE<'trainStatus'> {
+export interface ITokeniser extends EE<'trainStatus' | 'error'> {
     id: string;
     datasetID?: string;
-    train(text: Conversation[][], cb?: (vocab: number) => void, datasetID?: string): Promise<number>;
+    train(text: ConversationStream[], cb?: (vocab: number) => void, datasetID?: string): Promise<number>;
     getVocab(): string[];
     getMerges(): [string, string][];
     destroy(): void;

package/dist/training/DatasetBuilder.d.ts

@@ -15,7 +15,7 @@ export declare class DatasetBuilder {
     tokenizer: ITokeniser;
     blockSize: number;
     constructor(tokenizer: ITokeniser, blockSize?: number);
-    createTextDataset(flatTokens: Uint16Array, batchSize?: number, indexes?: Uint32Array, mask?: Uint8Array, ignoreIndex?: number): Promise<{
+    createTextDataset(flatTokens: Uint16Array[], batchSize?: number, indexes?: Uint32Array, mask?: Uint8Array[], ignoreIndex?: number): Promise<{
         dataset: Dataset<{
             xs: Tensor;
             ys: Tensor;

package/dist/training/DatasetBuilder.js

@@ -1,2 +1,2 @@
-import { i as e, n as t, r as n, t as r } from "../DatasetBuilder-B9UGNQT5.js";
+import { i as e, n as t, r as n, t as r } from "../DatasetBuilder-Ctb425Id.js";
 export { r as DatasetBuilder, t as flattenTokens, n as flattenTokensWithMask, e as shuffle };

package/dist/training/PreTrainer.js

@@ -1,4 +1,4 @@
-import { t as e } from "../DatasetBuilder-B9UGNQT5.js";
+import { t as e } from "../DatasetBuilder-Ctb425Id.js";
 import t from "./BasicTrainer.js";
 //#region lib/training/PreTrainer.ts
 var n = {

package/dist/training/SFTTrainer.js

@@ -1,4 +1,4 @@
-import { t as e } from "../DatasetBuilder-B9UGNQT5.js";
+import { t as e } from "../DatasetBuilder-Ctb425Id.js";
 import t from "./BasicTrainer.js";
 //#region lib/training/SFTTrainer.ts
 var n = {

package/dist/training/tasks/ConversationTask.d.ts

@@ -1,18 +1,17 @@
-import { Conversation, ITokeniser } from '../../../main';
+import { Conversation, ConversationStream, ITokeniser } from '../../../main';
 import { Task } from './Task';
 export default class ConversationTask extends Task {
-    private rawConvo;
-    private shuffledIndices;
-    private index;
+    private streams;
+    private streamIndex;
+    private currentCursor;
     get length(): number;
-    constructor(conversations: Conversation[][]);
+    constructor(conversations: ConversationStream[]);
     hasMoreConversations(): boolean;
-    nextConversation(): Conversation[] | null;
-    nextTokens(tokeniser: ITokeniser): number[] | null;
-    nextTokens(tokeniser: ITokeniser, masking: boolean): {
+    nextConversation(): Promise<Conversation[] | null>;
+    nextTokens(tokeniser: ITokeniser): Promise<number[] | null>;
+    nextTokens(tokeniser: ITokeniser, masking: boolean): Promise<{
         tokens: number[];
         mask: boolean[];
-    } | null;
-    shuffle(): void;
+    } | null>;
     estimateTokens(tokeniser: ITokeniser): Promise<number>;
 }

package/dist/training/tasks/ConversationTask.js

@@ -1,38 +1,29 @@
-import { i as e } from "../../DatasetBuilder-B9UGNQT5.js";
-import { Task as t } from "./Task.js";
+import { Task as e } from "./Task.js";
 //#region lib/training/tasks/ConversationTask.ts
-var n = class extends t {
-	rawConvo;
-	shuffledIndices = null;
-	index = 0;
+var t = class extends e {
+	streams;
+	streamIndex = 0;
+	currentCursor = null;
 	get length() {
-		return this.rawConvo.length;
+		return this.streams.length;
 	}
 	constructor(e) {
-		super(), this.rawConvo = e;
+		super(), this.streams = e;
 	}
 	hasMoreConversations() {
-		return this.index < this.rawConvo.length;
+		return this.streamIndex < this.streams.length;
 	}
-	nextConversation() {
-		if (this.index >= this.rawConvo.length) return null;
-		let e = this.rawConvo[this.shuffledIndices ? this.shuffledIndices[this.index] : this.index];
-		return this.index++, e;
+	async nextConversation() {
+		return this.streamIndex < this.streams.length ? (this.currentCursor ||= this.streams[this.streamIndex].cursor(), await this.currentCursor.next() || (this.streamIndex++, this.currentCursor = null, this.nextConversation())) : null;
 	}
-	nextTokens(e, t) {
-		let n = this.nextConversation();
+	async nextTokens(e, t) {
+		let n = await this.nextConversation();
 		return n ? e.encodeConversation(n, !1, t) : null;
 	}
-	shuffle() {
-		if (!this.shuffledIndices) {
-			this.shuffledIndices = new Uint32Array(this.rawConvo.length);
-			for (let e = 0; e < this.rawConvo.length; e++) this.shuffledIndices[e] = e;
-		}
-		e(this.shuffledIndices), this.index = 0;
-	}
 	async estimateTokens(e) {
-		return e.encodeConversation(this.rawConvo[0]).length * this.length;
+		let t = await this.streams[0].cursor().next();
+		return t ? e.encodeConversation(t).length * this.length : 0;
 	}
 };
 //#endregion
-export { n as default };
+export { t as default };

package/dist/training/tasks/PretrainingTask.d.ts

@@ -6,12 +6,12 @@ export default class PretrainingTask extends Task {
     get length(): number;
     constructor(texts: string[]);
     hasMoreConversations(): boolean;
-    nextConversation(): Conversation[] | null;
-    nextTokens(tokeniser: ITokeniser): number[] | null;
-    nextTokens(tokeniser: ITokeniser, masking: boolean): {
+    nextConversation(): Promise<Conversation[] | null>;
+    nextTokens(tokeniser: ITokeniser): Promise<number[] | null>;
+    nextTokens(tokeniser: ITokeniser, masking: boolean): Promise<{
         tokens: number[];
         mask: boolean[];
-    } | null;
+    } | null>;
     shuffle(): void;
     estimateTokens(tokeniser: ITokeniser): Promise<number>;
 }

package/dist/training/tasks/PretrainingTask.js

@@ -12,7 +12,7 @@ var t = class extends e {
 	hasMoreConversations() {
 		return this.index < this.rawText.length;
 	}
-	nextConversation() {
+	async nextConversation() {
 		if (this.index >= this.rawText.length) return null;
 		let e = {
 			role: "assistant",
@@ -20,7 +20,7 @@ var t = class extends e {
 		};
 		return this.index++, [e];
 	}
-	nextTokens(e, t) {
+	async nextTokens(e, t) {
 		if (this.index >= this.rawText.length) return null;
 		let n = e.encodeSequence(this.rawText[this.index]);
 		return this.index++, t ? {

package/dist/training/tasks/StartSentenceTask.d.ts

@@ -6,12 +6,12 @@ export default class StartSentenceTask extends Task {
     get length(): number;
     constructor(texts: string[]);
     hasMoreConversations(): boolean;
-    nextConversation(): Conversation[] | null;
-    nextTokens(tokeniser: ITokeniser): number[] | null;
-    nextTokens(tokeniser: ITokeniser, masking: boolean): {
+    nextConversation(): Promise<Conversation[] | null>;
+    nextTokens(tokeniser: ITokeniser): Promise<number[] | null>;
+    nextTokens(tokeniser: ITokeniser, masking: boolean): Promise<{
         tokens: number[];
         mask: boolean[];
-    } | null;
+    } | null>;
     shuffle(): void;
     private conversationFromString;
     estimateTokens(tokeniser: ITokeniser): Promise<number>;

package/dist/training/tasks/StartSentenceTask.js

@@ -12,13 +12,13 @@ var t = class extends e {
 	hasMoreConversations() {
 		return this.index < this.rawText.length;
 	}
-	nextConversation() {
+	async nextConversation() {
 		if (this.index >= this.rawText.length) return null;
 		let e = this.rawText[this.index];
 		return this.index++, this.conversationFromString(e);
 	}
-	nextTokens(e, t) {
-		let n = this.nextConversation();
+	async nextTokens(e, t) {
+		let n = await this.nextConversation();
 		return n ? e.encodeConversation(n, !1, t) : null;
 	}
 	shuffle() {

package/dist/training/tasks/Task.d.ts

@@ -2,21 +2,19 @@ import { Conversation, ITokeniser } from '../../../main';
 export declare abstract class Task {
     abstract get length(): number;
     abstract hasMoreConversations(): boolean;
-    abstract nextConversation(): Conversation[] | null;
-    abstract nextTokens(tokeniser: ITokeniser): number[] | null;
-    abstract nextTokens(tokeniser: ITokeniser, masking: boolean): {
+    abstract nextConversation(): Promise<Conversation[] | null>;
+    abstract nextTokens(tokeniser: ITokeniser): Promise<number[] | null>;
+    abstract nextTokens(tokeniser: ITokeniser, masking: boolean): Promise<{
         tokens: number[];
         mask: boolean[];
-    } | null;
-    abstract nextTokens(tokeniser: ITokeniser, masking?: boolean): number[] | {
+    } | null>;
+    abstract nextTokens(tokeniser: ITokeniser, masking?: boolean): Promise<number[] | {
         tokens: number[];
         mask: boolean[];
-    } | null;
-    abstract estimateTokens(tokeniser: ITokeniser): Promise<number>;
-    abstract shuffle(): void;
+    } | null>;
 }
-export declare function tokensFromTasks(tasks: Task[], tokenizer: ITokeniser, cb?: (tokens: number) => void): Promise<Uint16Array>;
+export declare function tokensFromTasks(tasks: Task[], tokenizer: ITokeniser, cb?: (tokens: number) => void): Promise<Uint16Array[]>;
 export declare function tokensFromTasks(tasks: Task[], tokenizer: ITokeniser, cb?: (tokens: number) => void, masking?: boolean): Promise<{
-    tokens: Uint16Array;
-    mask: Uint8Array;
+    tokens: Uint16Array[];
+    mask: Uint8Array[];
 }>;

package/dist/training/tasks/Task.js

@@ -1,9 +1,9 @@
 import { yieldIfNeeded as e } from "../../utilities/yielder.js";
 //#region lib/training/tasks/Task.ts
 var t = class {};
-function n(e, t, n, r, i, a) {
+async function n(e, t, n, r, i, a) {
 	for (let o of e) {
-		let e = o.nextTokens(n, a ? !0 : void 0);
+		let e = await o.nextTokens(n, a ? !0 : void 0);
 		if (e) {
 			let n = Array.isArray(e) ? e : e.tokens;
 			r.total += n.length;
@@ -11,7 +11,9 @@ function n(e, t, n, r, i, a) {
 			if (r.offset + n.length > o.length) {
 				let c = o.length - r.offset;
 				o.set(n.slice(0, c), r.offset);
-				let l = n.length - c, u = new Uint16Array(Math.max(Math.floor(i * .1) + 100, l));
+				let l = n.length - c;
+				if (l > i) throw Error(`Estimated tokens (${i}) is too small for the next batch of tokens (${l}).`);
+				let u = new Uint16Array(i);
 				if (u.set(n.slice(c), 0), t.push(u), a && s && !Array.isArray(e)) {
 					s.set(e.mask.slice(0, c).map((e) => +!!e), r.offset);
 					let t = new Uint8Array(u.length);
@@ -23,33 +25,18 @@ function n(e, t, n, r, i, a) {
 	}
 }
 async function r(t, r, i, a) {
-	let o = Math.min((await Promise.all(t.map((e) => e.estimateTokens(r)))).reduce((e, t) => e + t, 0), r.vocabSize * 1e4), s = [new Uint16Array(o)], c = a ? [new Uint8Array(o)] : null, l = {
+	let o = 1e4 * 1024, s = [new Uint16Array(o)], c = a ? [new Uint8Array(o)] : null, l = {
 		offset: 0,
 		total: 0
 	}, u = performance.now();
-	for (; l.offset < o && (n(t, s, r, l, o, c || void 0), !t.every((e) => !e.hasMoreConversations()));) u = await e(u, i, l.total);
-	if (s.length === 1) return c ? {
-		tokens: s[0].subarray(0, l.offset),
-		mask: c[0].subarray(0, l.offset)
-	} : s[0].subarray(0, l.offset);
-	let d = s.reduce((e, t) => e + t.length, 0) - (s[s.length - 1].length - l.offset), f = new Uint16Array(d), p = 0;
-	for (let e = 0; e < s.length; e++) {
-		let t = s[e];
-		e === s.length - 1 ? (f.set(t.subarray(0, l.offset), p), p += l.offset) : (f.set(t, p), p += t.length);
-	}
-	if (c) {
-		let e = new Uint8Array(d);
-		p = 0;
-		for (let t = 0; t < c.length; t++) {
-			let n = c[t];
-			t === c.length - 1 ? (e.set(n.subarray(0, l.offset), p), p += l.offset) : (e.set(n, p), p += n.length);
-		}
-		return {
-			tokens: f,
-			mask: e
-		};
-	}
-	return f;
+	for (; await n(t, s, r, l, o, c || void 0), !t.every((e) => !e.hasMoreConversations());) u = await e(u, i, l.total);
+	return s.length === 1 ? c ? {
+		tokens: [s[0].subarray(0, l.offset)],
+		mask: [c[0].subarray(0, l.offset)]
+	} : [s[0].subarray(0, l.offset)] : (s[s.length - 1] = s[s.length - 1].subarray(0, l.offset), c ? (c[c.length - 1] = c[c.length - 1].subarray(0, l.offset), {
+		tokens: s,
+		mask: c
+	}) : s);
 }
 //#endregion
 export { t as Task, r as tokensFromTasks };

package/dist/training/tasks/splitter.d.ts

@@ -1,5 +1,5 @@
 import { Task } from './Task';
-export default function splitValidation(tasks: Task[], validationSplit: number): {
+export default function splitValidation(tasks: Task[], validationSplit: number): Promise<{
     training: Task;
     validation: Task;
-};
+}>;

package/dist/training/tasks/splitter.js

@@ -1,18 +1,18 @@
-import e from "./ConversationTask.js";
+import { n as e } from "../../stream-DKl3GTDL.js";
+import t from "./ConversationTask.js";
 //#region lib/training/tasks/splitter.ts
-function t(t, n) {
-	if (n <= 0 || n >= 1) throw Error("validationSplit must be between 0 and 1");
-	t.forEach((e) => e.shuffle());
-	let r = [], i = [];
-	for (let e of t) for (; e.hasMoreConversations();) {
-		let t = e.nextConversation();
+async function n(n, r) {
+	if (r <= 0 || r >= 1) throw Error("validationSplit must be between 0 and 1");
+	let i = [], a = [];
+	for (let e of n) for (; e.hasMoreConversations();) {
+		let t = await e.nextConversation();
 		if (!t) break;
-		Math.random() < n ? i.push(t) : r.push(t);
+		Math.random() < r ? a.push(t) : i.push(t);
 	}
 	return {
-		training: new e(r),
-		validation: new e(i)
+		training: new t([new e(i)]),
+		validation: new t([new e(a)])
 	};
 }
 //#endregion
-export { t as default };
+export { n as default };

package/dist/training/validation.d.ts

@@ -2,7 +2,7 @@ import { ITokeniser, Task } from '../../main';
 import { Tensor } from '@tensorflow/tfjs-core';
 import { Dataset } from '@tensorflow/tfjs-data';
 import { DatasetBuilder, DatasetState } from './DatasetBuilder';
-export declare function createTrainValidationSplit(tasks: Task[] | Uint16Array, tokeniser: ITokeniser, datasetBuilder: DatasetBuilder, batchSize: number, validationSplit?: number, masking?: boolean): Promise<{
+export declare function createTrainValidationSplit(tasks: Task[] | Uint16Array[], tokeniser: ITokeniser, datasetBuilder: DatasetBuilder, batchSize: number, validationSplit?: number, masking?: boolean): Promise<{
     trainDataset: Dataset<{
         xs: Tensor;
         ys: Tensor;

package/dist/training/validation.js

@@ -1,2 +1,2 @@
-import { o as e } from "../main-DdsoTJAW.js";
+import { o as e } from "../main-Bgc7_9kb.js";
 export { e as createTrainValidationSplit };

package/dist/utilities/tokens.d.ts

@@ -0,0 +1,2 @@
+export declare function sliceUint16Shards(shards: readonly Uint16Array[], start: number, endExclusive: number, shardSize?: number): Uint16Array;
+export declare function sliceUint8Shards(shards: readonly Uint8Array[], start: number, endExclusive: number, shardSize?: number): Uint8Array;

package/dist/utilities/tokens.js

@@ -0,0 +1,40 @@
+//#region lib/utilities/tokens.ts
+function e(e, t, n, r = e[0]?.length ?? 0) {
+	if (e.length === 0) return new Uint16Array();
+	if (!Number.isInteger(t) || !Number.isInteger(n)) throw RangeError("start/endExclusive must be integers");
+	if (t < 0 || n < t) throw RangeError(`Invalid range [${t}, ${n})`);
+	if (r <= 0) throw RangeError("Invalid shardSize");
+	let i = e[e.length - 1], a = (e.length - 1) * r + i.length;
+	if (n > a) throw RangeError(`Slice end ${n} is out of bounds for total length ${a}`);
+	if (t === n) return new Uint16Array();
+	let o = new Uint16Array(n - t), s = 0, c = t;
+	for (; c < n;) {
+		let t = Math.floor(c / r), i = c - t * r, a = e[t], l = Math.min(n - c, a.length - i);
+		try {
+			o.set(a.subarray(i, i + l), s);
+		} catch (n) {
+			throw console.error(`Error slicing shards: shardIndex=${t}, shardOffset=${i}, take=${l}, writeOffset=${s}`), e.forEach((t, n) => {
+				t.length !== r && n !== e.length - 1 && console.error(`Shard ${n} has length ${t.length}, expected ${r}`);
+			}), n;
+		}
+		c += l, s += l;
+	}
+	return o;
+}
+function t(e, t, n, r = e[0]?.length ?? 0) {
+	if (e.length === 0) return new Uint8Array();
+	if (!Number.isInteger(t) || !Number.isInteger(n)) throw RangeError("start/endExclusive must be integers");
+	if (t < 0 || n < t) throw RangeError(`Invalid range [${t}, ${n})`);
+	if (r <= 0) throw RangeError("Invalid shardSize");
+	let i = e[e.length - 1], a = (e.length - 1) * r + i.length;
+	if (n > a) throw RangeError(`Slice end ${n} is out of bounds for total length ${a}`);
+	if (t === n) return new Uint8Array();
+	let o = new Uint8Array(n - t), s = 0, c = t;
+	for (; c < n;) {
+		let t = Math.floor(c / r), i = c - t * r, a = e[t], l = Math.min(n - c, a.length - i);
+		o.set(a.subarray(i, i + l), s), c += l, s += l;
+	}
+	return o;
+}
+//#endregion
+export { e as sliceUint16Shards, t as sliceUint8Shards };

package/dist/utilities/topP.js

@@ -6,6 +6,12 @@ function e(e, t) {
 	})).sort((e, t) => t.prob - e.prob), r = 0, i = Array(n.length).fill(0);
 	for (let e of n) if (r += e.prob, i[e.index] = e.prob, r >= t) break;
 	let a = i.reduce((e, t) => e + t, 0);
+	if (a === 0) {
+		let t = e[0], n = t.reduce((e, t) => e + t, 0);
+		if (n > 0) return t.map((e) => e / n);
+		let r = 1 / t.length;
+		return t.map(() => r);
+	}
 	return i.map((e) => e / a);
 }
 //#endregion

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@genai-fi/nanogpt",
-    "version": "0.21.0",
+    "version": "0.22.0",
     "type": "module",
     "main": "dist/main.js",
     "types": "dist/main.d.ts",
@@ -52,6 +52,7 @@
     "dependencies": {
         "@tensorflow/tfjs": "^4.22.0",
         "@tensorflow/tfjs-backend-webgpu": "^4.22.0",
+        "@zip.js/zip.js": "^2.8.34",
         "eventemitter3": "^5.0.1",
         "jszip": "^3.10.1",
         "papaparse": "^5.5.3",