@genai-fi/nanogpt 0.23.0 → 0.24.0

package/dist/{DatasetBuilder-C0iJT29K.js → DatasetBuilder-DU1G1OKX.js}

@@ -58,17 +58,17 @@ function b(e) {
 	return e != null && !ArrayBuffer.isView(e) && (Array.isArray(e) || typeof e == "object" && !(e instanceof f) && !(e instanceof Promise) && !t);
 }
 function x(e) {
-	return e == null || S(e) || Array.isArray(e) || typeof e == "object" && e instanceof f || l(e);
+	return e == null || ee(e) || Array.isArray(e) || typeof e == "object" && e instanceof f || l(e);
 }
-function S(e) {
+function ee(e) {
 	return e === null || typeof e != "object" && typeof e != "function";
 }
 //#endregion
 //#region node_modules/@tensorflow/tfjs-data/dist/util/deep_clone.js
-function C(e) {
-	return h(e, ee);
+function S(e) {
+	return h(e, C);
 }
-function ee(e) {
+function C(e) {
 	return e instanceof f ? {
 		value: e.clone(),
 		recurse: !1
@@ -249,7 +249,7 @@ var O = class {
 		};
 		let e = this.items[this.trav];
 		return this.trav++, {
-			value: C(e),
+			value: S(e),
 			done: !1
 		};
 	}
@@ -683,9 +683,9 @@ var ne = class {
 		let { batchSize: n = 32, noShuffle: r = !1, ignoreIndex: i = 65535 } = t || {}, a = e.getTokenCount();
 		if (a < this.blockSize + 1) throw Error(`Not enough tokens (${a}) for block size ${this.blockSize}`);
 		let o = Math.ceil(e.shardSize / this.blockSize), c = {
-			shuffledShards: new Uint32Array(e.getShardCount()),
-			shuffledIndexes: new Uint32Array(o),
-			lastShardIndexes: new Uint32Array(Math.ceil(e.getShardLength(e.getShardCount() - 1) / this.blockSize)),
+			shuffledShards: new Uint16Array(e.getShardCount()),
+			shuffledIndexes: new Uint16Array(o),
+			lastShardIndexes: new Uint16Array(Math.ceil(e.getShardLength(e.getShardCount() - 1) / this.blockSize)),
 			currentMask: null,
 			nextMask: null,
 			currentShard: null,
@@ -703,22 +703,19 @@ var ne = class {
 		})), {
 			dataset: Y(async function* () {
 				for (;;) {
-					let t = (c.shardIndex === c.shuffledShards.length - 1 ? c.lastShardIndexes : c.shuffledIndexes)[c.step], n = t * this.blockSize, a = c.currentShard, o = c.currentMask, s = $(c, e, r);
-					if (!a) break;
-					if (n + this.blockSize + 1 > a.length) {
-						console.warn("Index out of bounds for current shard, moving to next shard", t, n, a.length);
-						continue;
-					}
-					let l = new Int32Array(a.slice(n, n + this.blockSize)), u = new Int32Array(a.slice(n + 1, n + this.blockSize + 1));
-					if (o) {
-						let e = 0, t = o.slice(n + 1, n + this.blockSize + 1);
-						for (let n = 0; n < u.length; n++) t[n] === 0 && (u[n] = i, e++);
-						if (e === u.length) continue;
+					let t = (c.shardIndex === c.shuffledShards.length - 1 ? c.lastShardIndexes : c.shuffledIndexes)[c.step] * this.blockSize, n = c.currentShard, a = c.currentMask, o = $(c, e, r);
+					if (!n) break;
+					t + this.blockSize + 1 > n.length && (t = n.length - this.blockSize - 1);
+					let s = new Int32Array(n.slice(t, t + this.blockSize)), l = new Int32Array(n.slice(t + 1, t + this.blockSize + 1));
+					if (a) {
+						let e = 0, n = a.slice(t + 1, t + this.blockSize + 1);
+						for (let t = 0; t < l.length; t++) n[t] === 0 && (l[t] = i, e++);
+						if (e === l.length) continue;
 					}
 					yield {
-						xs: l,
-						ys: u
-					}, await s;
+						xs: s,
+						ys: l
+					}, await o;
 				}
 			}.bind(this)).batch(n).map((e) => {
 				let t = e;

package/dist/Generator.js

@@ -1,2 +1,2 @@
-import { a as e, o as t } from "./main-BSaDGH7I.js";
-export { e as default, t as isConversation };
+import { a as e, i as t } from "./main-Dz72vadm.js";
+export { t as default, e as isConversation };

package/dist/TeachableLLM.d.ts

@@ -7,7 +7,6 @@ import { default as Trainer, TrainingType } from './Trainer';
 import { ConversationStream } from './main';
 import { default as MemoryProfiler } from './utilities/profile';
 import { default as Model, ModelForwardAttributes } from './models/model';
-import { Task } from './training/tasks/Task';
 import { TrainingLogEntry, TrainingOptions } from './training/types';
 import { ModelMode, TransformersMetadata } from './loader/types';
 type TeachableLLMStatus = 'warmup' | 'awaitingTokens' | 'ready' | 'training' | 'loading' | 'busy' | 'error';
@@ -51,7 +50,7 @@ export default class TeachableLLM {
     set enableProfiler(value: boolean);
     getNumParams(): number;
     trainer(trainingType?: TrainingType, options?: TrainingOptions): Trainer;
-    train(text: Task[], options?: TrainingOptions, trainingType?: TrainingType): Promise<void>;
+    train(text: ConversationStream[], options?: TrainingOptions, trainingType?: TrainingType): Promise<void>;
     trainTokeniser(text: ConversationStream[]): Promise<number>;
     generator(): IGenerator;
     generateText(prompt: Conversation[], options?: IGenerateOptions): Promise<Conversation[]>;

package/dist/TeachableLLM.js

@@ -1,2 +1,2 @@
-import { i as e } from "./main-BSaDGH7I.js";
+import { r as e } from "./main-Dz72vadm.js";
 export { e as default };

package/dist/{Trainer-DBsyWJ4s.js → Trainer-Cr7csbTD.js}

@@ -2,7 +2,7 @@ import { packingSupported as e } from "./utilities/packed.js";
 import { t } from "./eventemitter3-D_qV3Lof.js";
 import n from "./training/PreTrainer.js";
 import { TokenStore as r } from "./training/tasks/TokenStore.js";
-import { tokensFromTasks as i } from "./training/tasks/Task.js";
+import { tokensFromStreams as i } from "./training/tasks/tokenStream.js";
 import { createTrainValidationDatasets as a, storeFromArray as o } from "./training/validation.js";
 import s from "./training/SFTTrainer.js";
 //#region node_modules/uuid/dist/stringify.js

package/dist/Trainer.d.ts

@@ -1,11 +1,11 @@
 import { ITokeniser } from './tokeniser/type';
 import { default as EE } from 'eventemitter3';
 import { default as Model, ModelForwardAttributes } from './models/model';
-import { Task } from './training/tasks/Task';
 import { TrainingOptions, TrainingLogEntry } from './training/types';
 import { AdamWOptimizer } from './training/AdamW';
 import { DatasetMetadata } from './loader/types';
 import { TokenStore } from './training/tasks/TokenStore';
+import { ConversationStream } from './data/stream';
 interface TrainingProgress {
     lastLog: TrainingLogEntry;
     progress: number;
@@ -34,7 +34,7 @@ export default class Trainer extends EE<'start' | 'stop' | 'log'> {
     dispose(): void;
     getTotalTokens(): number;
     setOptions(options: TrainingOptions): void;
-    prepare(tasks?: Task[] | Uint16Array[] | TokenStore, validation?: Uint16Array[] | TokenStore, datasets?: DatasetMetadata[]): Promise<void>;
+    prepare(tasks?: ConversationStream[] | Uint16Array[] | TokenStore, validation?: Uint16Array[] | TokenStore, datasets?: DatasetMetadata[]): Promise<void>;
     private configureModel;
     train(): Promise<void>;
     step(options?: TrainingOptions): Promise<void>;

package/dist/Trainer.js

@@ -1,2 +1,2 @@
-import { t as e } from "./Trainer-DBsyWJ4s.js";
+import { t as e } from "./Trainer-Cr7csbTD.js";
 export { e as default };

package/dist/data/stream.d.ts

@@ -1,19 +1,19 @@
 import { Conversation } from '../../tokeniser/type';
-export interface ConversationCursor {
-    next(): Promise<Conversation[] | null>;
-}
 export interface ConversationStream {
-    cursor(): ConversationCursor;
+    begin(cb: (conv: Conversation[]) => void, yieldCb?: () => void): Promise<void>;
+    step(cb: (conv: Conversation[]) => void): Promise<() => Promise<boolean>>;
 }
 export declare class MemoryConversationStream implements ConversationStream {
     private conversations;
     constructor(conversations: Conversation[][]);
-    cursor(): ConversationCursor;
+    step(cb: (conv: Conversation[]) => void): Promise<() => Promise<boolean>>;
+    begin(cb: (conv: Conversation[]) => void, yieldCb?: () => void): Promise<void>;
 }
 declare class JSONLFromReadableStream implements ConversationStream {
     private sourceFactory;
     constructor(sourceFactory: () => Promise<ReadableStream<Uint8Array>>);
-    cursor(): ConversationCursor;
+    step(cb: (conv: Conversation[]) => void): Promise<() => Promise<boolean>>;
+    begin(cb: (conv: Conversation[]) => void, yieldCb?: () => void): Promise<void>;
 }
 export declare class JSONLConversationStream extends JSONLFromReadableStream {
     constructor(file: File);

package/dist/data/stream.js

@@ -1,2 +1,2 @@
-import { n as e, r as t, t as n } from "../stream-BjdpSNqB.js";
+import { n as e, r as t, t as n } from "../stream-BpAwcvHz.js";
 export { n as JSONLConversationStream, e as MemoryConversationStream, t as ZipJSONLConversationStream };

package/dist/data/textLoader.js

@@ -1,7 +1,7 @@
 import { i as e, t } from "../chunk-CWhphoD1.js";
 import { loadPDF as n } from "./pdf.js";
 import { loadDOCX as r } from "./docx.js";
-import { n as i, r as a, t as o } from "../stream-BjdpSNqB.js";
+import { n as i, r as a, t as o } from "../stream-BpAwcvHz.js";
 //#endregion
 //#region lib/data/textLoader.ts
 var s = /* @__PURE__ */ e((/* @__PURE__ */ t(((e, t) => {

package/dist/loader/load.js

@@ -1,2 +1,2 @@
-import { c as e, l as t } from "../main-BSaDGH7I.js";
-export { e as VERSION, t as loadModel };
+import { c as e, s as t } from "../main-Dz72vadm.js";
+export { t as VERSION, e as loadModel };

package/dist/loader/loadHF.js

@@ -1,2 +1,2 @@
-import { u as e } from "../main-BSaDGH7I.js";
+import { l as e } from "../main-Dz72vadm.js";
 export { e as default };

package/dist/loader/loadTransformers.js

@@ -1,2 +1,2 @@
-import { f as e, p as t } from "../main-BSaDGH7I.js";
+import { d as e, f as t } from "../main-Dz72vadm.js";
 export { e as default, t as mapTransformersConfigToGPTConfig };

package/dist/loader/newZipLoad.js

@@ -1,2 +1,2 @@
-import { d as e } from "../main-BSaDGH7I.js";
+import { u as e } from "../main-Dz72vadm.js";
 export { e as default };

package/dist/loader/oldZipLoad.js

@@ -1,2 +1,2 @@
-import { m as e } from "../main-BSaDGH7I.js";
+import { p as e } from "../main-Dz72vadm.js";
 export { e as default };

package/dist/loader/save.js

@@ -1,2 +1,2 @@
-import { s as e } from "../main-BSaDGH7I.js";
+import { o as e } from "../main-Dz72vadm.js";
 export { e as saveModel };