@genai-fi/nanogpt 0.15.13 → 0.16.1

package/dist/training/tasks/ConversationTask.js

@@ -1,6 +1,8 @@
 import { Task as t } from "./Task.js";
+import { shuffle as s } from "../DatasetBuilder.js";
 class a extends t {
   rawConvo;
+  shuffledIndices = null;
   index = 0;
   get length() {
     return this.rawConvo.length;
@@ -14,20 +16,20 @@ class a extends t {
   nextConversation() {
     if (this.index >= this.rawConvo.length)
       return null;
-    const n = this.rawConvo[this.index];
+    const n = this.rawConvo[this.shuffledIndices ? this.shuffledIndices[this.index] : this.index];
     return this.index++, n;
   }
   nextTokens(n) {
-    const o = this.nextConversation();
-    return o ? n.encodeConversation(o) : null;
-  }
-  getRandomConversation() {
-    const n = Math.floor(Math.random() * this.rawConvo.length);
-    return this.rawConvo[n];
-  }
-  getRandomTokens(n) {
-    const o = Math.floor(Math.random() * this.rawConvo.length);
-    return n.encodeConversation(this.rawConvo[o]);
+    const e = this.nextConversation();
+    return e ? n.encodeConversation(e) : null;
+  }
+  shuffle() {
+    if (!this.shuffledIndices) {
+      this.shuffledIndices = new Uint32Array(this.rawConvo.length);
+      for (let n = 0; n < this.rawConvo.length; n++)
+        this.shuffledIndices[n] = n;
+    }
+    s(this.shuffledIndices), this.index = 0;
   }
   async estimateTokens(n) {
     return (await n.encodeConversation(this.rawConvo[0])).length * this.length;

package/dist/training/tasks/PretrainingTask.d.ts

@@ -8,7 +8,6 @@ export default class PretrainingTask extends Task {
     hasMoreConversations(): boolean;
     nextConversation(): Conversation[] | null;
     nextTokens(tokeniser: ITokeniser): number[] | null;
-    getRandomConversation(): Conversation[];
-    getRandomTokens(tokeniser: ITokeniser): number[];
+    shuffle(): void;
     estimateTokens(tokeniser: ITokeniser): Promise<number>;
 }

package/dist/training/tasks/PretrainingTask.js

@@ -1,5 +1,5 @@
 import { Task as n } from "./Task.js";
-class i extends n {
+class r extends n {
   rawText;
   index = 0;
   get length() {
@@ -26,18 +26,8 @@ class i extends n {
     const e = t.encodeSequence(this.rawText[this.index]);
     return this.index++, e;
   }
-  getRandomConversation() {
-    const t = Math.floor(Math.random() * this.rawText.length);
-    return [
-      {
-        role: "assistant",
-        content: this.rawText[t]
-      }
-    ];
-  }
-  getRandomTokens(t) {
-    const e = Math.floor(Math.random() * this.rawText.length);
-    return t.encodeSequence(this.rawText[e]);
+  shuffle() {
+    this.index = 0;
   }
   async estimateTokens(t) {
     return (await t.encodeConversation([
@@ -49,5 +39,5 @@ class i extends n {
   }
 }
 export {
-  i as default
+  r as default
 };

package/dist/training/tasks/StartSentenceTask.d.ts

@@ -8,8 +8,7 @@ export default class StartSentenceTask extends Task {
     hasMoreConversations(): boolean;
     nextConversation(): Conversation[] | null;
     nextTokens(tokeniser: ITokeniser): number[] | null;
-    getRandomConversation(): Conversation[];
-    getRandomTokens(tokeniser: ITokeniser): number[];
+    shuffle(): void;
     private conversationFromString;
     estimateTokens(tokeniser: ITokeniser): Promise<number>;
 }

package/dist/training/tasks/StartSentenceTask.js

@@ -21,13 +21,8 @@ class a extends e {
     const n = this.nextConversation();
     return n ? t.encodeConversation(n) : null;
   }
-  getRandomConversation() {
-    const t = Math.floor(Math.random() * this.rawText.length);
-    return this.conversationFromString(this.rawText[t]);
-  }
-  getRandomTokens(t) {
-    const n = this.getRandomConversation();
-    return t.encodeConversation(n);
+  shuffle() {
+    this.index = 0;
   }
   conversationFromString(t) {
     const n = t.indexOf(".");

package/dist/training/tasks/Task.d.ts

@@ -5,7 +5,6 @@ export declare abstract class Task {
     abstract nextConversation(): Conversation[] | null;
     abstract nextTokens(tokeniser: ITokeniser): number[] | null;
     abstract estimateTokens(tokeniser: ITokeniser): Promise<number>;
-    abstract getRandomConversation(): Conversation[];
-    abstract getRandomTokens(tokeniser: ITokeniser): number[];
+    abstract shuffle(): void;
 }
 export declare function tokensFromTasks(tasks: Task[], tokenizer: ITokeniser, cb?: (tokens: number) => void): Promise<Uint16Array>;

package/dist/training/tasks/splitter.d.ts

@@ -0,0 +1,5 @@
+import { Task } from './Task';
+export default function splitValidation(tasks: Task[], validationSplit: number): {
+    training: Task;
+    validation: Task;
+};

package/dist/training/tasks/splitter.js

@@ -0,0 +1,21 @@
+import s from "./ConversationTask.js";
+function f(e, o) {
+  if (o <= 0 || o >= 1)
+    throw new Error("validationSplit must be between 0 and 1");
+  e.forEach((n) => n.shuffle());
+  const r = [], a = [];
+  for (const n of e)
+    for (; n.hasMoreConversations(); ) {
+      const t = n.nextConversation();
+      if (!t)
+        break;
+      Math.random() < o ? a.push(t) : r.push(t);
+    }
+  return {
+    training: new s(r),
+    validation: new s(a)
+  };
+}
+export {
+  f as default
+};

package/dist/training/validation.js

@@ -39,8 +39,8 @@ import "../ops/webgl/adamAdjust.js";
 import "../ops/cpu/adamMoments.js";
 import "../ops/webgl/adamMoments.js";
 import { PAGE_FACTOR as m, shuffle as h } from "./DatasetBuilder.js";
-import "../papaparse.min-C0cScC2i.js";
 import { tokensFromTasks as k } from "./tasks/Task.js";
+import "../papaparse.min-C0cScC2i.js";
 import "../ops/cpu/matMulGelu.js";
 import "../matMulGelu-JNLZqKQp.js";
 import "../ops/grads/matMulGelu.js";

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@genai-fi/nanogpt",
-    "version": "0.15.13",
+    "version": "0.16.1",
     "type": "module",
     "main": "dist/main.js",
     "types": "dist/main.d.ts",