@genai-fi/nanogpt 0.1.5 → 0.1.6

package/dist/training/DatasetBuilder.d.ts

@@ -5,7 +5,7 @@ export declare class DatasetBuilder {
     blockSize: number;
     private tf;
     constructor(tf: typeof TF, tokenizer: ITokeniser, blockSize?: number);
-    createTextDataset(textData: string[], batchSize?: number): Promise<TF.data.Dataset<{
+    createTextDataset(textData: string[], batchSize?: number, start?: number, end?: number): Promise<TF.data.Dataset<{
         xs: TF.Tensor;
         ys: TF.Tensor;
     }>>;

package/dist/training/DatasetBuilder.js

@@ -1,19 +1,22 @@
-class l {
+class z {
   tokenizer;
   blockSize;
   tf;
-  constructor(s, i, o = 128) {
-    this.tokenizer = i, this.blockSize = o, this.tf = s;
+  constructor(s, o, i = 128) {
+    this.tokenizer = o, this.blockSize = i, this.tf = s;
   }
   // Create dataset from text files
-  async createTextDataset(s, i = 32) {
-    const o = await Promise.all(s.map((t) => this.tokenizer.encode(t))), a = this.tokenizer.eosToken >= 0, n = o.map((t) => a ? [...t, this.tokenizer.eosToken] : t).flat(), c = (function* () {
+  async createTextDataset(s, o = 32, i = 0, c = 1) {
+    const n = await Promise.all(s.map((t) => this.tokenizer.encode(t))), h = this.tokenizer.eosToken >= 0, a = n.map((t) => h ? [...t, this.tokenizer.eosToken] : t).flat().slice(
+      Math.floor(i * n.length),
+      c === 1 ? void 0 : Math.floor(c * n.length)
+    ), r = (function* () {
       for (; ; ) {
-        const t = Math.floor(Math.random() * (n.length - this.blockSize - 1)), e = n.slice(t, t + this.blockSize), r = n.slice(t + 1, t + this.blockSize + 1);
-        yield { xs: e, ys: r };
+        const t = Math.floor(Math.random() * (a.length - this.blockSize - 1)), e = a.slice(t, t + this.blockSize), l = a.slice(t + 1, t + this.blockSize + 1);
+        yield { xs: e, ys: l };
       }
     }).bind(this);
-    return this.tf.data.generator(c).batch(i).map((t) => {
+    return this.tf.data.generator(r).batch(o).map((t) => {
       const e = t;
       return this.tf.tidy(() => ({
         xs: e.xs.cast("int32"),
@@ -23,5 +26,5 @@ class l {
   }
 }
 export {
-  l as DatasetBuilder
+  z as DatasetBuilder
 };

package/dist/training/Trainer.js

@@ -1,8 +1,8 @@
 import { DatasetBuilder as d } from "./DatasetBuilder.js";
 import p from "./AdamExt.js";
-class g {
-  constructor(t, e, s, i = 1e-3) {
-    this.tokenizer = s, this.tf = t, this.model = e, this.learningRate = i, this.resetOptimizer(), this.datasetBuilder = new d(this.tf, s, e.config.blockSize);
+class u {
+  constructor(t, s, e, i = 1e-3) {
+    this.tokenizer = e, this.tf = t, this.model = s, this.learningRate = i, this.resetOptimizer(), this.datasetBuilder = new d(this.tf, e, s.config.blockSize);
   }
   model;
   optimizer;
@@ -21,7 +21,7 @@ class g {
   }
   resetOptimizer(t = { learningRateFactor: 1, beta1: 0.9, beta2: 0.99, epsilon: 1e-8 }) {
     this.optimizer && this.optimizer.dispose();
-    const e = new p(
+    const s = new p(
       t.learningRateFactor * this.learningRate,
       t.beta1,
       t.beta2,
@@ -33,53 +33,58 @@ class g {
         weightDecay: 0
       }
     );
-    this.optimizer = e;
+    this.optimizer = s;
   }
   printGradients(t) {
-    Object.keys(t).forEach((e) => {
-      const s = t[e];
-      console.log(`${e}:`), console.log(`  Shape: ${s.shape}`), console.log(`  Mean: ${this.tf.mean(s).dataSync()[0]}`), console.log(`  Std: ${this.tf.moments(s).variance.sqrt().dataSync()[0]}`), console.log(`  Min: ${this.tf.min(s).dataSync()[0]}`), console.log(`  Max: ${this.tf.max(s).dataSync()[0]}`), console.log(`  Norm: ${this.tf.norm(s).dataSync()[0]}`);
+    Object.keys(t).forEach((s) => {
+      const e = t[s];
+      console.log(`${s}:`), console.log(`  Shape: ${e.shape}`), console.log(`  Mean: ${this.tf.mean(e).dataSync()[0]}`), console.log(`  Std: ${this.tf.moments(e).variance.sqrt().dataSync()[0]}`), console.log(`  Min: ${this.tf.min(e).dataSync()[0]}`), console.log(`  Max: ${this.tf.max(e).dataSync()[0]}`), console.log(`  Norm: ${this.tf.norm(e).dataSync()[0]}`);
     });
   }
-  trainStep(t, e = !1, s = !1) {
+  trainStep(t, s = !1, e = !1) {
     return this.tf.tidy(() => {
-      const { xs: i, ys: r } = t, o = () => {
-        const { loss: l, logits: c } = this.model.forward(i, r, !0);
+      const { xs: i, ys: a } = t, o = () => {
+        const { loss: l, logits: c } = this.model.forward(i, a, !0);
         return c.dispose(), l;
-      }, { value: n, grads: a } = this.tf.variableGrads(o);
-      return e || (s && (console.log("-------"), this.printGradients(a), console.log("-------")), this.optimizer.applyGradients(a), this.tf.dispose(a)), n;
+      }, { value: n, grads: r } = this.tf.variableGrads(o);
+      return s || (e && (console.log("-------"), this.printGradients(r), console.log("-------")), this.optimizer.applyGradients(r), this.tf.dispose(r)), n;
     });
   }
   dummyPass() {
-    const t = this.tf.zeros([1, this.model.config.blockSize], "int32"), e = this.tf.zeros([1, this.model.config.blockSize, this.model.config.vocabSize]);
+    const t = this.tf.zeros([1, this.model.config.blockSize], "int32"), s = this.tf.zeros([1, this.model.config.blockSize, this.model.config.vocabSize]);
     try {
-      const s = this.trainStep({ xs: t, ys: e }, !0);
-      s.dataSync(), s.dispose();
-    } catch (s) {
-      console.error("Error during dummy pass:", s);
+      const e = this.trainStep({ xs: t, ys: s }, !0);
+      e.dataSync(), e.dispose();
+    } catch (e) {
+      console.error("Error during dummy pass:", e);
     } finally {
-      t.dispose(), e.dispose();
+      t.dispose(), s.dispose();
     }
   }
-  async trainBatch(t, e) {
+  async trainBatch(t, s) {
     try {
-      const s = this.trainStep(e, !1, !1);
-      return e.xs.dispose(), e.ys.dispose(), t.step++, t.totalSteps++, s.array().then((i) => (t.lastLoss = i, t.losses.push(t.lastLoss), s.dispose(), t.lastLoss));
-    } catch (s) {
-      throw console.error(`Error processing batch at step ${t.step}:`, s), this.tf.dispose(), s;
+      const e = this.trainStep(s, !1, !1);
+      return s.xs.dispose(), s.ys.dispose(), t.step++, t.totalSteps++, e.array().then((i) => (t.lastLoss = i, t.losses.push(t.lastLoss), e.dispose(), t.lastLoss));
+    } catch (e) {
+      throw console.error(`Error processing batch at step ${t.step}:`, e), this.tf.dispose(), e;
     }
   }
-  async createTrainValidationSplit(t, e = 32, s = 0.1) {
-    const i = Math.floor(t.length * (1 - s)), r = t.slice(0, i), o = t.slice(i), n = await this.datasetBuilder.createTextDataset(r, e), a = await this.datasetBuilder.createTextDataset(o, e);
-    return { trainDataset: n, validationDataset: a };
+  async createTrainValidationSplit(t, s = 32, e = 0.1) {
+    const i = await this.datasetBuilder.createTextDataset(t, s, 0, 1 - e), a = await this.datasetBuilder.createTextDataset(
+      t,
+      s,
+      1 - e,
+      1
+    );
+    return { trainDataset: i, validationDataset: a };
   }
-  async createDataset(t, e = 32) {
-    return await this.datasetBuilder.createTextDataset(t, e);
+  async createDataset(t, s = 32) {
+    return await this.datasetBuilder.createTextDataset(t, s);
   }
   dispose() {
     this.optimizer && this.optimizer.dispose();
   }
 }
 export {
-  g as default
+  u as default
 };

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@genai-fi/nanogpt",
-    "version": "0.1.5",
+    "version": "0.1.6",
     "type": "module",
     "main": "dist/main.js",
     "types": "dist/main.d.ts",