@genai-fi/nanogpt 0.5.1 → 0.5.3

package/dist/training/FullTrainer.js

@@ -1,7 +1,7 @@
 import { generateText as v } from "../utilities/generate.js";
 import L from "./Trainer.js";
 import x from "./Evaluator.js";
-import { a as h } from "../index-iNhkcAEQ.js";
+import { a as h } from "../index-CnHyhpKc.js";
 const D = {
   desiredLoss: 0.01,
   logInterval: 1,

package/dist/training/Trainer.js

@@ -1,13 +1,13 @@
-import { DatasetBuilder as d } from "./DatasetBuilder.js";
-import h from "./AdamExt.js";
-import { t as g, v as u, a as o } from "../index-iNhkcAEQ.js";
-import { m as y, n as f } from "../norm-D3676xIo.js";
-import { m as S, a as z } from "../moments-B06NlR_V.js";
-import { m as b } from "../max-CYaAjEEp.js";
-import { z as n } from "../zeros-NMYTayy7.js";
+import { DatasetBuilder as h, flattenTokens as d, PAGE_FACTOR as g } from "./DatasetBuilder.js";
+import u from "./AdamExt.js";
+import { t as f, v as y, a as m } from "../index-CnHyhpKc.js";
+import { m as S, n as z } from "../norm-BpWsOapl.js";
+import { m as w, a as T } from "../moments-DLTE6-1p.js";
+import { m as x } from "../max-CcnEArWK.js";
+import { z as p } from "../zeros-CYMicyqz.js";
 class G {
   constructor(t, s, e = 1e-3) {
-    this.tokenizer = s, this.model = t, this.learningRate = e, this.resetOptimizer(), this.datasetBuilder = new d(s, t.config.gpt.blockSize);
+    this.tokenizer = s, this.model = t, this.learningRate = e, this.resetOptimizer(), this.datasetBuilder = new h(s, t.config.gpt.blockSize);
   }
   model;
   optimizer;
@@ -29,7 +29,7 @@ class G {
   }
   resetOptimizer(t = { learningRateFactor: 1, beta1: 0.9, beta2: 0.99, epsilon: 1e-8 }) {
     this.optimizer && this.optimizer.dispose();
-    const s = new h(
+    const s = new u(
       t.learningRateFactor * this.learningRate,
       t.beta1,
       t.beta2,
@@ -46,21 +46,21 @@ class G {
   printGradients(t) {
     Object.keys(t).forEach((s) => {
       const e = t[s];
-      console.log(`${s}:`), console.log(`  Shape: ${e.shape}`), console.log(`  Mean: ${S(e).dataSync()[0]}`), console.log(`  Std: ${z(e).variance.sqrt().dataSync()[0]}`), console.log(`  Min: ${y(e).dataSync()[0]}`), console.log(`  Max: ${b(e).dataSync()[0]}`), console.log(`  Norm: ${f(e).dataSync()[0]}`);
+      console.log(`${s}:`), console.log(`  Shape: ${e.shape}`), console.log(`  Mean: ${w(e).dataSync()[0]}`), console.log(`  Std: ${T(e).variance.sqrt().dataSync()[0]}`), console.log(`  Min: ${S(e).dataSync()[0]}`), console.log(`  Max: ${x(e).dataSync()[0]}`), console.log(`  Norm: ${z(e).dataSync()[0]}`);
     });
   }
   trainStep(t, s = !1, e = !1) {
-    return g(() => {
+    return f(() => {
       this.model.getProfiler()?.startMemory();
-      const { xs: a, ys: r } = t, l = () => {
-        const [m, p] = this.model.forward({ training: !0 }, a, r);
-        return m.dispose(), p;
-      }, { value: c, grads: i } = u(l);
-      return s ? this.model.getProfiler()?.endMemory("Training") : (e && (console.log("-------"), this.printGradients(i), console.log("-------")), this.optimizer.applyGradients(i), this.model.getProfiler()?.endMemory("Training"), o(i)), c;
+      const { xs: a, ys: i } = t, o = () => {
+        const [l, c] = this.model.forward({ training: !0 }, a, i);
+        return l.dispose(), c;
+      }, { value: n, grads: r } = y(o);
+      return s ? this.model.getProfiler()?.endMemory("Training") : (e && (console.log("-------"), this.printGradients(r), console.log("-------")), this.optimizer.applyGradients(r), this.model.getProfiler()?.endMemory("Training"), m(r)), n;
     });
   }
   dummyPass() {
-    const t = n([1, this.model.config.gpt.blockSize], "int32"), s = n([1, this.model.config.gpt.blockSize], "int32");
+    const t = p([1, this.model.config.gpt.blockSize], "int32"), s = p([1, this.model.config.gpt.blockSize], "int32");
     try {
       const e = this.trainStep({ xs: t, ys: s }, !0);
       e.dataSync(), e.dispose();
@@ -75,20 +75,29 @@ class G {
       const e = this.trainStep(s, !1, !1);
       return s.xs.dispose(), s.ys.dispose(), t.step++, t.totalSteps++, e.array().then((a) => (t.lastLoss = a, t.losses.push(t.lastLoss), e.dispose(), t.lastLoss));
     } catch (e) {
-      throw console.error(`Error processing batch at step ${t.step}:`, e), o(), e;
+      throw console.error(`Error processing batch at step ${t.step}:`, e), m(), e;
     }
   }
   async createTrainValidationSplit(t, s = 32, e = 0.1) {
-    const a = await this.datasetBuilder.createTextDataset(t, s, 0, 1 - e), r = await this.datasetBuilder.createTextDataset(
-      t,
+    const a = await d(t, this.tokenizer), i = /* @__PURE__ */ new Set();
+    if (e > 0) {
+      const r = Math.floor(a.length / (this.datasetBuilder.blockSize * g)), l = Math.max(1, Math.floor(r * e));
+      for (; i.size < l; ) {
+        const c = Math.floor(Math.random() * r);
+        i.add(c);
+      }
+    }
+    const o = await this.datasetBuilder.createTextDataset(a, s, i, !1), n = await this.datasetBuilder.createTextDataset(
+      a,
       s,
-      1 - e,
-      1
+      i,
+      !0
     );
-    return { trainDataset: a, validationDataset: r };
+    return { trainDataset: o, validationDataset: n };
   }
   async createDataset(t, s = 32) {
-    return await this.datasetBuilder.createTextDataset(t, s);
+    const e = await d(t, this.tokenizer);
+    return await this.datasetBuilder.createTextDataset(e, s);
   }
   dispose() {
     this.optimizer && this.optimizer.dispose();

package/dist/training/sparseCrossEntropy.js

@@ -1,9 +1,9 @@
 import { gatherSub as L } from "../ops/gatherSub.js";
 import { scatterSub as y } from "../ops/scatterSub.js";
-import { e as u, c as i, z as S, t as f, s as G } from "../index-iNhkcAEQ.js";
-import { s as v } from "../softmax-BjsptB07.js";
-import { m as z } from "../max-CYaAjEEp.js";
-import { l as k } from "../log_sum_exp-CkumwesB.js";
+import { e as u, c as i, z as S, t as f, s as G } from "../index-CnHyhpKc.js";
+import { s as v } from "../softmax-DX6qXAbm.js";
+import { m as z } from "../max-CcnEArWK.js";
+import { l as k } from "../log_sum_exp-CRH7Np9v.js";
 function F(a, s) {
   return f(() => {
     const e = a.shape[a.shape.length - 1], o = a.shape.slice(0, -1).reduce((d, c) => d * c, 1), p = a.shape.length > 2 ? a.reshape([o, e]) : a, n = s.shape.length > 1 ? s.reshape([o]).cast("int32") : s.cast("int32"), t = z(p, -1, !0), r = G(p, t), h = k(r, -1);

package/dist/utilities/dummy.js

@@ -1,5 +1,5 @@
-import "../index-iNhkcAEQ.js";
-import { z as n } from "../zeros-NMYTayy7.js";
+import "../index-CnHyhpKc.js";
+import { z as n } from "../zeros-CYMicyqz.js";
 async function c(s) {
   const i = n([1, s.config.gpt.blockSize], "int32"), [t, o] = s.forward({ training: !1 }, i);
   await t.data(), t.dispose(), o && o.dispose(), i.dispose();

package/dist/utilities/generate.js

@@ -1,6 +1,6 @@
-import { t as y } from "../index-iNhkcAEQ.js";
-import { t as x } from "../tensor2d-tSxWdFMH.js";
-import { c as f } from "../concat-Cxbo2sOz.js";
+import { t as y } from "../index-CnHyhpKc.js";
+import { t as x } from "../tensor2d-CqtBzOKq.js";
+import { c as f } from "../concat-BRRtq4S2.js";
 async function A(o, r, a, c, T) {
   if (c <= 0)
     throw new Error("Length must be a positive integer");

package/dist/utilities/load.js

@@ -3,7 +3,7 @@ import { importWeights as b } from "./weights.js";
 import u from "../tokeniser/CharTokeniser.js";
 import F from "../NanoGPTModel.js";
 import { dummyPassAsync as j } from "./dummy.js";
-import { d as T } from "../index-iNhkcAEQ.js";
+import { d as T } from "../index-CnHyhpKc.js";
 import E from "../tokeniser/bpe.js";
 async function A(t) {
   const o = await fetch(t);

package/dist/utilities/profile.js

@@ -1,4 +1,4 @@
-import { m as s } from "../index-iNhkcAEQ.js";
+import { m as s } from "../index-CnHyhpKc.js";
 const m = 1024 * 1024;
 class i {
   log = /* @__PURE__ */ new Map();

package/dist/utilities/tokenParse.js

@@ -1,12 +1,21 @@
-function h(o) {
-  const r = Array.from(o), t = [], l = new RegExp("(\\p{P}|\\p{S}|\\s)", "gu");
-  let e = "";
-  for (let n = 0; n < r.length; n++) {
-    const s = r[n];
-    s === " " ? (t.push(e), e = s) : s.match(l) ? (t.push(e), t.push(s), e = "") : e += s;
+function c(l) {
+  const r = Array.from(l), s = [], o = new RegExp("(\\p{P}|\\p{S}|\\s)", "gu");
+  let t = "";
+  for (let e = 0; e < r.length; e++) {
+    const n = r[e];
+    if (n === " ")
+      (r[e + 1] ?? "") !== " " ? (s.push(t), t = n) : t += n;
+    else if (n.match(o)) {
+      s.push(t);
+      let h = n;
+      for (; e + 1 < r.length && r[e + 1] === n; )
+        h += r[e + 1], e++;
+      s.push(h), t = "";
+    } else
+      t += n;
   }
-  return e.length > 0 && t.push(e), t.filter((n) => n.length > 0);
+  return t.length > 0 && s.push(t), s.filter((e) => e.length > 0);
 }
 export {
-  h as default
+  c as default
 };

package/dist/utilities/weights.js

@@ -1,5 +1,5 @@
-import "../index-iNhkcAEQ.js";
-import { t as p } from "../tensor-CfiPXsW4.js";
+import "../index-CnHyhpKc.js";
+import { t as p } from "../tensor-IZex6Bwp.js";
 function h(n) {
   const e = n.reduce((s, o) => s + o.length, 0), a = new Float32Array(e);
   let t = 0;

package/dist/variable-BGvK-VN3.js

@@ -0,0 +1,23 @@
+import { E as i } from "./index-CnHyhpKc.js";
+/**
+ * @license
+ * Copyright 2018 Google LLC. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================================
+ */
+function m(r, a = !0, e, t) {
+  return i.makeVariable(r, a, e, t);
+}
+export {
+  m as v
+};

package/dist/{zeros-NMYTayy7.js → zeros-CYMicyqz.js}

@@ -1,4 +1,4 @@
-import { o as l, i as r, X as m, E as c, Y as i, l as p, Z as u, p as f } from "./index-iNhkcAEQ.js";
+import { o as m, j as r, X as l, E as c, Y as i, n as p, Z as u, q as f } from "./index-CnHyhpKc.js";
 /**
  * @license
  * Copyright 2020 Google LLC. All Rights Reserved.
@@ -17,11 +17,11 @@ import { o as l, i as r, X as m, E as c, Y as i, l as p, Z as u, p as f } from "
  */
 function x(a, e) {
   const o = r(a, "real", "complex"), s = r(e, "imag", "complex");
-  m(o.shape, s.shape, `real and imag shapes, ${o.shape} and ${s.shape}, must match in call to tf.complex().`);
+  l(o.shape, s.shape, `real and imag shapes, ${o.shape} and ${s.shape}, must match in call to tf.complex().`);
   const n = { real: o, imag: s };
   return c.runKernel(i, n);
 }
-const g = /* @__PURE__ */ l({ complex_: x });
+const g = /* @__PURE__ */ m({ complex_: x });
 /**
  * @license
  * Copyright 2018 Google LLC. All Rights Reserved.

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@genai-fi/nanogpt",
-    "version": "0.5.1",
+    "version": "0.5.3",
     "type": "module",
     "main": "dist/main.js",
     "types": "dist/main.d.ts",

package/dist/BaseLayer-BhrMN8JO.js

@@ -1,135 +0,0 @@
-import { E as p, V as v, c as _, e as o, W as V } from "./index-iNhkcAEQ.js";
-/**
- * @license
- * Copyright 2018 Google LLC. All Rights Reserved.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- * =============================================================================
- */
-function E(l, t = !0, r, e) {
-  return p.makeVariable(l, t, r, e);
-}
-class m {
-  parent;
-  config;
-  _variables = /* @__PURE__ */ new Map();
-  _trainable = !0;
-  children = [];
-  constructor(t, r) {
-    this.config = t, this.parent = r, this.parent && this.parent.children.push(this);
-  }
-  getProfiler() {
-    return this.config.layerConfig.profiler;
-  }
-  startMemory() {
-    this.config.layerConfig.profiler?.startMemory();
-  }
-  endMemory(t) {
-    this.config.layerConfig.profiler?.endMemory(t);
-  }
-  addVariable(t, r) {
-    this._variables.set(t, r || null);
-  }
-  get variables() {
-    const t = Array.from(this._variables.values()).filter((e) => e !== null), r = this.children.flatMap((e) => e.variables);
-    return [...t, ...r];
-  }
-  get trainableVariables() {
-    const t = Array.from(this._variables.values()).filter(
-      (e) => e !== null && e.trainable
-    ), r = this.children.flatMap((e) => e.trainableVariables);
-    return [...t, ...r];
-  }
-  get trainable() {
-    return this._trainable;
-  }
-  set trainable(t) {
-    this._trainable = t, this._variables.forEach((r) => {
-      r && (r.trainable = t);
-    }), this.children.forEach((r) => {
-      r.trainable = t;
-    });
-  }
-  getVariable(t) {
-    const r = this._variables.get(t);
-    if (!r)
-      throw new Error(`Variable ${t} not found`);
-    return r;
-  }
-  hasVariable(t) {
-    return this._variables.get(t) !== null;
-  }
-  setVariable(t, r) {
-    if (!this._variables.has(t))
-      throw new Error(`Variable ${t} not found`);
-    this._variables.set(t, r);
-  }
-  saveWeights(t) {
-    this._variables.forEach((r, e) => {
-      r && t.set(e, [r.clone()]);
-    }), this.children.forEach((r) => {
-      r.saveWeights(t);
-    });
-  }
-  loadWeights(t) {
-    this._variables.forEach((r, e) => {
-      const i = t.get(e)?.[0];
-      if (!i)
-        throw new Error(`Weights for ${e} not found`);
-      r ? r.assign(i) : this._variables.set(e, E(i, this._trainable));
-    }), this.children.forEach((r) => {
-      r.loadWeights(t);
-    });
-  }
-  dispose() {
-    this._variables.forEach((t) => {
-      t?.dispose();
-    }), this._variables.clear();
-  }
-  build() {
-  }
-  dropout(t) {
-    return t;
-  }
-  call(t, ...r) {
-    this.build();
-    const e = this.forward(t, ...r);
-    if (t.training && e instanceof v) {
-      const i = this.dropout(e);
-      return i !== e && e.dispose(), i;
-    } else
-      return e;
-  }
-  callCheckpoint(t, ...r) {
-    return this.build(), this.checkpointingFn(t, ...r);
-  }
-  checkpointingFn(t, ...r) {
-    const e = this.trainableVariables, s = _((...a) => {
-      const h = a[a.length - 1], n = a.slice(0, r.length), c = this.forward(t, ...n);
-      return h(n), { value: c, gradFunc: (f, u) => {
-        const b = o().state.activeTape;
-        o().state.activeTape = [];
-        const d = V((...g) => this.forward(t, ...g.slice(0, n.length)))([...u, ...e], f);
-        return o().state.activeTape = b, d;
-      } };
-    })(...r, ...e);
-    if (t.training) {
-      const a = this.dropout(s);
-      return a !== s && s.dispose(), a;
-    } else
-      return s;
-  }
-}
-export {
-  m as B,
-  E as v
-};