@genai-fi/nanogpt 0.4.3 → 0.4.5

package/dist/Generator.js

@@ -1,7 +1,7 @@
 import { E as u } from "./index-Dwqa6Zy2.js";
-import "./index-C4JCoBvj.js";
-import { t as d } from "./tensor2d-DqFGNs_K.js";
-import { c as p } from "./concat-CuRsVY-K.js";
+import "./index--6vO-cOz.js";
+import { t as d } from "./tensor2d-DUr_htjt.js";
+import { c as p } from "./concat-DvWM7HGZ.js";
 class w extends u {
   constructor(s, e) {
     super(), this.model = s, this.tokeniser = e;

package/dist/NanoGPTModel.js

@@ -1,17 +1,17 @@
 import { defaultConfig as x } from "./config.js";
 import W from "./layers/TransformerBlock.js";
-import { E as F, D as P, T as q, r as T, p as D } from "./TiedEmbedding-CnJ1bx4q.js";
+import { E as F, D as P, T as q, r as T, p as D } from "./TiedEmbedding-DznFwzcB.js";
 import K from "./layers/RoPECache.js";
 import N from "./layers/RMSNorm.js";
 import { estimateParameterCount as R } from "./utilities/parameters.js";
 import { createSoftmaxCrossEntropyWithGrad as A } from "./training/sparseCrossEntropy.js";
 import B from "./layers/BaseLayer.js";
-import { o as $, h as E, p as G, E as v, a6 as O, a7 as j, a8 as Q, t as w, a5 as V, f as C } from "./index-C4JCoBvj.js";
-import { r as _ } from "./reshape-Boe4DuIO.js";
-import { r as X } from "./range-9AzeApCc.js";
-import { e as H } from "./tfjs_backend-Cug-PH75.js";
-import { g as J } from "./gather-ZYRWhmXR.js";
-import { s as U } from "./softmax-Cujsg4ay.js";
+import { o as $, h as E, p as G, E as v, a9 as O, aa as j, ab as Q, t as w, a8 as V, f as C } from "./index--6vO-cOz.js";
+import { r as _ } from "./reshape-z51Eu-re.js";
+import { r as X } from "./range-C_vpUjBu.js";
+import { e as H } from "./tfjs_backend-DuKis_xG.js";
+import { g as J } from "./gather-C5D8PxwA.js";
+import { s as U } from "./softmax-Dsxflvdl.js";
 /**
  * @license
  * Copyright 2020 Google LLC. All Rights Reserved.
@@ -132,7 +132,7 @@ class wt extends B {
     }) : (this.ropeCache = new K(this.config.gpt), this.config.layerConfig.ropeCache = this.ropeCache), this.drop = st({ rate: this.config.gpt.dropout }), this.blocks = [];
     for (let e = 0; e < this.config.gpt.nLayer; e++)
       this.blocks.push(new W(e, this.config));
-    this.lnF = new N(this.config, 1e-8, "final_rms_norm");
+    this.lnF = new N(this.config, "final_rms_norm");
   }
   get checkpointing() {
     return this.config.layerConfig.checkpointAttention === !0 || this.config.layerConfig.checkpointMLP === !0;

package/dist/Reshape-CiAY8ltP.js

@@ -0,0 +1,212 @@
+import { ac as f, ad as g, n as p, ae as C, j as x } from "./index--6vO-cOz.js";
+import { u as I } from "./gpgpu_math-CUzjlO9A.js";
+/**
+ * @license
+ * Copyright 2018 Google LLC. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================================
+ */
+function R(t, e, o = "index") {
+  const s = f(e);
+  return s.map((n, r) => {
+    const i = `int ${t[r]} = ${o} / ${n}`, u = r === s.length - 1 ? `int ${t[r + 1]} = ${o} - ${t[r]} * ${n}` : `index -= ${t[r]} * ${n}`;
+    return `${i}; ${u};`;
+  }).join("");
+}
+function y(t, e) {
+  const o = t.length, s = t.map((r) => `${e}[${r}]`), n = new Array(o - 1);
+  n[o - 2] = s[o - 1];
+  for (let r = o - 3; r >= 0; --r)
+    n[r] = `(${n[r + 1]} * ${s[r + 1]})`;
+  return n;
+}
+function S(t, e, o = "index") {
+  const s = t.map((r, i) => i), n = y(s, e);
+  return n.map((r, i) => {
+    const u = `int ${t[i]} = ${o} / ${n[i]}`, a = i === n.length - 1 ? `int ${t[i + 1]} = ${o} - ${t[i]} * ${n[i]}` : `index -= ${t[i]} * ${n[i]}`;
+    return `${u}; ${a};`;
+  }).join("");
+}
+function F(t) {
+  const e = f(t).map((o) => o.toString());
+  return `
+  int getFlatIndex(ivec3 coords) {
+    return coords.x * ${e[0]} + coords.y * ${e[1]} + coords.z;
+  }
+`;
+}
+function v() {
+  return `
+  int getFlatIndex(ivec3 coords) {
+    return coords.x * outShapeStrides[0] + coords.y * outShapeStrides[1] + coords.z;
+  }
+`;
+}
+/**
+ * @license
+ * Copyright 2017 Google LLC. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================================
+ */
+function h(t, e = 2) {
+  return p(t.slice(0, t.length - e));
+}
+function m(t) {
+  if (t.length === 0)
+    throw Error("Cannot get rows and columns of an empty shape array.");
+  return [
+    t.length > 1 ? t[t.length - 2] : 1,
+    t[t.length - 1]
+  ];
+}
+function d(t) {
+  return t % 2 === 0;
+}
+function $(t, e) {
+  if (t = t.slice(-2), e = e.slice(-2), g(t, e) || !t.length || !e.length || t[0] === 0 || t[1] === 0 || e[0] === 0 || e[1] === 0)
+    return !0;
+  if (t.length !== e.length) {
+    const o = t[t.length - 1], s = e[e.length - 1];
+    if (o === s || d(o) && d(s) && (t[0] === 1 || e[0] === 1))
+      return !0;
+  }
+  return t[1] === e[1] && d(t[0]) && d(e[0]);
+}
+/**
+ * @license
+ * Copyright 2018 Google LLC. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================================
+ */
+class b {
+  constructor(e, o) {
+    this.variableNames = ["A"], this.packedInputs = !0, this.packedOutput = !0, this.customUniforms = [{ name: "inputShape", type: "ivec3" }], this.outputShape = e, this.enableShapeUniforms = I(this.outputShape.length);
+    let s = "";
+    for (let n = 0; n < 4; n++) {
+      let r = "thisRC = rc;";
+      n % 2 === 1 && (r += "thisRC.z += 1;"), n > 1 && (r += "thisRC.y += 1;"), s += `
+        ${r}
+        ${n > 0 ? "if(thisRC.y < rows && thisRC.z < cols){" : ""}
+          int flatIndex = getFlatIndex(thisRC);
+
+          ivec3 inputRC = inputCoordsFromReshapedOutCoords(flatIndex);
+          vec2 inputRCInnerDims = vec2(float(inputRC.y),float(inputRC.z));
+
+          result[${n}] =
+            getChannel(getA(inputRC.x, inputRC.y, inputRC.z), inputRCInnerDims);
+        ${n > 0 ? "}" : ""}
+      `;
+    }
+    this.userCode = `
+      ${w(o, this.enableShapeUniforms)}
+      ${this.enableShapeUniforms ? v() : F(e)}
+
+      void main() {
+        ivec3 rc = getOutputCoords();
+
+        vec4 result = vec4(0.);
+
+        ivec3 thisRC;
+        int rows = ${this.enableShapeUniforms ? "outShape[1]" : e[1]};
+        int cols = ${this.enableShapeUniforms ? "outShape[2]" : e[2]};
+
+        ${s}
+
+        setOutput(result);
+      }
+    `;
+  }
+}
+function w(t, e) {
+  return `
+    ivec3 inputCoordsFromReshapedOutCoords(int index) {
+      ${e ? S(["r", "c", "d"], "inputShape") : R(["r", "c", "d"], t)}
+      return ivec3(r, c, d);
+    }
+  `;
+}
+/**
+ * @license
+ * Copyright 2020 Google LLC. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================================
+ */
+function D(t, e, o) {
+  const s = [
+    h(t.shape),
+    ...m(t.shape)
+  ], n = {
+    dtype: t.dtype,
+    shape: s,
+    dataId: t.dataId
+  }, r = [
+    h(e),
+    ...m(e)
+  ], i = new b(r, s), u = !0, a = [s], c = o.runWebGLProgram(i, [n], t.dtype, a, u);
+  return { dataId: c.dataId, shape: e, dtype: c.dtype };
+}
+/**
+ * @license
+ * Copyright 2020 Google LLC. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================================
+ */
+function k(t) {
+  const { inputs: e, backend: o, attrs: s } = t, { x: n } = e, { shape: r } = s, i = o, u = p(n.shape), a = C(r, u), c = p(a);
+  x(u === c, () => `The new shape (${a}) has ${c} elements and the old shape (${n.shape}) has ${u} elements. The new shape and old shape must have the same number of elements.`);
+  const l = i.texData.get(n.dataId);
+  return l.isPacked && !$(n.shape, a) && !(l.texture !== null && $(l.shape, a)) ? D(n, a, i) : (i.incRef(n.dataId), { dataId: n.dataId, shape: a, dtype: n.dtype });
+}
+export {
+  k as r
+};

package/dist/TeachableLLM.js

@@ -3,15 +3,15 @@ import l from "./NanoGPTModel.js";
 import { saveModel as d } from "./utilities/save.js";
 import { loadModel as f } from "./utilities/load.js";
 import u from "./Generator.js";
-import _ from "./Trainer.js";
-import { E as p } from "./index-Dwqa6Zy2.js";
+import p from "./Trainer.js";
+import { E as _ } from "./index-Dwqa6Zy2.js";
 import { dummyPassAsync as m } from "./utilities/dummy.js";
 import c from "./tokeniser/CharTokeniser.js";
 import g from "./tokeniser/bpe.js";
 import "./papaparse.min-C8l2Kvo1.js";
 import "./index-Tf7vU29b.js";
 import "./jszip.min-CjP2V1VV.js";
-import "./index-C4JCoBvj.js";
+import "./index--6vO-cOz.js";
 import "./ops/cpu/scatterSub.js";
 import "./ops/webgl/scatterSub.js";
 import "./ops/cpu/gatherSub.js";
@@ -31,9 +31,18 @@ import "./ops/webgl/appendCache.js";
 import "./ops/cpu/fusedSoftmax.js";
 import "./ops/webgl/fusedSoftmax.js";
 import "./ops/grads/fusedSoftmax.js";
+import "./ops/cpu/matMulGelu.js";
+import "./ops/webgl/matMulGelu.js";
+import "./ops/grads/matMulGelu.js";
+import "./ops/cpu/gelu.js";
+import "./ops/webgl/gelu.js";
+import "./ops/grads/gelu.js";
+import "./ops/cpu/normRMS.js";
+import "./ops/webgl/normRMS.js";
+import "./ops/grads/normRMS.js";
 import w from "./utilities/profile.js";
 class a {
-  ee = new p();
+  ee = new _();
   _config;
   _model;
   _tokeniser;
@@ -120,7 +129,7 @@ class a {
   trainer() {
     if (!this._model || !this._tokeniser)
       throw new Error("Model or tokeniser is not initialized.");
-    const t = new _(this._model, this._tokeniser);
+    const t = new p(this._model, this._tokeniser);
     return t.on("start", () => this.setStatus("training")), t.on("stop", () => this.setStatus("ready")), t.on("log", async (e) => {
       const i = this.ee.listeners("trainStep");
       for (const o of i)