@genai-fi/nanogpt 0.2.4 → 0.2.6

package/dist/NanoGPTModel.js

@@ -1,9 +1,11 @@
-import { defaultConfig as z } from "./config.js";
-import $ from "./layers/TransformerBlock.js";
+import { defaultConfig as $ } from "./config.js";
+import z from "./layers/TransformerBlock.js";
 import S from "./layers/TiedEmbedding.js";
 import I from "./layers/RoPECache.js";
 import _ from "./layers/RMSNorm.js";
-class x {
+import { estimateParameterCount as W } from "./utilities/parameters.js";
+import { createSoftmaxCrossEntropyWithGrad as C } from "./training/sparseCrossEntropy.js";
+class K {
   config;
   wte;
   // Token embeddings
@@ -19,7 +21,7 @@ class x {
   log = [];
   // Training log
   constructor(t, e = {}) {
-    this.tf = t, this.config = { ...z, ...e }, this.wte = new S(t, {
+    this.tf = t, this.config = { ...$, ...e }, this.wte = new S(t, {
       vocabSize: this.config.vocabSize,
       embedDim: this.config.nEmbed,
       name: "token_embedding"
@@ -30,7 +32,7 @@ class x {
       embeddingsInitializer: this.tf.initializers.randomNormal({ mean: 0, stddev: 0.02 })
     }) : this.ropeCache = new I(t, this.config), this.drop = this.tf.layers.dropout({ rate: this.config.dropout }), this.blocks = [];
     for (let o = 0; o < this.config.nLayer; o++)
-      this.blocks.push(new $(this.tf, o, this.config, this.ropeCache));
+      this.blocks.push(new z(this.tf, o, this.config, this.ropeCache));
     this.lnF = new _(t, [this.config.nEmbed], 1e-8, "final_rms_norm");
   }
   get variables() {
@@ -58,9 +60,9 @@ class x {
     return this.tf.tidy(() => {
       const i = this.wte.embed(t);
       if (this.config.useRope === !1) {
-        const [, s] = t.shape, r = this.config.blockSize, l = this.tf.range(0, s, 1, "int32"), n = this.tf.mod(
-          this.tf.add(l, this.tf.scalar(e, "int32")),
-          this.tf.scalar(r, "int32")
+        const [, s] = t.shape, l = this.config.blockSize, r = this.tf.range(0, s, 1, "int32"), n = this.tf.mod(
+          this.tf.add(r, this.tf.scalar(e, "int32")),
+          this.tf.scalar(l, "int32")
         ), h = this.wpe.apply(n), c = i.add(h);
         return this.drop.apply(c, { training: o });
       } else
@@ -94,7 +96,7 @@ class x {
   }
   calculateLoss(t, e) {
     try {
-      return this.tf.losses.softmaxCrossEntropy(e, t, this.tf.Reduction.MEAN);
+      return C()(t, e).mean();
     } catch (o) {
       throw console.error("Error computing loss:", o), new Error(`Loss computation failed: ${o}`);
     }
@@ -107,27 +109,27 @@ class x {
         throw new Error("No attentions for rollout");
       const [e, o, i] = t[0].shape;
       for (const s of t) {
-        const [r, l, n] = s.shape;
-        if (r !== e || l !== o || n !== i)
+        const [l, r, n] = s.shape;
+        if (l !== e || r !== o || n !== i)
           throw new Error(
-            `Inconsistent attention shapes in rollout: expected [${e},${o},${i}] got [${r},${l},${n}]`
+            `Inconsistent attention shapes in rollout: expected [${e},${o},${i}] got [${l},${r},${n}]`
           );
       }
       if (o === i) {
         const s = this.tf.eye(i, i).expandDims(0);
-        let r = s.tile([e, 1, 1]);
-        for (const l of t) {
-          const n = l.add(s);
-          r = n.div(n.sum(-1, !0)).matMul(r);
+        let l = s.tile([e, 1, 1]);
+        for (const r of t) {
+          const n = r.add(s);
+          l = n.div(n.sum(-1, !0)).matMul(l);
         }
-        return r;
+        return l;
       }
       if (o === 1) {
         let s = null;
-        const r = this.tf.tensor1d([i - 1], "int32"), l = this.tf.oneHot(r, i).reshape([1, 1, i]).tile([e, 1, 1]);
-        r.dispose();
+        const l = this.tf.tensor1d([i - 1], "int32"), r = this.tf.oneHot(l, i).reshape([1, 1, i]).tile([e, 1, 1]);
+        l.dispose();
         for (const n of t) {
-          let h = n.add(l);
+          let h = n.add(r);
           h = h.div(h.sum(-1, !0)), s == null ? s = h : (s = s.mul(h), s = s.div(s.sum(-1, !0)));
         }
         return s;
@@ -137,56 +139,53 @@ class x {
   }
   forward(t, e, o = !1, i = !1, s) {
     return this.validateInput(t), this.tf.tidy(() => {
-      const r = s?.[0]?.length ?? 0;
-      let l = this.inputPhase(t, r, o);
+      const l = s?.[0]?.length ?? 0;
+      let r = this.inputPhase(t, l, o);
       const n = [];
       if (s && s.length !== this.blocks.length)
         throw console.error("Cache", s), new Error(`Cache length ${s.length} does not match number of blocks ${this.blocks.length}`);
       for (let a = 0; a < this.blocks.length; a++) {
         const d = this.blocks[a], {
           output: g,
-          attention: m,
-          cache: p
-        } = d.call(l, o, i, s ? s[a] : void 0);
-        l = g, i && m && n.push(m), s && p ? (s[a]?.k.dispose(), s[a]?.v.dispose(), s[a] = p) : p && (p.k.dispose(), p.v.dispose());
+          attention: u,
+          cache: f
+        } = d.call(r, o, i, s ? s[a] : void 0);
+        r = g, i && u && n.push(u), s && f ? (s[a]?.k.dispose(), s[a]?.v.dispose(), s[a] = f) : f && (f.k.dispose(), f.v.dispose());
       }
       let h;
-      i && n.length > 0 && (h = this.computeAttentionRollout(n)), l = this.lnF.apply(l);
-      const c = this.wte.project(l);
-      let f;
-      return e && (f = this.calculateLoss(c, e)), { logits: c, loss: f, attention: i ? h : void 0 };
+      i && n.length > 0 && (h = this.computeAttentionRollout(n)), r = this.lnF.apply(r);
+      const c = this.wte.project(r);
+      let p;
+      return e && (p = this.calculateLoss(c, e)), { logits: c, loss: p, attention: i ? h : void 0 };
     });
   }
   generate(t, e, o) {
-    const i = o?.temperature ?? 1, s = o?.topK, r = o?.usePadding ?? !1, l = o?.includeAttention ?? !1;
+    const i = o?.temperature ?? 1, s = o?.topK, l = o?.usePadding ?? !1, r = o?.includeAttention ?? !1;
     return this.tf.tidy(() => {
       const n = t, h = n.shape[1], c = h <= this.config.blockSize ? n : n.slice(
         [0, h - this.config.blockSize],
         [n.shape[0], this.config.blockSize]
-      ), f = r ? this.config.blockSize - c.shape[1] : 0, a = f > 0 ? this.tf.pad(c, [
+      ), p = l ? this.config.blockSize - c.shape[1] : 0, a = p > 0 ? this.tf.pad(c, [
         [0, 0],
-        [0, f]
-      ]) : c, { logits: d, attention: g } = this.forward(a, void 0, !1, l, e), m = d.shape[1] - 1 - f, p = d.slice([0, m, 0], [d.shape[0], 1, d.shape[2]]), w = g ? g.slice([0, m, 0], [g.shape[0], 1, g.shape[2]]) : void 0, b = p.div(i);
-      let u;
+        [0, p]
+      ]) : c, { logits: d, attention: g } = this.forward(a, void 0, !1, r, e), u = d.shape[1] - 1 - p, f = d.slice([0, u, 0], [d.shape[0], 1, d.shape[2]]), w = g ? g.slice([0, u, 0], [g.shape[0], 1, g.shape[2]]) : void 0, b = f.div(i);
+      let m;
       if (s) {
-        const { values: E, indices: v } = this.tf.topk(b, s), y = this.tf.multinomial(E.squeeze([1]), 1);
-        u = this.tf.gather(v.squeeze([1]), y, 1);
+        const { values: v, indices: y } = this.tf.topk(b, s), E = this.tf.multinomial(v.squeeze([1]), 1);
+        m = this.tf.gather(y.squeeze([1]), E, 1);
       } else
-        u = this.tf.multinomial(b.squeeze([1]), 1);
+        m = this.tf.multinomial(b.squeeze([1]), 1);
       let k;
-      return o?.includeProbabilities && (k = this.tf.softmax(b.squeeze([1]))), u = u.reshape([1, 1]), { output: u, attention: w?.squeeze([1]), probabilities: k };
+      return o?.includeProbabilities && (k = this.tf.softmax(b.squeeze([1]))), m = m.reshape([1, 1]), { output: m, attention: w?.squeeze([1]), probabilities: k };
     });
   }
   getNumParams() {
-    const t = this.config.vocabSize * this.config.nEmbed, e = this.config.nLayer * (4 * this.config.nEmbed * this.config.nEmbed + // qkv + proj
-    2 * this.config.nEmbed), o = this.config.nLayer * (this.config.mlpFactor * this.config.nEmbed * this.config.nEmbed + // fc
-    this.config.nEmbed * this.config.mlpFactor * this.config.nEmbed), i = this.config.nEmbed;
-    return t + e + o + i;
+    return W(this.config);
   }
   dispose() {
     this.wte.dispose(), this.wpe && this.wpe.dispose(), this.drop.dispose(), this.blocks.forEach((t) => t.dispose()), this.lnF.dispose();
   }
 }
 export {
-  x as default
+  K as default
 };

package/dist/TeachableLLM.js

@@ -9,6 +9,9 @@ import { dummyPassAsync as h } from "./utilities/dummy.js";
 import g from "./tokeniser/CharTokeniser.js";
 import "./papaparse.min-C8l2Kvo1.js";
 import "./index-Tf7vU29b.js";
+import "./jszip.min-CjP2V1VV.js";
+import "./ops/scatterSub.js";
+import "./ops/gatherSub.js";
 class a extends c {
   _config;
   _model;

package/dist/complex-D6Bq1XDf.js

@@ -0,0 +1,27 @@
+import { o as t, c as s, b as n, E as m, C as r } from "./index-D1SlunD-.js";
+/**
+ * @license
+ * Copyright 2020 Google LLC. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================================
+ */
+function l(o, c) {
+  const a = s(o, "real", "complex"), e = s(c, "imag", "complex");
+  n(a.shape, e.shape, `real and imag shapes, ${a.shape} and ${e.shape}, must match in call to tf.complex().`);
+  const p = { real: a, imag: e };
+  return m.runKernel(r, p);
+}
+const i = /* @__PURE__ */ t({ complex_: l });
+export {
+  i as c
+};

package/dist/data/docx.d.ts

@@ -0,0 +1 @@
+export declare function loadDOCX(file: Blob | Uint8Array): Promise<string[]>;

package/dist/data/docx.js

@@ -0,0 +1,15 @@
+import { j as a } from "../jszip.min-CjP2V1VV.js";
+async function c(n) {
+  const t = await (await a.loadAsync(n)).file("word/document.xml")?.async("string");
+  if (!t) throw new Error("Failed to load document.xml");
+  return s(t).split(`
+`).filter((r) => r.trim().length > 10);
+}
+function s(n) {
+  const t = new DOMParser().parseFromString(n, "application/xml");
+  return Array.from(t.getElementsByTagName("w:t")).map((r) => r.textContent).join(`
+`);
+}
+export {
+  c as loadDOCX
+};

package/dist/data/pdf.d.ts

@@ -1 +1 @@
-export declare function loadPDF(file: File, maxSize?: number): Promise<string[]>;
+export declare function loadPDF(file: Blob | Uint8Array, maxSize?: number): Promise<string[]>;