@genai-fi/nanogpt 0.18.0 → 0.19.0

package/dist/{index-DSGwv2Yx.js → index-CUXkjxiT.js}

@@ -3226,34 +3226,34 @@ function vs() {
 vs();
 export {
   di as $,
-  Ja as A,
-  ur as B,
+  Kr as A,
+  K as B,
   yr as C,
-  So as D,
+  ur as D,
   f as E,
   Yr as F,
   Qr as G,
-  wo as H,
-  Gs as I,
-  Rt as J,
-  yo as K,
-  se as L,
+  Ja as H,
+  So as I,
+  wo as J,
+  Gs as K,
+  Rt as L,
   Na as M,
-  z as N,
+  yo as N,
   H as O,
   Oa as P,
-  Ue as Q,
+  se as Q,
   Xa as R,
   uo as S,
   Co as T,
-  Cs as U,
-  Tn as V,
-  vn as W,
-  qa as X,
-  ni as Y,
-  hs as Z,
+  z as U,
+  Ue as V,
+  Cs as W,
+  Tn as X,
+  vn as Y,
+  qa as Z,
   os as _,
-  Ho as a,
+  q as a,
   ea as a$,
   gi as a0,
   tt as a1,
@@ -3318,7 +3318,7 @@ export {
   $s as ax,
   xt as ay,
   he as az,
-  q as b,
+  ni as b,
   Qe as b$,
   na as b0,
   la as b1,
@@ -3383,7 +3383,7 @@ export {
   Fr as bx,
   Mr as by,
   xr as bz,
-  Jo as c,
+  Ho as c,
   en as c$,
   tn as c0,
   za as c1,
@@ -3448,7 +3448,7 @@ export {
   gs as cx,
   cr as cy,
   Tr as cz,
-  $ as d,
+  M as d,
   to as d0,
   Za as d1,
   Bo as d2,
@@ -3491,26 +3491,26 @@ export {
   ja as dx,
   ko as dy,
   Io as dz,
-  ei as e,
+  Jo as e,
   as as f,
   ai as g,
-  S as h,
-  nt as i,
-  V as j,
+  ei as h,
+  $ as i,
+  S as j,
   Fn as k,
-  M as l,
+  nt as l,
   b as m,
-  T as n,
+  V as n,
   N as o,
   Qt as p,
-  Ct as q,
+  T as q,
   ri as r,
   si as s,
   A as t,
-  Jn as u,
-  p as v,
-  An as w,
-  wr as x,
-  Kr as y,
-  K as z
+  Ct as u,
+  hs as v,
+  Jn as w,
+  p as x,
+  An as y,
+  wr as z
 };

package/dist/{index-BQvB7LCC.js → index-CjOWnMXP.js}

@@ -1,20 +1,20 @@
-import { W as Mt } from "./backend_webgpu-DI2wXEC2.js";
-import { ae as Et, ab as X, v as L, dc as Ut, dd as Ht, bX as Gt, N as D, _ as j, bb as Xt, ai as Ye, aU as Kt, a7 as qt, am as fe, bP as Yt, c7 as jt, c8 as Qt, bV as Zt, cO as Jt, ar as es, ac as De, ah as te, aW as ts, bm as ss, bn as os, bo as is, c9 as as, ca as rs, cb as ns, cc as us, cd as ds, ce as ls, aM as cs, aP as hs, bp as ps, cy as fs, cP as ms, cQ as gs, B as xs, S as Cs, br as ws, bd as ys, de as Ss, aQ as bs, aq as vs, bS as ks, bT as Is, af as Rs, bY as Ps, C as $s, cS as Ds, ao as Ns, x as zs, bt as As, cD as Fs, bu as Ws, cz as Ls, cT as Vs, cA as Bs, bv as Ts, bw as _s, bf as Os, bx as Ms, by as Es, cB as Us, cf as Hs, bz as Gs, cF as Xs, cG as Ks, df as qs, cg as Ys, cU as js, cV as Qs, dg as Zs, c0 as Js, K as eo, be as to, aG as so, cW as oo, bA as io, bB as ao, aN as ro, y as no, bZ as uo, cp as lo, bg as co, F as ho, b_ as po, dh as fo, a9 as at, bs as mo, cE as go, di as xo, al as Co, G as wo, a5 as ke, a$ as yo, b0 as So, cq as bo, ch as vo, ci as ko, cj as Io, aH as Ro, b1 as Po, b2 as $o, dj as Do, aO as No, b3 as zo, b4 as Ao, bD as Fo, cl as Wo, ck as Lo, cY as Vo, b$ as Bo, bE as To, cC as _o, cZ as Oo, c_ as Mo, dk as Eo, aZ as Uo, b5 as Ho, cm as Go, M as Xo, D as Ko, dl as qo, ap as Yo, bi as jo, bj as Qo, bF as Zo, d3 as Jo, bG as ei, P as ti, a6 as si, bH as oi, c$ as ii, aI as ai, c1 as ri, X as ni, aX as ui, cn as di, A as li, aJ as ci, b9 as hi, d0 as pi, ba as fi, d1 as mi, bJ as gi, bh as xi, b6 as Ci, bK as wi, ak as yi, dm as Si, aL as bi, bL as vi, aF as ki, co as Ii, bM as Ri, bN as Pi, bC as $i, bI as Di, dn as Ni, dp as zi, T as Ai, av as rt, dq as Fi, L as Wi, H as Li, c3 as Vi, d2 as Bi, b7 as Ti, aK as _i, cr as Oi, dr as Mi, c5 as Ei, cs as Ui, bq as Hi, ds as Gi, ct as Xi, bl as Ki, b8 as qi, bO as Yi, a as ji } from "./index-DSGwv2Yx.js";
-import { i as Qi, a as Zi, c as b, f as v, M as Y, b as nt, d as ut, e as dt } from "./webgpu_util-DuofJBMo.js";
+import { W as Mt } from "./backend_webgpu-BqASlsbV.js";
+import { ae as Et, ab as X, x as L, dc as Ut, dd as Ht, bX as Gt, U as D, _ as j, bb as Xt, ai as Ye, aU as Kt, a7 as qt, am as fe, bP as Yt, c7 as jt, c8 as Qt, bV as Zt, cO as Jt, ar as es, ac as De, ah as te, aW as ts, bm as ss, bn as os, bo as is, c9 as as, ca as rs, cb as ns, cc as us, cd as ds, ce as ls, aM as cs, aP as hs, bp as ps, cy as fs, cP as ms, cQ as gs, D as xs, S as Cs, br as ws, bd as ys, de as Ss, aQ as bs, aq as vs, bS as ks, bT as Is, af as Rs, bY as Ps, C as $s, cS as Ds, ao as Ns, z as zs, bt as As, cD as Fs, bu as Ws, cz as Ls, cT as Vs, cA as Bs, bv as Ts, bw as _s, bf as Os, bx as Ms, by as Es, cB as Us, cf as Hs, bz as Gs, cF as Xs, cG as Ks, df as qs, cg as Ys, cU as js, cV as Qs, dg as Zs, c0 as Js, N as eo, be as to, aG as so, cW as oo, bA as io, bB as ao, aN as ro, A as no, bZ as uo, cp as lo, bg as co, F as ho, b_ as po, dh as fo, a9 as at, bs as mo, cE as go, di as xo, al as Co, G as wo, a5 as ke, a$ as yo, b0 as So, cq as bo, ch as vo, ci as ko, cj as Io, aH as Ro, b1 as Po, b2 as $o, dj as Do, aO as No, b3 as zo, b4 as Ao, bD as Fo, cl as Wo, ck as Lo, cY as Vo, b$ as Bo, bE as To, cC as _o, cZ as Oo, c_ as Mo, dk as Eo, aZ as Uo, b5 as Ho, cm as Go, M as Xo, I as Ko, dl as qo, ap as Yo, bi as jo, bj as Qo, bF as Zo, d3 as Jo, bG as ei, P as ti, a6 as si, bH as oi, c$ as ii, aI as ai, c1 as ri, Z as ni, aX as ui, cn as di, H as li, aJ as ci, b9 as hi, d0 as pi, ba as fi, d1 as mi, bJ as gi, bh as xi, b6 as Ci, bK as wi, ak as yi, dm as Si, aL as bi, bL as vi, aF as ki, co as Ii, bM as Ri, bN as Pi, bC as $i, bI as Di, dn as Ni, dp as zi, T as Ai, av as rt, dq as Fi, Q as Wi, J as Li, c3 as Vi, d2 as Bi, b7 as Ti, aK as _i, cr as Oi, dr as Mi, c5 as Ei, cs as Ui, bq as Hi, ds as Gi, ct as Xi, bl as Ki, b8 as qi, bO as Yi, c as ji } from "./index-CUXkjxiT.js";
+import { i as Qi, a as Zi, c as b, f as v, M as Y, b as nt, d as ut, e as dt } from "./webgpu_util-DYlGSwOJ.js";
 import { g as _e, B as F } from "./binary_op_util-pKXltfxI.js";
-import { S as Ji, a as ea } from "./selu_util-BXdhy_W6.js";
-import { E as ta, t as sa, u as oa, w as ia, x as aa, y as ra, f as je, z as lt, A as ct, B as ht, C as na, D as ua, F as da, G as la, H as ca, I as ha, J as pa, K as fa, L as ma, M as ga } from "./backend_util-qwSFfxYx.js";
-import { t as W, e as S, h as Q, b as G, c as Ie, P as pt, d as xa, a as Ca } from "./webgpu_program-CbjdYLYk.js";
-import { r as R, a as wa } from "./Reshape-pwprEaej.js";
-import { s as ya } from "./shared-DRWDyk9w.js";
-import { c as Oe, a as Ce, b as we, d as Me, e as Sa, g as ft } from "./axis_util-QWWgLjut.js";
-import { h as ye, i as Ne, j as Se, b as Z, d as Ee, g as Ue, k as mt } from "./step-DQY6_ABw.js";
-import { p as ba, a as va, s as ka, b as Ia } from "./slice_util-DPY56GzQ.js";
-import { z as Ra } from "./zeros-Bw0puq_w.js";
-import { c as me, a as Pa } from "./concat_util-C1Mxe27t.js";
-import { c as $a, a as Da } from "./segment_util-4zuHV5IG.js";
+import { S as Ji, a as ea } from "./selu_util-BLhIqRkw.js";
+import { E as ta, t as sa, u as oa, w as ia, x as aa, y as ra, f as je, z as lt, A as ct, B as ht, C as na, D as ua, F as da, G as la, H as ca, I as ha, J as pa, K as fa, L as ma, M as ga } from "./backend_util-GaFarB78.js";
+import { t as W, e as S, h as Q, b as G, c as Ie, P as pt, d as xa, a as Ca } from "./webgpu_program-B4HmApL1.js";
+import { r as R, a as wa } from "./Reshape-CPBkTIH2.js";
+import { s as ya } from "./shared-CagdqkLh.js";
+import { c as Oe, a as Ce, b as we, d as Me, e as Sa, g as ft } from "./axis_util-GTVlo58H.js";
+import { h as ye, i as Ne, j as Se, b as Z, d as Ee, g as Ue, k as mt } from "./step-BTxPtq1r.js";
+import { p as ba, a as va, s as ka, b as Ia } from "./slice_util-CC35pLmT.js";
+import { z as Ra } from "./zeros-DvZpK8s6.js";
+import { c as me, a as Pa } from "./concat_util-D18dJ4fD.js";
+import { c as $a, a as Da } from "./segment_util-Dasb2Zaf.js";
 import { n as Na, a as za } from "./non_max_suppression_impl-B2W7YjZB.js";
-import { c as He } from "./scatter_nd_util-C-x73Cj6.js";
+import { c as He } from "./scatter_nd_util-ByNJaL6I.js";
 Qi() && Et(
   "webgpu",
   async () => {

package/dist/{kernel_funcs_utils-DGqzNlHT.js → kernel_funcs_utils-pq0CK9co.js}

@@ -1,6 +1,6 @@
-import { _ as B, N as G, aU as K, a7 as W, aH as z, aV as V, ab as N, aI as F, am as S } from "./index-DSGwv2Yx.js";
-import { u as O, f as H } from "./gpgpu_math-3bCb5ooU.js";
-import { f as v } from "./backend_util-qwSFfxYx.js";
+import { _ as B, U as G, aU as K, a7 as W, aH as z, aV as V, ab as N, aI as F, am as S } from "./index-CUXkjxiT.js";
+import { u as O, f as H } from "./gpgpu_math-BwvV12df.js";
+import { f as v } from "./backend_util-GaFarB78.js";
 function Y(t, e) {
   return ["x", "y", "z", "w", "u", "v"].slice(0, e).map((s) => `${t}.${s}`);
 }
@@ -200,12 +200,12 @@ const be = {
   backendName: "webgl",
   kernelFunc: oe
 };
-const k = "return (a < 0.) ? b * a : a;", U = `
+const U = "return (a < 0.) ? b * a : a;", k = `
   vec4 aLessThanZero = vec4(lessThan(a, vec4(0.)));
   return (aLessThanZero * (b * a)) + ((vec4(1.0) - aLessThanZero) * a);
 `;
 function ue(t) {
-  const { inputs: e, backend: s } = t, { x: r, alpha: u } = e, n = N().getBool("WEBGL_PACK_BINARY_OPERATIONS") ? new E(U, r.shape, u.shape) : new b(k, r.shape, u.shape);
+  const { inputs: e, backend: s } = t, { x: r, alpha: u } = e, n = N().getBool("WEBGL_PACK_BINARY_OPERATIONS") ? new E(k, r.shape, u.shape) : new b(U, r.shape, u.shape);
   return s.runWebGLProgram(n, [r, u], "float32");
 }
 const Ne = {
@@ -273,7 +273,7 @@ function Ae(t, e = !1) {
   if (t === "relu6")
     return e ? ae : Q;
   if (t === "prelu")
-    return e ? U : k;
+    return e ? k : U;
   if (t === "leakyrelu")
     return e ? R : w;
   if (t === "sigmoid")

package/dist/layers/BaseLayer.js

@@ -1,4 +1,4 @@
-import { a2 as p, e as s, a4 as g } from "../index-DSGwv2Yx.js";
+import { a2 as p, h as s, a4 as g } from "../index-CUXkjxiT.js";
 import b from "./WeightStore.js";
 class T {
   parent;
@@ -59,11 +59,11 @@ class T {
   checkpointingFn(t, ...e) {
     const r = this.trainableVariables;
     return p((...i) => {
-      const o = i[i.length - 1], a = i.slice(0, e.length), n = this.forward(t, ...a);
-      return o(a), { value: n, gradFunc: (h, l) => {
+      const o = i[i.length - 1], a = i.slice(0, e.length), h = this.forward(t, ...a);
+      return o(a), { value: h, gradFunc: (n, l) => {
         const c = s().state.activeTape;
         s().state.activeTape = [];
-        const d = g((...u) => this.forward(t, ...u.slice(0, a.length)))([...l, ...r], h);
+        const d = g((...u) => this.forward(t, ...u.slice(0, a.length)))([...l, ...r], n);
         return s().state.activeTape = c, d;
       } };
     })(...e, ...r);

package/dist/layers/CausalSelfAttention.d.ts

@@ -15,6 +15,7 @@ interface AttentionForwardAttributes extends ForwardAttributes {
     attentionScores?: AttentionScores;
     pastKV?: KVCache;
     seed?: number;
+    ropePositionOffset?: number;
 }
 export interface CausalSelfAttentionConfig {
     useQKNorm?: boolean;

package/dist/layers/CausalSelfAttention.js

@@ -1,20 +1,20 @@
 import { attentionMask as R } from "../ops/attentionMask.js";
 import J from "./BaseLayer.js";
-import { r as v } from "../rope-CC5RjmKU.js";
+import { r as v } from "../rope-DcrZM_e6.js";
 import { appendCache as A } from "../ops/appendCache.js";
-import { k as c, t as L } from "../index-DSGwv2Yx.js";
+import { k as c, t as L } from "../index-CUXkjxiT.js";
 import { softmax16 as y } from "../ops/softmax16.js";
-import { b as M } from "../matMul16-BIT70Vya.js";
-import { p as K } from "../pack16-WlOSOuZA.js";
+import { b as M } from "../matMul16-BcVC_E62.js";
+import { p as K } from "../pack16-bqltoUlR.js";
 import { transpose16 as j } from "../ops/transpose16.js";
 import { dot16 as E } from "../ops/dot16.js";
 import { reshape16 as _ } from "../ops/reshape16.js";
-import { isPackedTensor as l } from "../utilities/packed.js";
+import { isPackedTensor as f } from "../utilities/packed.js";
 import { qkv as q } from "../ops/qkv.js";
 import { normRMS as O } from "../ops/normRMS.js";
 import { dropout16 as x } from "../ops/dropout16.js";
-import { v as P } from "../variable-SSATClyt.js";
-import { r as S } from "../random_normal-CIm8lk2-.js";
+import { v as P } from "../variable-Ck482e3n.js";
+import { r as S } from "../random_normal-IBRrha8a.js";
 class it extends J {
   constructor(t, o, s, i) {
     super(o, i), this.attentionConfig = s, this.index = t, this.units = o.nEmbed * 3, this.projUnits = o.nEmbed, this.ATTN = `block_${this.index}_cAttn`, this.PROJ = `block_${this.index}_cProj`, this.addVariable(this.ATTN), this.addVariable(this.PROJ), this.divisor = 1 / Math.sqrt(o.nEmbed / o.nHead);
@@ -39,11 +39,11 @@ class it extends J {
     return i.dispose(), e;
   }
   getQKV(t) {
-    const o = l(t) ? K(this.getVariable(this.ATTN)) : this.getVariable(this.ATTN), s = q(t, o, this.config.nHead);
-    return l(t) && o.dispose(), s;
+    const o = f(t) ? K(this.getVariable(this.ATTN)) : this.getVariable(this.ATTN), s = q(t, o, this.config.nHead);
+    return f(t) && o.dispose(), s;
   }
   getOutputProjection(t) {
-    const o = t.shape[0], s = t.shape[2], i = this.config.nEmbed, e = l(t), r = j(t, [0, 2, 1, 3]), n = _(r, [o, s, e ? i / 2 : i]);
+    const o = t.shape[0], s = t.shape[2], i = this.config.nEmbed, e = f(t), r = j(t, [0, 2, 1, 3]), n = _(r, [o, s, e ? i / 2 : i]);
     r.dispose();
     const p = e ? K(this.getVariable(this.PROJ)) : this.getVariable(this.PROJ), a = E(n, p);
     return e && p.dispose(), n.dispose(), a;
@@ -59,13 +59,13 @@ class it extends J {
   forward(t, o) {
     return L(() => {
       this.startMemory();
-      const [s, i, e] = this.getQKV(o), r = t.pastKV ? t.pastKV.cumulativeLength : 0, n = t.ropeCache, p = n ? v(s, n, r) : s, a = n ? v(i, n, r) : i, h = this.attentionConfig.useQKNorm ?? !1, m = h ? O(p) : p;
+      const [s, i, e] = this.getQKV(o), r = t.pastKV ? t.pastKV.cumulativeLength : t.ropePositionOffset || 0, n = t.ropeCache, p = n ? v(s, n, r) : s, a = n ? v(i, n, r) : i, h = this.attentionConfig.useQKNorm ?? !1, m = h ? O(p) : p;
       h && p.dispose();
-      const f = h ? O(a) : a;
+      const l = h ? O(a) : a;
       h && a.dispose(), n && (s.dispose(), i.dispose());
       const T = t.pastKV ? t.pastKV.length : 0;
-      t.pastKV && !t.training && this.updateCache(f, e, t.pastKV);
-      const u = t.pastKV?.k ? t.pastKV.k : f, V = t.pastKV?.v ? t.pastKV.v : e;
+      t.pastKV && !t.training && this.updateCache(l, e, t.pastKV);
+      const u = t.pastKV?.k ? t.pastKV.k : l, V = t.pastKV?.v ? t.pastKV.v : e;
       let d;
       T > 0 ? d = this.getAttentionScores(m, u, T) : d = this.getAttentionScores(m, u), m.dispose(), t.pastKV || u.dispose();
       const g = M(d, V), b = t.attentionScores !== void 0 && t.attentionScores.attentionOut !== void 0;

package/dist/layers/LoRA.js

@@ -1,8 +1,8 @@
-import { b as m, t as n } from "../index-DSGwv2Yx.js";
+import { a as m, t as n } from "../index-CUXkjxiT.js";
 import { p } from "../index-DmeWGGmS.js";
-import { v as g } from "../variable-SSATClyt.js";
-import { r as S } from "../random_normal-CIm8lk2-.js";
-import { z as _ } from "../zeros-Bw0puq_w.js";
+import { v as g } from "../variable-Ck482e3n.js";
+import { r as S } from "../random_normal-IBRrha8a.js";
+import { z as _ } from "../zeros-DvZpK8s6.js";
 class B {
   weightStore;
   alpha;

package/dist/layers/MLP.js

@@ -1,10 +1,10 @@
-import { t as M } from "../index-DSGwv2Yx.js";
+import { t as M } from "../index-CUXkjxiT.js";
 import f from "./BaseLayer.js";
-import { b as h } from "../matMul16-BIT70Vya.js";
+import { b as h } from "../matMul16-BcVC_E62.js";
 import { reshape16 as d } from "../ops/reshape16.js";
 import { dropout16 as L } from "../ops/dropout16.js";
-import { v as n } from "../variable-SSATClyt.js";
-import { r as m } from "../random_normal-CIm8lk2-.js";
+import { v as n } from "../variable-Ck482e3n.js";
+import { r as m } from "../random_normal-IBRrha8a.js";
 class N extends f {
   index;
   hiddenUnits;

package/dist/layers/PositionEmbedding.js

@@ -1,9 +1,9 @@
-import { t as c, an as p, b as s } from "../index-DSGwv2Yx.js";
+import { t as c, an as p, a as s } from "../index-CUXkjxiT.js";
 import f from "./BaseLayer.js";
-import { E as u } from "../random_width-B_fVXhGx.js";
-import { r as b } from "../exports_initializers-DIOZQt_L.js";
-import { m as g } from "../mod-BXjLYwvM.js";
-import { r as l } from "../range-BDxO73mk.js";
+import { E as u } from "../random_width-DN5ZtQkM.js";
+import { r as b } from "../exports_initializers-CYzKLjN7.js";
+import { m as g } from "../mod-CSdCpRjf.js";
+import { r as l } from "../range-C-CjF-LI.js";
 function h(e) {
   return new u(e);
 }

package/dist/layers/RMSNorm.js

@@ -1,8 +1,8 @@
-import { t as i } from "../index-DSGwv2Yx.js";
+import { t as i } from "../index-CUXkjxiT.js";
 import e from "./BaseLayer.js";
 import { normRMS as m } from "../ops/normRMS.js";
-import { v as a } from "../variable-SSATClyt.js";
-import { o as M } from "../ones-DbVB5N58.js";
+import { v as a } from "../variable-Ck482e3n.js";
+import { o as M } from "../ones-BytntneX.js";
 class l extends e {
   GAMMA;
   rmsConfig;

package/dist/layers/RoPECache.js

@@ -1,7 +1,7 @@
-import { d as a, b as n, p as c, t as p, k as r } from "../index-DSGwv2Yx.js";
-import { r as l } from "../zeros_like-rOHr54NY.js";
-import { c as f, s as m } from "../unsorted_segment_sum-BfFVV9Zm.js";
-import { r as h } from "../range-BDxO73mk.js";
+import { i as a, a as n, p as c, t as p, k as r } from "../index-CUXkjxiT.js";
+import { r as l } from "../zeros_like-CWjDdwr-.js";
+import { c as f, s as m } from "../unsorted_segment_sum-DJvk5xnh.js";
+import { r as h } from "../range-C-CjF-LI.js";
 class x {
   constructor(s) {
     this.config = s;

package/dist/layers/TiedEmbedding.js

@@ -1,13 +1,13 @@
-import "../random_width-B_fVXhGx.js";
-import "../index-DSGwv2Yx.js";
-import { r as s } from "../exports_initializers-DIOZQt_L.js";
+import "../random_width-DN5ZtQkM.js";
+import "../index-CUXkjxiT.js";
+import { r as s } from "../exports_initializers-CYzKLjN7.js";
 import a from "./BaseLayer.js";
 import { dot16 as o } from "../ops/dot16.js";
 import { isPackedTensor as r } from "../utilities/packed.js";
-import { p as m } from "../pack16-WlOSOuZA.js";
+import { p as m } from "../pack16-bqltoUlR.js";
 import { transpose16 as d } from "../ops/transpose16.js";
-import { v as p } from "../variable-SSATClyt.js";
-import { g as h } from "../gather-DEyjXNb1.js";
+import { v as p } from "../variable-Ck482e3n.js";
+import { g as h } from "../gather-qIqEqaGn.js";
 class g extends a {
   vocabSize;
   embedDim;

package/dist/layers/TransformerBlock.d.ts

@@ -8,6 +8,7 @@ interface BlockAttributes extends ForwardAttributes {
     pastKV?: KVCache;
     seed?: number;
     attentionScores?: AttentionScores;
+    ropePositionOffset?: number;
 }
 export type TransformerBlockConfig = MLPConfig & RMSNormConfig & CausalSelfAttentionConfig;
 export default class Block extends BaseLayer<BlockAttributes> {

package/dist/layers/TransformerBlock.js

@@ -2,7 +2,7 @@ import p from "./CausalSelfAttention.js";
 import h from "./MLP.js";
 import l from "./RMSNorm.js";
 import m from "./BaseLayer.js";
-import { k as n, t as u } from "../index-DSGwv2Yx.js";
+import { k as n, t as u } from "../index-CUXkjxiT.js";
 import { add16 as d } from "../ops/add16.js";
 class _ extends m {
   ln1;

package/dist/layers/WeightStore.js

@@ -1,6 +1,6 @@
-import { u as o } from "../index-DSGwv2Yx.js";
+import { w as o } from "../index-CUXkjxiT.js";
 import { p as h } from "../index-DmeWGGmS.js";
-import { v as b } from "../variable-SSATClyt.js";
+import { v as b } from "../variable-Ck482e3n.js";
 class d {
   _variables = /* @__PURE__ */ new Map();
   touchedVariables = /* @__PURE__ */ new Set();

package/dist/loader/loadTransformers.js

@@ -1,7 +1,7 @@
 import p from "../tokeniser/CharTokeniser.js";
 import _ from "../tokeniser/bpe.js";
 import { load_safetensors as b } from "../utilities/safetensors.js";
-import { aa as c } from "../index-DSGwv2Yx.js";
+import { aa as c } from "../index-CUXkjxiT.js";
 import { dummyPassAsync as u } from "../utilities/dummy.js";
 import y from "../models/factory.js";
 function h(e) {
@@ -28,15 +28,15 @@ function h(e) {
     windowSize: e.windowSize
   }, a;
 }
-async function G(e, a, t, n) {
-  const m = h(e), r = (a.type ?? "char") === "char" ? new p(a.vocab) : new _(a.vocab, a.merges);
-  a.datasetID && (r.datasetID = a.datasetID);
-  const i = await b(n), s = /* @__PURE__ */ new Map();
-  for (const [l, d] of Object.entries(i))
-    s.set(l, [d]);
+async function G(e, a, r, n) {
+  const m = h(e), o = (a.type ?? "char") === "char" ? new p(a.vocab) : new _(a.vocab, a.merges);
+  a.datasetID && (o.datasetID = a.datasetID), a.id ? o.id = a.id : o.generateID();
+  const d = await b(n), s = /* @__PURE__ */ new Map();
+  for (const [i, l] of Object.entries(d))
+    s.set(i, [l]);
   c();
-  const o = y(m);
-  return o.metaData = t, await u(o), o.weightStore.loadWeights(s, !!t.url), { model: o, tokeniser: r, metaData: t };
+  const t = y(m);
+  return t.metaData = r, await u(t), t.weightStore.loadWeights(s, !!r.url), { model: t, tokeniser: o, metaData: r };
 }
 export {
   G as default,

package/dist/loader/oldZipLoad.js

@@ -1,22 +1,22 @@
-import { aa as y } from "../index-DSGwv2Yx.js";
-import "../random_width-B_fVXhGx.js";
-import "../zeros_like-rOHr54NY.js";
+import { aa as y } from "../index-CUXkjxiT.js";
+import "../random_width-DN5ZtQkM.js";
+import "../zeros_like-CWjDdwr-.js";
 import "../Generator.js";
 import "../index-Cp39cXWe.js";
-import "../dataset-DlqAN81i.js";
+import "../dataset-CGGp1z9P.js";
 import "../ops/cpu/attentionMask.js";
 import "../ops/webgl/attentionMask.js";
 import "../ops/grads/attentionMask.js";
 import "../ops/cpu/rope.js";
 import "../ops/webgl/rope.js";
-import "../rope-CC5RjmKU.js";
+import "../rope-DcrZM_e6.js";
 import "../ops/cpu/appendCache.js";
 import "../ops/webgl/appendCache.js";
 import "../ops/grads/softmax16.js";
-import "../matMul16-BIT70Vya.js";
+import "../matMul16-BcVC_E62.js";
 import "../ops/webgl/matMul16.js";
 import "../ops/cpu/matMul16.js";
-import "../pack16-WlOSOuZA.js";
+import "../pack16-bqltoUlR.js";
 import "../ops/transpose16.js";
 import "../ops/reshape16.js";
 import "../ops/cpu/qkv.js";
@@ -44,11 +44,11 @@ import "../ops/cpu/scatterSub.js";
 import "../ops/webgl/scatterSub.js";
 import "../papaparse.min-C0cScC2i.js";
 import "../ops/cpu/matMulGelu.js";
-import "../matMulGelu-CsZnh18H.js";
+import "../matMulGelu-JNLZqKQp.js";
 import "../ops/grads/matMulGelu.js";
 import "../ops/cpu/gelu.js";
 import "../ops/webgl/gelu.js";
-import "../gelu-DpTCC3eB.js";
+import "../gelu-B220X1Go.js";
 import "../ops/webgl/log.js";
 import "../checks/normRMS.js";
 import "../checks/normRMSGrad.js";

package/dist/loader/save.js

@@ -1,31 +1,38 @@
 import { z as y } from "../jszip.min-BZhlzntC.js";
-import b from "../tokeniser/CharTokeniser.js";
-import { save_safetensors as _ } from "../utilities/safetensors.js";
-import { VERSION as m } from "./load.js";
-async function d(i, e, o, t) {
-  const g = /* @__PURE__ */ new Map();
-  i.weightStore.saveWeights(g);
-  const n = new y();
-  if (t?.optimizer) {
-    const a = await t.optimizer.saveMoments();
-    n.file("optimizer.safetensors", a, { binary: !0 }), n.file("optimizer_config.json", JSON.stringify(t.optimizer.serializeConfig()), {
+import _ from "../tokeniser/CharTokeniser.js";
+import { save_safetensors as b } from "../utilities/safetensors.js";
+import { VERSION as h } from "./load.js";
+function m(i) {
+  if (i.length > 1e3) {
+    const n = Math.ceil(i.length / 1e3);
+    return i.filter((o, a) => a % n === 0 || a === i.length - 1);
+  }
+  return i;
+}
+async function u(i, n, o, a) {
+  const s = /* @__PURE__ */ new Map();
+  i.weightStore.saveWeights(s);
+  const e = new y();
+  if (a?.optimizer) {
+    const t = await a.optimizer.saveMoments();
+    e.file("optimizer.safetensors", t, { binary: !0 }), e.file("optimizer_config.json", JSON.stringify(a.optimizer.serializeConfig()), {
       binary: !1
     });
   }
-  t?.trainingLog && n.file("training_log.json", JSON.stringify(t.trainingLog, void 0, 4), {
+  a?.trainingLog && e.file("training_log.json", JSON.stringify(m(a.trainingLog), void 0, 4), {
     binary: !1
   });
-  const s = {};
-  g.forEach((a, f) => {
-    a.length === 1 && (s[f] = a[0]);
+  const g = {};
+  s.forEach((t, f) => {
+    t.length === 1 && (g[f] = t[0]);
   });
-  const c = await _(s);
-  n.file("model.safetensors", c, { binary: !0 });
+  const c = await b(g);
+  e.file("model.safetensors", c, { binary: !0 });
   const p = i.config.modelType;
   let r;
   p === "GenAI_NanoGPT_v1" ? r = {
     model_type: "GenAI_NanoGPT_v1",
-    vocab_size: e.getVocab().length,
+    vocab_size: n.getVocab().length,
     hidden_size: i.config.nEmbed,
     num_hidden_layers: i.config.nLayer,
     num_attention_heads: i.config.nHead,
@@ -34,7 +41,7 @@ async function d(i, e, o, t) {
     useRope: i.config.useRope
   } : r = {
     model_type: "GenAI_NanoGPT_v2",
-    vocab_size: e.getVocab().length,
+    vocab_size: n.getVocab().length,
     hidden_size: i.config.nEmbed,
     num_hidden_layers: i.config.nLayer,
     num_attention_heads: i.config.nHead,
@@ -43,11 +50,11 @@ async function d(i, e, o, t) {
     loraConfig: i.config.loraConfig ? Object.fromEntries(i.config.loraConfig) : void 0,
     loraName: i.config.loraName,
     windowSize: i.config.windowSize
-  }, n.file("config.json", JSON.stringify(r, void 0, 4), {
+  }, e.file("config.json", JSON.stringify(r, void 0, 4), {
     binary: !1
   });
   const l = {
-    version: m,
+    version: h,
     application: "@genai-fi/nanogpt",
     meta: o?.metadata,
     name: o?.name,
@@ -59,24 +66,25 @@ async function d(i, e, o, t) {
     generationSettings: i.metaData?.generationSettings || void 0,
     actionLog: i.metaData?.actionLog || void 0
   };
-  if (n.file("meta.json", JSON.stringify(l, void 0, 4), {
+  if (e.file("meta.json", JSON.stringify(l, void 0, 4), {
     binary: !1
-  }), n.file(
+  }), e.file(
     "tokeniser.json",
     JSON.stringify({
-      type: e instanceof b ? "char" : "bpe",
-      vocab: e.getVocab(),
-      merges: e.getMerges(),
-      datasetID: e.datasetID
+      type: n instanceof _ ? "char" : "bpe",
+      vocab: n.getVocab(),
+      merges: n.getMerges(),
+      datasetID: n.datasetID,
+      id: n.id
     }),
     {
       binary: !1
     }
   ), o?.files)
-    for (const [a, f] of Object.entries(o.files))
-      n.file(a, JSON.stringify(f), { binary: !1 });
-  return n.generateAsync({ type: "blob" });
+    for (const [t, f] of Object.entries(o.files))
+      e.file(t, JSON.stringify(f), { binary: !1 });
+  return e.generateAsync({ type: "blob" });
 }
 export {
-  d as saveModel
+  u as saveModel
 };

package/dist/loader/types.d.ts

@@ -29,6 +29,7 @@ export interface TransformersTokeniser {
     vocab: string[];
     merges: [string, string][];
     datasetID?: string;
+    id?: string;
 }
 export type ModelPhase = 'untrained' | 'pretrained' | 'finetuned';
 export interface DatasetMetadata {