@genai-fi/nanogpt 0.18.0 → 0.19.0

package/dist/{tfjs_backend-Bk3PmK91.js → tfjs_backend-BGnG-ppu.js}

@@ -1,26 +1,26 @@
-import { o as g, n as w, v as A, ag as Te, d as ke, h as M, m as Q, E as q, z as ae, N as ue, _ as le, a2 as fe, bb as he, aD as pe, bc as Ie, t as S, J as $e, a_ as Ee } from "./index-DSGwv2Yx.js";
-import { t as be } from "./tensor1d-BMl0eZYV.js";
-import { r as Le } from "./random_normal-CIm8lk2-.js";
-import { s as P } from "./slice-BvItlgXu.js";
-import { r as c } from "./reshape-BIN71H3p.js";
-import { g as Ne } from "./gather-DEyjXNb1.js";
-import { e as Pe } from "./step-DQY6_ABw.js";
-import { c as ve } from "./clip_by_value-CLAD4h_I.js";
-import { t as Ce } from "./tile-CsWlVKKz.js";
-import { s as Fe, b as Me, c as je, g as Ve } from "./selu_util-BXdhy_W6.js";
-import { m as $ } from "./mat_mul-DP86qZtZ.js";
-import { t as Ue } from "./transpose-Qxz-4os3.js";
-import { c as j } from "./concat-Dqk7Xk7h.js";
-import { g as xe, r as Be } from "./dropout_util-N0z8Os-K.js";
-import { f as Ge } from "./floor-CymuCmTO.js";
-function Je(e) {
+import { o as g, q as w, x as A, ag as Te, i as ke, j as M, m as Q, E as J, B as ae, U as ue, _ as le, a2 as fe, bb as he, aD as pe, bc as Ie, t as S, L as $e, a_ as Ee } from "./index-CUXkjxiT.js";
+import { t as be } from "./tensor1d-LMGMIUlr.js";
+import { r as Le } from "./random_normal-IBRrha8a.js";
+import { s as P } from "./slice-BzS11Qh0.js";
+import { r as c } from "./reshape-BDOuCSNW.js";
+import { g as Ne } from "./gather-qIqEqaGn.js";
+import { e as Pe } from "./step-BTxPtq1r.js";
+import { c as Ce } from "./clip_by_value-DDA7rrcT.js";
+import { t as Fe } from "./tile-CFy-xTO6.js";
+import { s as ve, b as Me, c as je, g as Ue } from "./selu_util-BLhIqRkw.js";
+import { m as $ } from "./mat_mul-DhG0Newp.js";
+import { t as Ve } from "./transpose-9kRxIXWR.js";
+import { c as j } from "./concat-CAQpCret.js";
+import { g as xe, r as Be } from "./dropout_util--NxWuYg2.js";
+import { f as Ge } from "./floor-BQtb-Azg.js";
+function qe(e) {
   return j(
     e,
     0
     /* axis */
   );
 }
-const qe = /* @__PURE__ */ g({ concat1d_: Je });
+const Je = /* @__PURE__ */ g({ concat1d_: qe });
 function Ke(e, n) {
   return j(e, n);
 }
@@ -52,7 +52,7 @@ function en(e, n, t) {
   const s = w(e, "x", "slice4d");
   return A(s.rank === 4, () => `slice4d expects a rank-4 tensor, but got a rank-${s.rank} tensor`), P(s, n, t);
 }
-const U = /* @__PURE__ */ g({ slice4d_: en });
+const V = /* @__PURE__ */ g({ slice4d_: en });
 function nn(e, n, t, s) {
   const r = w(e, "x", "dropout");
   if (A(r.dtype === "float32", () => `x has to be a floating point tensor since it's going to be scaled, but got a ${r.dtype} tensor instead.`), A(n >= 0 && n < 1, () => `rate must be a float in the range [0, 1), but got ${n}.`), n === 0)
@@ -62,7 +62,7 @@ function nn(e, n, t, s) {
 }
 const tn = /* @__PURE__ */ g({ dropout_: nn });
 function sn({ a: e, b: n, transposeA: t = !1, transposeB: s = !1, bias: r, activation: o = "linear", preluActivationWeights: i, leakyreluAlpha: f = 0.2 }) {
-  if (Fe(q.state.gradientDepth, o) === !1) {
+  if (ve(J.state.gradientDepth, o) === !1) {
     let _ = $(e, n, t, s);
     return r != null && (_ = M(_, r)), Me(_, o, i, f);
   }
@@ -70,38 +70,38 @@ function sn({ a: e, b: n, transposeA: t = !1, transposeB: s = !1, bias: r, activ
   [a, u] = ae(a, u);
   const m = t ? a.shape[a.rank - 2] : a.shape[a.rank - 1], d = s ? u.shape[u.rank - 1] : u.shape[u.rank - 2], T = t ? a.shape[a.rank - 1] : a.shape[a.rank - 2], h = s ? u.shape[u.rank - 2] : u.shape[u.rank - 1], ne = a.shape.slice(0, -2), I = u.shape.slice(0, -2), te = ue(ne), se = ue(I);
   A(m === d, () => `Error in fused matMul: inner shapes (${m}) and (${d}) of Tensors with shapes ${a.shape} and ${u.shape} and transposeA=${t} and transposeB=${s} must match.`);
-  const B = le(a.shape.slice(0, -2), u.shape.slice(0, -2)).concat([T, h]), G = t ? c(a, [te, m, T]) : c(a, [te, T, m]), J = s ? c(u, [se, h, d]) : c(u, [se, d, h]);
+  const B = le(a.shape.slice(0, -2), u.shape.slice(0, -2)).concat([T, h]), G = t ? c(a, [te, m, T]) : c(a, [te, T, m]), q = s ? c(u, [se, h, d]) : c(u, [se, d, h]);
   let E;
   r != null && (E = w(r, "bias", "fused matMul"), [E] = ae(E, a), le(B, E.shape));
   let re;
   i != null && (re = w(i, "prelu weights", "fused matMul"));
-  const oe = (_, v) => {
-    const [O, D, y, V] = v, k = je(c(_, y.shape), y, o);
+  const oe = (_, C) => {
+    const [O, D, y, U] = C, k = je(c(_, y.shape), y, o);
     let b, L;
     if (!t && !s ? (b = $(k, D, !1, !0), L = $(O, k, !0, !1)) : !t && s ? (b = $(k, D, !1, !1), L = $(k, O, !0, !1)) : t && !s ? (b = $(D, k, !1, !0), L = $(O, k, !1, !1)) : (b = $(D, k, !0, !0), L = $(k, O, !0, !0)), r != null) {
-      const ye = Ve(V, k);
+      const ye = Ue(U, k);
       return [b, L, ye];
     } else
       return [b, L];
   }, ie = {
     a: G,
-    b: J,
+    b: q,
     bias: E,
     preluActivationWeights: re
   }, ce = { transposeA: t, transposeB: s, activation: o, leakyreluAlpha: f };
-  return r == null ? fe((v, O, D) => {
+  return r == null ? fe((C, O, D) => {
     const y = (
       // tslint:disable-next-line: no-unnecessary-type-assertion
-      q.runKernel(he, ie, ce)
+      J.runKernel(he, ie, ce)
     );
-    return D([v, O, y]), { value: c(y, B), gradFunc: oe };
-  })(G, J) : fe((v, O, D, y) => {
-    const V = (
+    return D([C, O, y]), { value: c(y, B), gradFunc: oe };
+  })(G, q) : fe((C, O, D, y) => {
+    const U = (
       // tslint:disable-next-line: no-unnecessary-type-assertion
-      q.runKernel(he, ie, ce)
+      J.runKernel(he, ie, ce)
     );
-    return y([v, O, V, D]), { value: c(V, B), gradFunc: oe };
-  })(G, J, E);
+    return y([C, O, U, D]), { value: c(U, B), gradFunc: oe };
+  })(G, q, E);
 }
 const de = /* @__PURE__ */ g({ fusedMatMul_: sn });
 class Ae extends Error {
@@ -119,9 +119,9 @@ class l extends Error {
     super(n), Object.setPrototypeOf(this, l.prototype);
   }
 }
-class F extends Error {
+class v extends Error {
   constructor(n) {
-    super(n), Object.setPrototypeOf(this, F.prototype);
+    super(n), Object.setPrototypeOf(this, v.prototype);
   }
 }
 class ee extends Error {
@@ -144,16 +144,16 @@ function me(e, n) {
   if (!e)
     throw new ee(n);
 }
-function vn(e, n) {
+function Cn(e, n) {
   let t = 0;
   for (const s of e)
     s === n && t++;
   return t;
 }
-function Cn(e) {
+function Fn(e) {
   return e.length === 1 ? e[0] : e;
 }
-function Fn(e) {
+function vn(e) {
   return Array.isArray(e) ? e : [e];
 }
 function Mn(e) {
@@ -164,7 +164,7 @@ function jn(e) {
   return e.length <= 1 || e.indexOf("_") === -1 ? e : e.replace(/[_]+(\w|$)/g, (n, t) => t.toUpperCase());
 }
 let p = {};
-function Vn(e) {
+function Un(e) {
   if (e == null)
     return null;
   const n = {};
@@ -182,7 +182,7 @@ function W(e) {
       }
     }
 }
-function Un(e, n = {}, t = {}, s = "object", r = !1) {
+function Vn(e, n = {}, t = {}, s = "object", r = !1) {
   if (typeof e == "string") {
     const o = e;
     let i;
@@ -255,7 +255,7 @@ function x(e, n, t) {
   if (t != null && e.indexOf(t) < 0)
     throw new l(`${t} is not a valid ${n}.  Valid values are ${e} or null/undefined.`);
 }
-function Jn(e, n, t = 0, s = 1 / 0) {
+function qn(e, n, t = 0, s = 1 / 0) {
   return me(t >= 0), me(s >= t), Array.isArray(e) && e.length >= t && e.length <= s && e.every((r) => typeof r === n);
 }
 function on(e, n) {
@@ -264,7 +264,7 @@ function on(e, n) {
 function Oe(e) {
   return e === null ? "null" : Array.isArray(e) ? "[" + e.map((n) => Oe(n)).join(",") + "]" : typeof e == "string" ? `"${e}"` : `${e}`;
 }
-function qn(e, n, t) {
+function Jn(e, n, t) {
   let s = t != null ? t() : pe(), r;
   return (...i) => {
     const f = t != null ? t() : pe();
@@ -288,18 +288,18 @@ function Wn(e) {
 function Yn(e) {
   x(ln, "PoolMode", e);
 }
-const C = [], ge = "/";
+const F = [], ge = "/";
 function Hn(e, n) {
-  C.push(e);
+  F.push(e);
   try {
     const t = n();
-    return C.pop(), t;
+    return F.pop(), t;
   } catch (t) {
-    throw C.pop(), t;
+    throw F.pop(), t;
   }
 }
 function hn() {
-  return C.length === 0 ? "" : C.join(ge) + ge;
+  return F.length === 0 ? "" : F.join(ge) + ge;
 }
 function Qn(e) {
   if (!De(e))
@@ -401,7 +401,7 @@ function R(e, n, t) {
       case 3:
         return z(e, [n, 0, 0], [t, e.shape[1], e.shape[2]]);
       case 4:
-        return U(e, [n, 0, 0, 0], [t, e.shape[1], e.shape[2], e.shape[3]]);
+        return V(e, [n, 0, 0, 0], [t, e.shape[1], e.shape[2], e.shape[3]]);
       case 5:
         return P(e, [n, 0, 0, 0, 0], [
           t,
@@ -434,7 +434,7 @@ function Z(e, n, t) {
       case 3:
         return z(e, [0, 0, n], [e.shape[0], e.shape[1], t]);
       case 4:
-        return U(e, [0, 0, 0, n], [e.shape[0], e.shape[1], e.shape[2], t]);
+        return V(e, [0, 0, 0, n], [e.shape[0], e.shape[1], e.shape[2], t]);
       default:
         throw new l(`sliceAlongLastAxis() received an unsupported tensor rank: ${e.rank}`);
     }
@@ -470,9 +470,9 @@ function at(e, n, t, s) {
           case 1:
             return R(e, n, t);
           case 2:
-            return U(e, [0, n, 0, 0], [e.shape[0], t, e.shape[2], e.shape[3]]);
+            return V(e, [0, n, 0, 0], [e.shape[0], t, e.shape[2], e.shape[3]]);
           case 3:
-            return U(e, [0, 0, n, 0], [e.shape[0], e.shape[1], t, e.shape[3]]);
+            return V(e, [0, 0, n, 0], [e.shape[0], e.shape[1], t, e.shape[3]]);
           case 4:
             return Z(e, n, t);
           default:
@@ -490,7 +490,7 @@ function ut(e, n = -1) {
 function lt(e, n) {
   switch (e.rank) {
     case 1:
-      return qe([e, n]);
+      return Je([e, n]);
     case 2:
       return Re([e, n], 0);
     case 3:
@@ -504,18 +504,18 @@ function lt(e, n) {
 function mn(e, n) {
   if (Array.isArray(n) || (n = [n]), e.rank !== n.length)
     throw new l(`The length of input n (${n.length}) does not match the number of dimensions in input x (${e.rank})`);
-  return Ce(e, n);
+  return Fe(e, n);
 }
 function ft(e, n = 0, t = 1, s, r) {
   return Le(e, n, t, s, r);
 }
 function ht(e, n, t, s) {
   if (e.rank < 2 || n.rank < 2)
-    throw new F(`dot requires both inputs to be rank >= 2 but got x shape = ${e.shape} and y shape = ${n.shape}`);
+    throw new v(`dot requires both inputs to be rank >= 2 but got x shape = ${e.shape} and y shape = ${n.shape}`);
   if (n.rank >= 3) {
     const r = e.shape.slice(-1)[0], o = n.shape.slice(-2)[0];
     if (r !== o)
-      throw new F(`If rank y >= 3, then the second last dim of y must equal the last dim of x but got x shape = ${e.shape} and  y shape = ${n.shape}`);
+      throw new v(`If rank y >= 3, then the second last dim of y must equal the last dim of x but got x shape = ${e.shape} and  y shape = ${n.shape}`);
   }
   if (e.rank === 2 && n.rank === 2)
     return de({
@@ -530,7 +530,7 @@ function ht(e, n, t, s) {
     const r = e.shape.slice(), o = r.pop();
     e = c(e, [-1, o]);
     const i = n.shape.slice(), f = i.pop(), a = i.pop(), u = [...i, f], m = Array.from({ length: n.rank }, (ne, I) => I === 0 ? n.rank - 2 : I <= n.rank - 2 ? I - 1 : I);
-    n = c(Ue(n, m), [a, -1]);
+    n = c(Ve(n, m), [a, -1]);
     const d = [...r, ...u];
     return c(de({
       a: e,
@@ -576,7 +576,7 @@ function mt(e, n, t) {
 }
 function gt(e, n = 1) {
   if (n !== 1)
-    throw new F(`Support for alpha values other than 1 (${n}) is not implemented yet.`);
+    throw new v(`Support for alpha values other than 1 (${n}) is not implemented yet.`);
   return Pe(e);
 }
 function kt(e) {
@@ -588,7 +588,7 @@ function $t(e, n, t, s) {
 function wt(e) {
   return S(() => {
     const n = M(0.5, Q(0.2, e));
-    return ve(n, 0, 1);
+    return Ce(n, 0, 1);
   });
 }
 function At(e, n, t = !1) {
@@ -599,17 +599,17 @@ export {
   Ae as A,
   pt as B,
   tt as C,
-  vn as D,
+  Cn as D,
   gt as E,
   wt as F,
   kt as G,
   nt as H,
   zn as I,
-  Jn as J,
+  qn as J,
   mt as K,
   at as L,
   Zn as M,
-  F as N,
+  v as N,
   on as O,
   Kn as P,
   Wn as Q,
@@ -629,13 +629,13 @@ export {
   _e as b,
   x as c,
   ht as d,
-  Un as e,
+  Vn as e,
   Xn as f,
   Qn as g,
-  Cn as h,
-  Fn as i,
+  Fn as h,
+  vn as i,
   st as j,
-  qn as k,
+  Jn as k,
   it as l,
   dt as m,
   Hn as n,
@@ -643,7 +643,7 @@ export {
   rt as p,
   jn as q,
   ft as r,
-  Vn as s,
+  Un as s,
   Mn as t,
   Bn as u,
   xn as v,

package/dist/{tile-CsWlVKKz.js → tile-CFy-xTO6.js}

@@ -1,4 +1,4 @@
-import { o as e, n as a, v as i, E as c, T as l } from "./index-DSGwv2Yx.js";
+import { o as e, q as a, x as i, E as c, T as l } from "./index-CUXkjxiT.js";
 function u(r, t) {
   const n = a(r, "x", "tile", "string_or_numeric");
   i(n.rank === t.length, () => `Error in transpose: rank of input ${n.rank} must match length of reps ${t}.`);

package/dist/tokeniser/BaseTokeniser.d.ts

@@ -14,7 +14,7 @@ export default abstract class BaseTokeniser extends EE<'trainStatus'> implements
     isSpecialToken(index: number): boolean;
     protected addSpecialTokens(): void;
     protected addSpecialToken(token: string, index: number): void;
-    protected generateID(): void;
+    generateID(): void;
     abstract train(text: Conversation[][], cb?: (vocab: number) => void, datasetID?: string): Promise<number>;
     abstract getVocab(): string[];
     abstract getMerges(): [string, string][];
@@ -23,6 +23,10 @@ export default abstract class BaseTokeniser extends EE<'trainStatus'> implements
     encodeSequence(text: string): number[];
     encodeAsSequence(conversation: Conversation[], completion?: boolean): number[];
     encodeConversation(conversation: Conversation[], completion?: boolean): number[];
+    encodeConversation(conversation: Conversation[], completion: boolean, masking: boolean): {
+        tokens: number[];
+        mask: boolean[];
+    };
     abstract decode(tokens: number[]): string;
     decodeConversation(tokens: number[] | Uint16Array): Conversation[];
     getSpecialTokenIndex(token: string): number | undefined;

package/dist/tokeniser/BaseTokeniser.js

@@ -1,5 +1,5 @@
-import { E as r } from "../index-DvYrXKkX.js";
-const l = [
+import { E as d } from "../index-DvYrXKkX.js";
+const u = [
   "<eos>",
   "<bos>",
   "",
@@ -11,105 +11,114 @@ const l = [
   "<|system_start|>",
   "<|system_end|>"
 ];
-class T extends r {
+class T extends d {
   id = "untrained";
   datasetID;
   specialTokens = /* @__PURE__ */ new Map();
   specialTokenSet = /* @__PURE__ */ new Set();
-  isSpecialToken(s) {
-    return this.specialTokenSet.has(s);
+  isSpecialToken(t) {
+    return this.specialTokenSet.has(t);
   }
   addSpecialTokens() {
-    l.forEach((s, t) => {
-      this.addToken(s, t), this.specialTokens.set(s, t), this.specialTokenSet.add(t);
+    u.forEach((t, n) => {
+      this.addToken(t, n), this.specialTokens.set(t, n), this.specialTokenSet.add(n);
     });
   }
-  addSpecialToken(s, t) {
-    this.specialTokens.set(s, t), this.specialTokenSet.add(t);
+  addSpecialToken(t, n) {
+    this.specialTokens.set(t, n), this.specialTokenSet.add(n);
   }
   generateID() {
-    const s = this.getVocab();
-    let t = 2166136261, e = 2654435769;
-    for (let a = 0; a < s.length; a++) {
-      const i = s[a];
-      t ^= i.length, t = Math.imul(t, 16777619), e ^= a, e = Math.imul(e, 2246822507);
-      for (let c = 0; c < i.length; c++) {
-        const h = i.charCodeAt(c);
-        t ^= h, t = Math.imul(t, 16777619), e ^= h, e = Math.imul(e, 3266489909);
+    const t = this.getVocab();
+    let n = 2166136261, e = 2654435769;
+    if (t.length === 0) {
+      this.id = "untrained";
+      return;
+    }
+    for (let a = 0; a < t.length; a++) {
+      const i = t[a];
+      n ^= i.length, n = Math.imul(n, 16777619), e ^= a, e = Math.imul(e, 2246822507);
+      for (let h = 0; h < i.length; h++) {
+        const c = i.charCodeAt(h);
+        n ^= c, n = Math.imul(n, 16777619), e ^= c, e = Math.imul(e, 3266489909);
       }
     }
-    const o = (t >>> 0).toString(36), n = (e >>> 0).toString(36);
-    this.id = "tokeniser_" + o + "_" + n;
+    const o = (n >>> 0).toString(36), s = (e >>> 0).toString(36);
+    this.id = "tokeniser_" + o + "_" + s;
   }
-  encodeSequence(s) {
-    const t = this.encode(s);
-    return [this.bosToken, ...t, this.eosToken];
+  encodeSequence(t) {
+    const n = this.encode(t);
+    return [this.bosToken, ...n, this.eosToken];
   }
-  encodeAsSequence(s, t) {
-    const e = s.flatMap((o) => this.encode(o.content));
-    return t ? [this.bosToken, ...e, this.eosToken, this.bosToken] : [this.bosToken, ...e, this.eosToken];
+  encodeAsSequence(t, n) {
+    const e = t.flatMap((o) => this.encode(o.content));
+    return n ? [this.bosToken, ...e, this.eosToken, this.bosToken] : [this.bosToken, ...e, this.eosToken];
   }
-  encodeConversation(s, t) {
-    const e = [[this.bosToken]], o = [
+  encodeConversation(t, n, e) {
+    const o = [[this.bosToken]];
+    let s;
+    e && (s = [[!1]]);
+    const a = [
       this.getSpecialTokenIndex("<|user_start|>"),
       this.getSpecialTokenIndex("<|assistant_start|>"),
       this.getSpecialTokenIndex("<|system_start|>")
-    ], n = [
+    ], i = [
       this.getSpecialTokenIndex("<|user_end|>"),
       this.getSpecialTokenIndex("<|assistant_end|>"),
       this.getSpecialTokenIndex("<|system_end|>")
     ];
-    for (const i of s) {
-      const c = this.encode(i.content);
-      switch (i.role) {
+    for (const c of t) {
+      let l = !1;
+      const r = this.encode(c.content);
+      switch (c.role) {
         case "user":
-          e.push([o[0]]);
+          o.push([a[0]]), l = !0;
           break;
         case "assistant":
-          e.push([o[1]]);
+          o.push([a[1]]);
           break;
         case "system":
-          e.push([o[2]]);
+          o.push([a[2]]), l = !0;
           break;
       }
-      switch (e.push(c), i.role) {
+      switch (o.push(r), c.role) {
         case "user":
-          e.push([n[0]]);
+          o.push([i[0]]);
           break;
         case "assistant":
-          e.push([n[1]]);
+          o.push([i[1]]);
           break;
         case "system":
-          e.push([n[2]]);
+          o.push([i[2]]);
           break;
       }
+      e && s && l ? (s.push([!1]), s.push(r.map(() => !1)), s.push([!1])) : e && s && (s.push([!1]), s.push(r.map(() => !0)), s.push([!0]));
     }
-    const a = e.flat();
-    return t ? a.push(o[1]) : a.push(this.eosToken), a;
+    const h = o.flat();
+    return n ? (h.push(a[1]), e && s && s.push([!1])) : (h.push(this.eosToken), e && s && s.push([!0])), e && s ? { tokens: h, mask: s.flat() } : h;
   }
-  decodeConversation(s) {
-    const t = [];
+  decodeConversation(t) {
+    const n = [];
     let e = 0;
-    for (; e < s.length; ) {
-      const o = s[e];
-      let n = null;
-      if (o === this.getSpecialTokenIndex("<|user_start|>") ? n = "user" : o === this.getSpecialTokenIndex("<|assistant_start|>") ? n = "assistant" : o === this.getSpecialTokenIndex("<|system_start|>") ? n = "system" : o === this.bosToken || (o === this.eosToken ? n = null : (n = "text", e--)), n) {
+    for (; e < t.length; ) {
+      const o = t[e];
+      let s = null;
+      if (o === this.getSpecialTokenIndex("<|user_start|>") ? s = "user" : o === this.getSpecialTokenIndex("<|assistant_start|>") ? s = "assistant" : o === this.getSpecialTokenIndex("<|system_start|>") ? s = "system" : o === this.bosToken || (o === this.eosToken ? s = null : (s = "text", e--)), s) {
         e++;
         const a = [];
-        for (; e < s.length && s[e] !== this.getSpecialTokenIndex(`<|${n}_end|>`) && s[e] !== this.eosToken; )
-          a.push(s[e]), e++;
+        for (; e < t.length && t[e] !== this.getSpecialTokenIndex(`<|${s}_end|>`) && t[e] !== this.eosToken; )
+          a.push(t[e]), e++;
         const i = this.decode(a);
-        t.push({ role: n, content: i });
+        n.push({ role: s, content: i });
       }
       e++;
     }
-    return t;
+    return n;
   }
-  getSpecialTokenIndex(s) {
-    return this.specialTokens.get(s);
+  getSpecialTokenIndex(t) {
+    return this.specialTokens.get(t);
   }
 }
 export {
-  l as SPECIALS,
+  u as SPECIALS,
   T as default
 };

package/dist/tokeniser/type.d.ts

@@ -13,6 +13,14 @@ export interface ITokeniser extends EE<'trainStatus'> {
     destroy(): void;
     encode(text: string): number[];
     encodeConversation(conversation: Conversation[], completion?: boolean): number[];
+    encodeConversation(conversation: Conversation[], completion: boolean, masking: boolean): {
+        tokens: number[];
+        mask: boolean[];
+    };
+    encodeConversation(conversation: Conversation[], completion?: boolean, masking?: boolean): number[] | {
+        tokens: number[];
+        mask: boolean[];
+    };
     encodeSequence(text: string): number[];
     encodeAsSequence(conversation: Conversation[], completion?: boolean): number[];
     decode(tokens: number[] | Uint16Array): string;

package/dist/training/AdamW.js

@@ -1,10 +1,10 @@
 import { adamAdjust as B } from "../ops/adamAdjust.js";
 import { adamMoments as N } from "../ops/adamMoments.js";
-import { O as S, e as b, t as c, b as M, l as w } from "../index-DSGwv2Yx.js";
+import { O as S, h as b, t as c, a as M, d as w } from "../index-CUXkjxiT.js";
 import R from "./LRScheduler.js";
 import { clipScale as f } from "../ops/globalNorm.js";
 import { save_safetensors as v, load_safetensors as A } from "../utilities/safetensors.js";
-import { z as O } from "../zeros-Bw0puq_w.js";
+import { z as O } from "../zeros-DvZpK8s6.js";
 class _ extends S {
   constructor(t) {
     super(), this.config = t, this.accBeta1 = t.accBeta1 ?? t.beta1, this.accBeta2 = t.accBeta2 ?? t.beta2, this.learningRate = t.learningRate, this.beta1 = t.beta1, this.beta2 = t.beta2, this.weightDecay = t.weightDecay, this.lossScaling = t.lossScaling, this.clipNorm = t.clipNorm, this.orthGrad = t.orthoGrad ?? !1, t.epsilon === null || t.epsilon === void 0 ? this.epsilon = b().backend.epsilon() : this.epsilon = t.epsilon, this.lrScheduler = new R(t.learningRate, t);

package/dist/training/BasicTrainer.d.ts

@@ -19,6 +19,7 @@ export default class BasicTrainer {
     protected _layerDrop: number;
     protected _dropout: number;
     constructor(model: Model<ModelForwardAttributes>, tokenizer: ITokeniser, optConfig?: Partial<AdamWOptimizerConfig>, optimizer?: AdamWOptimizer);
+    setLossMasking(): void;
     setGradientCheckpointing(enabled: boolean): void;
     setMixedPrecision(enabled: boolean): void;
     setLabelSmoothing(smoothing: number): void;