@genai-fi/nanogpt 0.16.1 → 0.17.0

package/dist/Trainer.js

@@ -1,9 +1,29 @@
-import { E as g } from "./index-DvYrXKkX.js";
-import o from "./training/PreTrainer.js";
-import { createTrainValidationSplit as d } from "./training/validation.js";
-import h from "./training/SFTTrainer.js";
-import p from "./training/tasks/splitter.js";
-class l extends g {
+import { E as f } from "./index-DvYrXKkX.js";
+import l from "./training/PreTrainer.js";
+import { createTrainValidationSplit as p } from "./training/validation.js";
+import g from "./training/SFTTrainer.js";
+import m from "./training/tasks/splitter.js";
+const n = [];
+for (let a = 0; a < 256; ++a)
+  n.push((a + 256).toString(16).slice(1));
+function u(a, t = 0) {
+  return (n[a[t + 0]] + n[a[t + 1]] + n[a[t + 2]] + n[a[t + 3]] + "-" + n[a[t + 4]] + n[a[t + 5]] + "-" + n[a[t + 6]] + n[a[t + 7]] + "-" + n[a[t + 8]] + n[a[t + 9]] + "-" + n[a[t + 10]] + n[a[t + 11]] + n[a[t + 12]] + n[a[t + 13]] + n[a[t + 14]] + n[a[t + 15]]).toLowerCase();
+}
+const w = new Uint8Array(16);
+function S() {
+  return crypto.getRandomValues(w);
+}
+function c(a, t, i) {
+  return crypto.randomUUID ? crypto.randomUUID() : D(a);
+}
+function D(a, t, i) {
+  a = a || {};
+  const e = a.random ?? a.rng?.() ?? S();
+  if (e.length < 16)
+    throw new Error("Random bytes length must be >= 16");
+  return e[6] = e[6] & 15 | 64, e[8] = e[8] & 63 | 128, u(e);
+}
+class d extends f {
   trainer;
   trainingType = "pretraining";
   hasTrained = !1;
@@ -18,21 +38,21 @@ class l extends g {
     logInterval: 10
   };
   tokenizer;
-  constructor(t, i, e = "pretraining", a) {
-    if (super(), t instanceof l) {
-      const r = i || t.options, n = t.options;
-      let s = !1;
-      t.trainingType === "sft" && r.sftMode !== n.sftMode && (s = !0), e !== t.trainingType && (s = !0), s ? (t.trainingType === "sft" ? this.trainer = new h(t.model, t.tokenizer, r) : this.trainer = new o(t.model, t.tokenizer, r), this.trainingType = e, this.options = r, this.tokenizer = t.tokenizer) : (this.trainer = t.trainer, this.trainingType = e, this.options = r, this.trainer.updateOptimizer(this.options), this.log = t.log, this.progress = t.progress, this.totalSamples = t.totalSamples, this.tokenizer = t.tokenizer, r.batchSize === n.batchSize && (this.trainDataset = t.trainDataset, this.validationDataset = t.validationDataset));
+  constructor(t, i, e = "pretraining", r) {
+    if (super(), t instanceof d) {
+      const s = i || t.options, h = t.options;
+      let o = !1;
+      t.trainingType === "sft" && s.sftMode !== h.sftMode && (o = !0), e !== t.trainingType && (o = !0), o ? (t.trainingType === "sft" ? this.trainer = new g(t.model, t.tokenizer, s) : this.trainer = new l(t.model, t.tokenizer, s), this.trainingType = e, this.options = s, this.tokenizer = t.tokenizer) : (this.trainer = t.trainer, this.trainingType = e, this.options = s, this.trainer.updateOptimizer(this.options), this.log = t.log, this.progress = t.progress, this.totalSamples = t.totalSamples, this.tokenizer = t.tokenizer, s.batchSize === h.batchSize && (this.trainDataset = t.trainDataset, this.validationDataset = t.validationDataset));
       return;
     }
     if (!i)
       throw new Error("Tokeniser must be provided when initializing Trainer with a model");
     if (!t)
       throw new Error("Model must be provided when initializing Trainer");
-    this.options = a || {
+    this.options = r || {
       batchSize: 32,
       sftMode: "full"
-    }, e === "sft" ? this.trainer = new h(t, i, a) : this.trainer = new o(t, i, a), this.trainingType = e, this.tokenizer = i;
+    }, e === "sft" ? this.trainer = new g(t, i, r) : this.trainer = new l(t, i, r), this.trainingType = e, this.tokenizer = i;
   }
   get model() {
     return this.trainer.model;
@@ -81,41 +101,70 @@ class l extends g {
   }
   async prepare(t = []) {
     const i = this.options;
-    if (this.trainingType === "pretraining" && this.trainer instanceof o) {
-      const { trainDataset: e, validationDataset: a, size: r, trainState: n } = await d(
+    if (this.trainingType === "pretraining" && this.trainer instanceof l) {
+      const { trainDataset: e, validationDataset: r, size: s, trainState: h } = await p(
         t,
         this.trainer.tokenizer,
         this.trainer.datasetBuilder,
         i?.batchSize || 32,
         i?.validationSplit || 0.1
-      ), s = r * (1 - (i?.validationSplit || 0));
-      this.trainDataset = e, this.validationDataset = a, this.totalSamples = s, this.options.epochSteps = Math.ceil(n.shuffledIndexes.length / (i?.batchSize || 32)), this.trainer.updateOptimizer(this.options);
-    } else if (this.trainingType === "sft" && this.trainer instanceof h) {
+      ), o = s * (1 - (i?.validationSplit || 0));
+      this.trainDataset = e, this.validationDataset = r, this.totalSamples = o, this.options.epochSteps = Math.ceil(h.shuffledIndexes.length / (i?.batchSize || 32)), this.trainer.updateOptimizer(this.options);
+    } else if (this.trainingType === "sft" && this.trainer instanceof g) {
       if (t instanceof Uint16Array)
         throw new Error("SFT training requires Task[] input");
-      const e = p(t, i?.validationSplit || 0.1), a = await this.trainer.datasetBuilder.createSFTDataset(
-        [e.training],
-        i?.batchSize || 32,
-        -100
-      ), r = await this.trainer.datasetBuilder.createSFTDataset(
-        [e.validation],
-        i?.batchSize || 32,
-        -100
-      );
-      this.validationDataset = r, this.trainDataset = a, this.totalSamples = t.reduce((n, s) => n + s.length, 0), this.options.epochSteps = Math.ceil(this.totalSamples / (i?.batchSize || 32)), this.trainer.updateOptimizer(this.options);
+      if (i?.validationSplit && i.validationSplit > 0) {
+        const e = m(t, i?.validationSplit), r = await this.trainer.datasetBuilder.createSFTDataset(
+          [e.training],
+          i?.batchSize || 32,
+          -100
+        ), s = await this.trainer.datasetBuilder.createSFTDataset(
+          [e.validation],
+          i?.batchSize || 32,
+          -100
+        );
+        this.validationDataset = s, this.trainDataset = r;
+      } else {
+        const e = await this.trainer.datasetBuilder.createSFTDataset(
+          t,
+          i?.batchSize || 32,
+          -100
+        );
+        this.trainDataset = e;
+      }
+      this.totalSamples = t.reduce((e, r) => e + r.length, 0), this.options.epochSteps = Math.ceil(this.totalSamples / (i?.batchSize || 32)), this.trainer.updateOptimizer(this.options);
     }
   }
   configureModel(t) {
     const i = t?.sftMode || "full";
     if (this.trainingType === "pretraining" && (this.trainer.model.hasLoRA() && this.trainer.model.detachLoRA(), this.trainer.model.weightStore.setTrainable(["*"])), this.trainingType === "sft") {
       if (i === "lora") {
-        if (!t?.loraConfig)
-          throw new Error("LoRA configuration must be provided for lora mode");
-        if (this.trainer.model.hasLoRA()) {
-          const e = this.trainer.model.lora;
-          (e.alpha !== t.loraConfig.alpha || e.rank !== t.loraConfig.rank) && (this.trainer.model.detachLoRA(), this.trainer.model.attachLoRA(t.loraConfig));
-        } else
-          this.trainer.model.attachLoRA(t.loraConfig);
+        const e = this.trainer.model;
+        if (t?.loraName)
+          if (e.hasLoRA(t.loraName)) {
+            if (e.attachLoRA(t.loraName), t.loraConfig) {
+              const r = e.lora;
+              (r.alpha !== t.loraConfig.alpha || r.rank !== t.loraConfig.rank) && (e.detachLoRA(), e.deleteLoRA(t.loraName), e.createLoRA(t.loraName, t.loraConfig), e.attachLoRA(t.loraName), console.warn("Resetting LoRA with new configuration."));
+            }
+          } else if (t.loraConfig)
+            e.createLoRA(t.loraName, t.loraConfig), e.attachLoRA(t.loraName);
+          else
+            throw new Error(
+              `LoRA configuration must be provided to create LoRA with name ${t.loraName}`
+            );
+        else if (t?.loraConfig)
+          if (e.hasLoRA()) {
+            const r = e.lora;
+            if (r.alpha !== t.loraConfig.alpha || r.rank !== t.loraConfig.rank) {
+              e.detachLoRA();
+              const s = t.loraName || c();
+              e.createLoRA(s, t.loraConfig), e.attachLoRA(s);
+            }
+          } else {
+            const r = t.loraName || c();
+            e.createLoRA(r, t.loraConfig), e.attachLoRA(r);
+          }
+        else if (!e.hasLoRA()) throw new Error("LoRA configuration must be provided for lora SFT mode");
       } else
         this.trainer.model.hasLoRA() && this.trainer.model.detachLoRA();
       i === "last-layer" ? this.trainer.model.weightStore.setTrainable([
@@ -143,8 +192,8 @@ class l extends g {
             )
           };
           const e = this.listeners("log");
-          for (const a of e)
-            await a(i, this.progress);
+          for (const r of e)
+            await r(i, this.progress);
         }
       },
       this.validationDataset
@@ -155,8 +204,8 @@ class l extends g {
       throw new Error("Dataset not prepared");
     this.hasTrained || this.trainer.setLearningRate(t?.learningRate || 1e-3), this.hasTrained = !0, this.emit("start");
     const { log: i } = await this.trainer.stepDataset(this.trainDataset, t || {}, this.validationDataset), e = this.listeners("log");
-    for (const a of e)
-      await a(i, {
+    for (const r of e)
+      await r(i, {
         lastLog: i,
         progress: i.totalSamples / this.totalSamples,
         remaining: Math.max(0, (this.totalSamples - i.totalSamples) / i.totalSamples * i.duration)
@@ -174,5 +223,5 @@ class l extends g {
   }
 }
 export {
-  l as default
+  d as default
 };

package/dist/layers/LoRA.d.ts

@@ -3,10 +3,12 @@ export default class LoRA {
     private weightStore;
     readonly alpha: number;
     readonly rank: number;
-    private variables;
+    readonly variables: Set<string>;
     private scale;
-    constructor(weightStore: WeightStore, alpha: number, rank: number, variables: string[]);
+    readonly name: string;
+    constructor(name: string, weightStore: WeightStore, alpha: number, rank: number, variables: string[]);
+    attach(): void;
     merge(): void;
-    detach(merge?: boolean): void;
+    detach(): void;
     dispose(): void;
 }

package/dist/layers/LoRA.js

@@ -1,52 +1,58 @@
-import { a as b, t as c } from "../index-CUXkjxiT.js";
+import { a as m, t as n } from "../index-CUXkjxiT.js";
 import { p } from "../index-DmeWGGmS.js";
-import { v as w } from "../variable-Ck482e3n.js";
+import { v as g } from "../variable-Ck482e3n.js";
 import { r as S } from "../random_normal-IBRrha8a.js";
-import { z as m } from "../zeros-DvZpK8s6.js";
-class $ {
+import { z as _ } from "../zeros-DvZpK8s6.js";
+class B {
   weightStore;
   alpha;
   rank;
   variables;
   scale;
-  constructor(e, a, i, r) {
-    this.weightStore = e, this.alpha = a, this.rank = i;
-    const o = p(r), d = e.variableNames.filter((t) => o(t));
-    if (this.variables = new Set(d), this.scale = b(a / i), this.weightStore.onWeightRead)
-      throw new Error("LoRA cannot be applied to a WeightStore that already has a onWeightRead hook.");
-    this.variables.forEach((t) => {
-      const s = this.weightStore.getRawVariable(t), [h, l] = s.shape, g = `${t}_loraA`, n = `${t}_loraB`;
-      if (s.shape.length !== 2) {
+  name;
+  constructor(t, e, a, s, r) {
+    this.name = t, this.weightStore = e, this.alpha = a, this.rank = s;
+    const c = p(r), w = e.variableNames.filter(
+      (i) => c(i) && !i.endsWith("_loraA") && !i.endsWith("_loraB")
+    );
+    this.variables = new Set(w), this.scale = m(a / s), this.variables.forEach((i) => {
+      const o = this.weightStore.getRawVariable(i), [d, b] = o.shape, h = `${i}_${this.name}_loraA`, l = `${i}_${this.name}_loraB`;
+      if (o.shape.length !== 2) {
         console.warn(
-          `LoRA currently only supports 2D weight matrices. Variable ${t} has shape ${s.shape}`
-        ), this.variables.delete(t);
+          `LoRA currently only supports 2D weight matrices. Variable ${i} has shape ${o.shape}`
+        ), this.variables.delete(i);
         return;
       }
-      this.weightStore.hasVariable(g) || this.weightStore.hasVariable(n) || (this.weightStore.addVariable(
-        g,
-        w(S([h, this.rank], 0, 0.02), !0, g)
-      ), this.weightStore.addVariable(n, w(m([this.rank, l]), !0, n)));
-    }), this.weightStore.onWeightRead = (t, s) => this.variables.has(t) ? c(() => {
-      const h = this.weightStore.getRawVariable(`${t}_loraA`), l = this.weightStore.getRawVariable(`${t}_loraB`);
-      return s.add(h.matMul(l).mul(this.scale));
-    }) : s, this.weightStore.setTrainable(["*_loraA", "*_loraB"]);
+      this.weightStore.hasVariable(h) || this.weightStore.hasVariable(l) || (this.weightStore.addVariable(
+        h,
+        g(S([d, this.rank], 0, 0.02), !0, h)
+      ), this.weightStore.addVariable(l, g(_([this.rank, b]), !0, l)));
+    });
+  }
+  attach() {
+    if (this.weightStore.onWeightRead)
+      throw new Error("LoRA cannot be applied to a WeightStore that already has a onWeightRead hook.");
+    this.weightStore.onWeightRead = (t, e) => this.variables.has(t) ? n(() => {
+      const a = this.weightStore.getRawVariable(`${t}_${this.name}_loraA`), s = this.weightStore.getRawVariable(`${t}_${this.name}_loraB`);
+      return e.add(a.matMul(s).mul(this.scale));
+    }) : e, this.weightStore.setTrainable([`*_${this.name}_loraA`, `*_${this.name}_loraB`]);
   }
   merge() {
-    this.variables.forEach((e) => {
-      const a = this.weightStore.getRawVariable(e), i = this.weightStore.getRawVariable(`${e}_loraA`), r = this.weightStore.getRawVariable(`${e}_loraB`), o = c(() => a.add(i.matMul(r).mul(this.scale)));
-      a.assign(o), o.dispose();
+    this.variables.forEach((t) => {
+      const e = this.weightStore.getRawVariable(t), a = this.weightStore.getRawVariable(`${t}_${this.name}_loraA`), s = this.weightStore.getRawVariable(`${t}_${this.name}_loraB`), r = n(() => e.add(a.matMul(s).mul(this.scale)));
+      e.assign(r), r.dispose();
     });
   }
-  detach(e = !1) {
-    e && this.merge(), this.variables.forEach((a) => {
-      const i = `${a}_loraA`, r = `${a}_loraB`;
-      this.weightStore.getRawVariable(i).dispose(), this.weightStore.getRawVariable(r).dispose(), this.weightStore.deleteVariable(i), this.weightStore.deleteVariable(r);
-    }), this.weightStore.onWeightRead = void 0, this.weightStore.setTrainable(["*"]), this.variables.clear();
+  detach() {
+    this.weightStore.onWeightRead = void 0, this.weightStore.setTrainable(["*"]);
   }
   dispose() {
-    this.detach(), this.scale.dispose();
+    this.detach(), this.scale.dispose(), this.variables.forEach((t) => {
+      const e = `${t}_${this.name}_loraA`, a = `${t}_${this.name}_loraB`;
+      this.weightStore.getRawVariable(e).dispose(), this.weightStore.getRawVariable(a).dispose(), this.weightStore.deleteVariable(e), this.weightStore.deleteVariable(a);
+    }), this.variables.clear();
   }
 }
 export {
-  $ as default
+  B as default
 };

package/dist/loader/load.js

@@ -1,68 +1,68 @@
 import { z as i } from "../jszip.min-BZhlzntC.js";
 import u from "./oldZipLoad.js";
-import m from "./newZipLoad.js";
-import d from "./loadHF.js";
+import d from "./newZipLoad.js";
+import w from "./loadHF.js";
 import { loadZipMeta as s } from "./loadZipMeta.js";
-import { load_safetensors as w } from "../utilities/safetensors.js";
+import { load_safetensors as g } from "../utilities/safetensors.js";
 const j = 2;
-async function h(t) {
-  const e = await fetch(t);
-  if (!e.ok)
-    throw new Error(`Failed to fetch ${t}: ${e.statusText}`);
-  return e.arrayBuffer();
+async function h(e) {
+  const t = await fetch(e);
+  if (!t.ok)
+    throw new Error(`Failed to fetch ${e}: ${t.statusText}`);
+  return t.arrayBuffer();
 }
-async function p(t, e, o) {
-  const r = t.file("model.safetensors");
-  if (!r)
+async function p(e, t, o) {
+  const a = e.file("model.safetensors");
+  if (!a)
     return;
-  const n = await r.async("arraybuffer"), c = await w(n), a = /* @__PURE__ */ new Map();
-  for (const [l, g] of Object.entries(c))
-    a.set(l, [g]);
-  e.weightStore.loadWeights(a, o);
+  const r = await a.async("arraybuffer"), c = await g(r), n = /* @__PURE__ */ new Map();
+  for (const [l, m] of Object.entries(c))
+    n.set(l, [m]);
+  t.weightStore.loadWeights(n, o);
 }
-async function y(t, e) {
-  const o = t.file("config.json");
+async function y(e, t) {
+  const o = e.file("config.json");
   if (!o)
     return;
-  const r = await o.async("string"), n = JSON.parse(r);
-  if (n.loraConfig) {
-    if (e.hasLoRA())
+  const a = await o.async("string"), r = JSON.parse(a);
+  if (r.loraName) {
+    if (t.hasLoRA())
       throw new Error("Model already has LoRA attached");
-    e.attachLoRA(n.loraConfig);
+    t.attachLoRA(r.loraName);
   }
 }
-async function f(t, e) {
-  if (e.reference) {
-    const o = await C(e.reference);
-    return await p(t, o.model, !!e.url), await y(t, o.model), o.model.config.loraConfig && o.model.attachLoRA(o.model.config.loraConfig), {
+async function f(e, t) {
+  if (t.reference) {
+    const o = await L(t.reference);
+    return await p(e, o.model, !!t.url), await y(e, o.model), o.model.config.loraName && o.model.attachLoRA(o.model.config.loraName), {
       ...o,
       metaData: {
         ...o.metaData,
-        ...e
+        ...t
       }
     };
   } else {
-    if (t.file("manifest.json"))
-      return u(t, e);
+    if (e.file("manifest.json"))
+      return u(e, t);
     {
-      const o = await m(t, e);
-      return o.model.config.loraConfig && o.model.attachLoRA(o.model.config.loraConfig), o;
+      const o = await d(e, t);
+      return o.model.config.loraName && o.model.attachLoRA(o.model.config.loraName), o;
     }
   }
 }
-async function C(t, e) {
-  if (typeof t == "string")
-    if (t.startsWith("http://") || t.startsWith("https://")) {
-      const o = await h(t), r = await i.loadAsync(o), n = await s(r);
-      return n.url = t, f(r, n);
+async function L(e, t) {
+  if (typeof e == "string")
+    if (e.startsWith("http://") || e.startsWith("https://")) {
+      const o = await h(e), a = await i.loadAsync(o), r = await s(a);
+      return r.url = e, f(a, r);
     } else
-      return d(t);
+      return w(e);
   else {
-    const o = await i.loadAsync(t), r = await s(o);
-    return r.url = e?.sourceURL || void 0, f(o, r);
+    const o = await i.loadAsync(e), a = await s(o);
+    return a.url = t?.sourceURL || void 0, f(o, a);
   }
 }
 export {
   j as VERSION,
-  C as loadModel
+  L as loadModel
 };

package/dist/loader/loadTransformers.js

@@ -1,8 +1,8 @@
 import p from "../tokeniser/CharTokeniser.js";
 import _ from "../tokeniser/bpe.js";
-import { load_safetensors as c } from "../utilities/safetensors.js";
-import { aa as u } from "../index-CUXkjxiT.js";
-import { dummyPassAsync as b } from "../utilities/dummy.js";
+import { load_safetensors as b } from "../utilities/safetensors.js";
+import { aa as c } from "../index-CUXkjxiT.js";
+import { dummyPassAsync as u } from "../utilities/dummy.js";
 import y from "../models/factory.js";
 function h(e) {
   let o;
@@ -23,19 +23,20 @@ function h(e) {
     nHead: e.num_attention_heads,
     nEmbed: e.hidden_size,
     mlpFactor: e.mlpFactor,
-    loraConfig: e.loraConfig,
+    loraConfig: e.loraConfig ? new Map(Object.entries(e.loraConfig)) : void 0,
+    loraName: e.loraName,
     windowSize: e.windowSize
   }, o;
 }
-async function G(e, o, r, t) {
-  const n = h(e), m = (o.type ?? "char") === "char" ? new p(o.vocab) : new _(o.vocab, o.merges), i = await c(t), s = /* @__PURE__ */ new Map();
+async function S(e, o, r, s) {
+  const n = h(e), m = (o.type ?? "char") === "char" ? new p(o.vocab) : new _(o.vocab, o.merges), i = await b(s), t = /* @__PURE__ */ new Map();
   for (const [l, d] of Object.entries(i))
-    s.set(l, [d]);
-  u();
+    t.set(l, [d]);
+  c();
   const a = y(n);
-  return a.metaData = r, await b(a), a.weightStore.loadWeights(s, !!r.url), { model: a, tokeniser: m, metaData: r };
+  return a.metaData = r, await u(a), a.weightStore.loadWeights(t, !!r.url), { model: a, tokeniser: m, metaData: r };
 }
 export {
-  G as default,
+  S as default,
   h as mapTransformersConfigToGPTConfig
 };

package/dist/loader/save.js

@@ -1,19 +1,19 @@
 import { z as p } from "../jszip.min-BZhlzntC.js";
-import _ from "../tokeniser/CharTokeniser.js";
-import { save_safetensors as l } from "../utilities/safetensors.js";
+import l from "../tokeniser/CharTokeniser.js";
+import { save_safetensors as _ } from "../utilities/safetensors.js";
 import { VERSION as b } from "./load.js";
 async function z(e, i, a) {
-  const s = /* @__PURE__ */ new Map();
-  e.weightStore.saveWeights(s);
-  const n = new p(), r = {};
-  s.forEach((t, f) => {
-    t.length === 1 && (r[f] = t[0]);
+  const r = /* @__PURE__ */ new Map();
+  e.weightStore.saveWeights(r);
+  const n = new p(), s = {};
+  r.forEach((o, f) => {
+    o.length === 1 && (s[f] = o[0]);
   });
-  const c = await l(r);
+  const c = await _(s);
   n.file("model.safetensors", c, { binary: !0 });
   const g = e.config.modelType;
-  let o;
-  if (g === "GenAI_NanoGPT_v1" ? o = {
+  let t;
+  if (g === "GenAI_NanoGPT_v1" ? t = {
     model_type: "GenAI_NanoGPT_v1",
     vocab_size: i.getVocab().length,
     hidden_size: e.config.nEmbed,
@@ -22,7 +22,7 @@ async function z(e, i, a) {
     block_size: e.config.blockSize,
     mlpFactor: e.config.mlpFactor,
     useRope: e.config.useRope
-  } : o = {
+  } : t = {
     model_type: "GenAI_NanoGPT_v2",
     vocab_size: i.getVocab().length,
     hidden_size: e.config.nEmbed,
@@ -30,9 +30,10 @@ async function z(e, i, a) {
     num_attention_heads: e.config.nHead,
     block_size: e.config.blockSize,
     mlpFactor: e.config.mlpFactor,
-    loraConfig: e.config.loraConfig,
+    loraConfig: e.config.loraConfig ? Object.fromEntries(e.config.loraConfig) : void 0,
+    loraName: e.config.loraName,
     windowSize: e.config.windowSize
-  }, n.file("config.json", JSON.stringify(o, void 0, 4), {
+  }, n.file("config.json", JSON.stringify(t, void 0, 4), {
     binary: !1
   }), n.file(
     "meta.json",
@@ -55,7 +56,7 @@ async function z(e, i, a) {
   ), n.file(
     "tokeniser.json",
     JSON.stringify({
-      type: i instanceof _ ? "char" : "bpe",
+      type: i instanceof l ? "char" : "bpe",
       vocab: i.getVocab(),
       merges: await i.getMerges()
     }),
@@ -63,8 +64,8 @@ async function z(e, i, a) {
       binary: !1
     }
   ), a?.files)
-    for (const [t, f] of Object.entries(a.files))
-      n.file(t, JSON.stringify(f), { binary: !1 });
+    for (const [o, f] of Object.entries(a.files))
+      n.file(o, JSON.stringify(f), { binary: !1 });
   return n.generateAsync({ type: "blob" });
 }
 export {

package/dist/loader/types.d.ts

@@ -8,7 +8,8 @@ export interface TransformersConfigBase {
     num_attention_heads: number;
     block_size: number;
     mlpFactor: number;
-    loraConfig?: LoRAConfig;
+    loraConfig?: Record<string, LoRAConfig>;
+    loraName?: string;
 }
 export interface TransformersConfigV1 extends TransformersConfigBase {
     model_type: 'GenAI_NanoGPT_v1';

package/dist/models/config.d.ts

@@ -11,7 +11,8 @@ export interface GPTConfigBase {
     nHead: number;
     nEmbed: number;
     mlpFactor: number;
-    loraConfig?: LoRAConfig;
+    loraConfig?: Map<string, LoRAConfig>;
+    loraName?: string;
 }
 export interface GPTConfigV1 extends GPTConfigBase {
     modelType: 'GenAI_NanoGPT_v1';

package/dist/models/config.js

@@ -1,4 +1,4 @@
-const a = {
+const t = {
   modelType: "GenAI_NanoGPT_v2",
   vocabSize: 2e3,
   blockSize: 128,
@@ -11,33 +11,40 @@ const a = {
   // Embedding dimension
   mlpFactor: 4
 };
-function e(r, o) {
-  if (typeof r[o] != "number" || Number.isNaN(r[o]))
-    throw new Error(`Invalid config: "${o}" must be a number.`);
+function n(e, a) {
+  if (typeof e[a] != "number" || Number.isNaN(e[a]))
+    throw new Error(`Invalid config: "${a}" must be a number.`);
 }
-function i(r) {
-  const o = (n) => typeof n == "object" && n !== null && !Array.isArray(n);
-  if (!o(r))
+function l(e) {
+  const a = (o) => typeof o == "object" && o !== null && !Array.isArray(o);
+  if (!a(e))
     throw new Error("Invalid config: expected an object.");
-  if (e(r, "vocabSize"), e(r, "blockSize"), e(r, "nLayer"), e(r, "nHead"), e(r, "nEmbed"), e(r, "mlpFactor"), r.loraConfig !== void 0) {
-    if (!o(r.loraConfig))
+  if (n(e, "vocabSize"), n(e, "blockSize"), n(e, "nLayer"), n(e, "nHead"), n(e, "nEmbed"), n(e, "mlpFactor"), e.loraConfig !== void 0) {
+    if (!a(e.loraConfig))
       throw new Error('Invalid config: "loraConfig" must be an object.');
-    if (e(r.loraConfig, "rank"), e(r.loraConfig, "alpha"), !Array.isArray(r.loraConfig.variables) || !r.loraConfig.variables.every((n) => typeof n == "string"))
-      throw new Error('Invalid config: "loraConfig.variables" must be a string array.');
+    const o = Object.values(e.loraConfig);
+    if (!o.every((r) => a(r)))
+      throw new Error('Invalid config: each entry in "loraConfig" must be an object.');
+    if (!o.every((r) => "rank" in r && "alpha" in r && "variables" in r))
+      throw new Error('Invalid config: each LoRA config must have "rank", "alpha", and "variables" fields.');
+    o.forEach((r) => {
+      if (n(r, "rank"), n(r, "alpha"), !Array.isArray(r.variables) || !r.variables.every((i) => typeof i == "string"))
+        throw new Error('Invalid config: "variables" must be a string array.');
+    });
   }
-  if (r.modelType === "GenAI_NanoGPT_v1") {
-    if (typeof r.useRope != "boolean")
+  if (e.modelType === "GenAI_NanoGPT_v1") {
+    if (typeof e.useRope != "boolean")
       throw new Error('Invalid config: "useRope" must be a boolean for GenAI_NanoGPT_v1.');
     return;
   }
-  if (r.modelType === "GenAI_NanoGPT_v2") {
-    if (r.windowSize !== void 0 && typeof r.windowSize != "string")
+  if (e.modelType === "GenAI_NanoGPT_v2") {
+    if (e.windowSize !== void 0 && typeof e.windowSize != "string")
       throw new Error('Invalid config: "windowSize" must be a string for GenAI_NanoGPT_v2.');
     return;
   }
   throw new Error('Invalid config: "modelType" must be "GenAI_NanoGPT_v1" or "GenAI_NanoGPT_v2".');
 }
 export {
-  a as defaultConfig,
-  i as validateConfig
+  t as defaultConfig,
+  l as validateConfig
 };

package/dist/models/model.d.ts

@@ -21,9 +21,16 @@ export default abstract class Model<T extends ModelForwardAttributes, C extends
     trainingState: TrainingState | null;
     metaData: TransformersMetadata;
     private loraLayer?;
-    attachLoRA(loraConfig: LoRAConfig): void;
+    private loraMap;
+    constructor(config: C);
+    createLoRA(name: string, loraConfig: LoRAConfig): void;
+    deleteLoRA(name: string): void;
+    renameLoRA(oldName: string, newName: string): void;
+    mergeLoRA(name: string): void;
+    attachLoRA(name: string): void;
     detachLoRA(): void;
-    hasLoRA(): boolean;
+    hasLoRA(name?: string): boolean;
+    listLoRAs(): string[];
     get lora(): LoRA | null;
     abstract getClassName(): string;
     abstract forward(attrs: T, idx: Tensor): Tensor;