@genai-fi/nanogpt 0.16.1 → 0.17.0

package/dist/models/model.js

@@ -1,4 +1,4 @@
-import r from "../layers/BaseLayer.js";
+import i from "../layers/BaseLayer.js";
 import "../index-CUXkjxiT.js";
 import "../random_width-DN5ZtQkM.js";
 import "../zeros_like-CWjDdwr-.js";
@@ -40,7 +40,7 @@ import "../ops/webgl/adamAdjust.js";
 import "../ops/cpu/adamMoments.js";
 import "../ops/webgl/adamMoments.js";
 import "../papaparse.min-C0cScC2i.js";
-import { estimateParameterCount as o } from "../utilities/parameters.js";
+import { estimateParameterCount as a } from "../utilities/parameters.js";
 import "../ops/cpu/matMulGelu.js";
 import "../matMulGelu-JNLZqKQp.js";
 import "../ops/grads/matMulGelu.js";
@@ -50,37 +50,75 @@ import "../gelu-B220X1Go.js";
 import "../ops/webgl/log.js";
 import "../checks/normRMS.js";
 import "../checks/normRMSGrad.js";
-import i from "../layers/LoRA.js";
-class et extends r {
+import e from "../layers/LoRA.js";
+class st extends i {
   lossScaling = 128;
   trainingState = null;
   metaData = { version: 2, application: "@genai-fi/nanogpt" };
   loraLayer;
-  /*constructor(config: GPTConfig) {
-      super(config);
-      if (config.loraConfig) {
-          console.log('Attaching LoRA layer with config:', config.loraConfig);
-          this.attachLoRA(config.loraConfig);
-      }
-  }*/
+  loraMap = /* @__PURE__ */ new Map();
+  constructor(t) {
+    super(t), t.loraConfig && (console.log(t.loraConfig), t.loraConfig.forEach((r, o) => {
+      this.createLoRA(o, r);
+    }));
+  }
+  createLoRA(t, r) {
+    if (this.loraMap.has(t))
+      return;
+    const o = new e(t, this.weightStore, r.alpha, r.rank, r.variables);
+    this.loraMap.set(t, o), this.config.loraConfig = this.config.loraConfig || /* @__PURE__ */ new Map(), this.config.loraConfig.set(t, r), console.log(`Created LoRA ${t} with rank ${r.rank} and alpha ${r.alpha}`);
+  }
+  deleteLoRA(t) {
+    const r = this.loraMap.get(t);
+    if (!r)
+      throw new Error(`No LoRA with name ${t} exists.`);
+    this.loraLayer === r && this.detachLoRA(), r.dispose(), this.loraMap.delete(t), this.config.loraConfig && this.config.loraConfig.delete(t);
+  }
+  renameLoRA(t, r) {
+    if (!this.loraMap.has(t))
+      throw new Error(`No LoRA with name ${t} exists.`);
+    if (this.loraMap.has(r))
+      throw new Error(`LoRA with name ${r} already exists.`);
+    const o = this.loraMap.get(t);
+    this.loraMap.set(r, o), this.loraMap.delete(t), this.config.loraConfig && (this.config.loraConfig.delete(t), this.config.loraConfig.set(r, {
+      rank: o.rank,
+      alpha: o.alpha,
+      variables: Array.from(o.variables)
+    }));
+  }
+  mergeLoRA(t) {
+    const r = this.loraMap.get(t);
+    if (!r)
+      throw new Error(`No LoRA with name ${t} exists.`);
+    r.merge(), this.deleteLoRA(t);
+  }
   attachLoRA(t) {
-    if (this.loraLayer)
-      throw new Error("LoRA is already attached to this model.");
-    this.config.loraConfig = t, this.loraLayer = new i(this.weightStore, t.alpha, t.rank, t.variables);
+    if (this.loraLayer) {
+      if (this.loraLayer.name === t)
+        return;
+      this.detachLoRA();
+    }
+    const r = this.loraMap.get(t);
+    if (!r)
+      throw new Error(`No LoRA with name ${t} exists.`);
+    r.attach(), this.loraLayer = r, this.config.loraName = t;
   }
   detachLoRA() {
     if (!this.loraLayer)
       throw new Error("No LoRA layer is attached to this model.");
-    this.loraLayer.dispose(), this.loraLayer = void 0, delete this.config.loraConfig;
+    this.loraLayer.detach(), this.loraLayer = void 0, this.config.loraName = void 0;
+  }
+  hasLoRA(t) {
+    return t ? this.loraMap.has(t) : !!this.loraLayer;
   }
-  hasLoRA() {
-    return !!this.loraLayer;
+  listLoRAs() {
+    return Array.from(this.loraMap.keys());
   }
   get lora() {
     return this.loraLayer || null;
   }
   getNumParams() {
-    return o(this.config);
+    return a(this.config);
   }
   validateInput(t) {
     if (t.shape.length !== 2)
@@ -92,5 +130,5 @@ class et extends r {
   }
 }
 export {
-  et as default
+  st as default
 };

package/dist/training/types.d.ts

@@ -51,6 +51,7 @@ export interface TrainingOptions extends Partial<AdamWOptimizerConfig> {
     mixedPrecision?: boolean;
     trainableWeights?: string[];
     loraConfig?: LoRAConfig;
+    loraName?: string;
     sftMode: 'full' | 'lora' | 'last-layer';
     maskedLoss?: boolean;
     metrics?: TrainingMetrics[];

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@genai-fi/nanogpt",
-    "version": "0.16.1",
+    "version": "0.17.0",
     "type": "module",
     "main": "dist/main.js",
     "types": "dist/main.d.ts",
@@ -58,6 +58,7 @@
         "papaparse": "^5.5.3",
         "patch-package": "^8.0.1",
         "pdfjs-dist": "^5.4.149",
-        "picomatch": "^4.0.3"
+        "picomatch": "^4.0.3",
+        "uuid": "^14.0.0"
     }
 }