@mineproness/tensorloop 2.2.6 → 2.2.8

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mineproness/tensorloop",
-  "version": "2.2.6",
+  "version": "2.2.8",
   "description": "Raw-JAVASCRIPT Based Matirx and AI LLM Package called Tensorloop.",
   "keywords": [
     "ai",

package/src/Linear/index.js

@@ -181,7 +181,7 @@ export class Linear {
         })
         // console.log((await this.gpu.readBuffer(this.input.size, this.input))[0])
         await this.gpu.RunPipline(encoder, this.dInputBackward, bind2, Math.ceil((this.embeddingSize * tokenCount) / 256));
-        await this.gpu.RunPipline(encoder, this.backwardPipline, bind, Math.ceil((this.vocabSize * tokenCount) / 256));
+        await this.gpu.RunPipline(encoder, this.backwardPipline, bind, Math.ceil((this.vocabSize * this.embeddingSize) / 256));
         await this.device.queue.submit([encoder.finish()])
         return this.gradientBuffer
 

package/src/RMSNorm/index.js

@@ -47,7 +47,7 @@ export class RMSNorm {
     }
     let context = input.size / (this.embeddingSize * 4)
     if (!this.rms) this.rms = await this.op.createBuffer(4 * context);
-    if (this.rms.size !== 4 * context * this.embeddingSize) {
+    if (this.rms.size !== 4 * context) {
       this.rms.destroy()
       this.rms = await this.op.createBuffer(4 * context);
     }

package/src/shaders/LB.wgsl

@@ -44,14 +44,14 @@ fn main(@builtin(global_invocation_id) id: vec3<u32>) {
         let dW = inputValue * gradient;
         dWeights += dW;
     }
-    let vClap = weight[index] - params.lr * clamp(dWeights, -1.0, 1.0);
+    let vClap = weight[index] - params.lr * clamp(dWeights, -8.0, 8.0);
     weight[index] = vClap;
     if (inputIndex == 0u) {
         var dBias = 0.0;
         for (var token = 0u; token < u32(params.tokenCount); token++) {
             dBias += outGradient[token * u32(params.vocabSize) + outputIndex];
         }
-        bias[outputIndex] = bias[outputIndex] - params.lr * clamp(dBias, -1.0, 1.0);
+        bias[outputIndex] = bias[outputIndex] - params.lr * clamp(dBias, -8.0, 8.0);
 
     }
 }