@mineproness/tensorloop 2.1.80 → 2.1.81

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mineproness/tensorloop",
-  "version": "2.1.80",
+  "version": "2.1.81",
   "description": "Raw-JAVASCRIPT Based Matirx and AI LLM Package called Tensorloop.",
   "keywords": [
     "ai",

package/src/RMSNorm/index.js

@@ -69,7 +69,7 @@ export class RMSNorm {
     const bindGroup2 = await this.op.createBindGroup(this.updateGamma, 0, [this.input, outputGradient, this.rms, this.gammabuffer, params]);
     const encoder = await this.op.CreateEncoder()
     await this.op.RunPipline(encoder, this.backwardShader, bindGroup, Math.ceil((this.input.size) / (this.embeddingSize * 4)))
-    await this.op.RunPipline(encoder, this.updateGamma, bindGroup2, Math.ceil(this.embeddingSize / 64))
+    await this.op.RunPipline(encoder, this.updateGamma, bindGroup2, Math.ceil(this.embeddingSize))
     await this.op.submitQueue(encoder)
     if (this.prevOut) this.prevOut.destroy()
     this.prevOut = outputGradient

package/src/shaders/LB.wgsl

@@ -44,14 +44,14 @@ fn main(@builtin(global_invocation_id) id: vec3<u32>) {
         let dW = inputValue * gradient;
         dWeights += dW;
     }
-    let vClap = weight[index] - params.lr * clamp(dWeights, -15.0, 15.0);
+    let vClap = weight[index] - params.lr * clamp(dWeights, -20.0, 20.0);
     weight[index] = vClap;
     if (inputIndex == 0u) {
         var dBias = 0.0;
         for (var token = 0u; token < u32(params.tokenCount); token++) {
             dBias += outGradient[token * u32(params.vocabSize) + outputIndex];
         }
-        bias[outputIndex] = bias[outputIndex] - params.lr * clamp(dBias, -15.0, 15.0);
+        bias[outputIndex] = bias[outputIndex] - params.lr * clamp(dBias, -20.0, 20.0);
 
     }
 }

package/src/shaders/Norm/updateGamma.wgsl

@@ -21,16 +21,15 @@ const EPSILON: f32 = 1e-6;
 
 @compute @workgroup_size(64)
 fn main(@builtin(local_invocation_index) lid: u32, @builtin(workgroup_id) wid: vec3<u32>) {
-    let dim = wid.x * 64u + lid;
+    let dim = wid.x;
     let T = arrayLength(&sumSquares);
     var sum = 0.0;
     if (dim < D) {
         var token = lid;
-
         while (token < T) {
             let idx = token * D + dim;
-            let r = sqrt(sumSquares[token] / f32(D) + EPSILON);
-            sum += gradOutput[idx] * input[idx] / r;
+            let rms = sqrt(sumSquares[token] / f32(D) + EPSILON);
+            sum += gradOutput[idx] * input[idx] / rms;
             token += 64u;
         }
     }
@@ -47,8 +46,7 @@ fn main(@builtin(local_invocation_index) lid: u32, @builtin(workgroup_id) wid: v
         workgroupBarrier();
         stride /= 2u;
     }
-    if (lid == 0u && wid.x * 64u < D) {
-        let dimToUpdate = wid.x * 64u;
-        gamma[dimToUpdate] -= params[0] * partial[0];
+    if (lid == 0u && dim < D) {
+        gamma[dim] -= params[0] * partial[0];
     }
-}
+}