@mineproness/tensorloop 2.2.4 → 2.2.6

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mineproness/tensorloop",
-  "version": "2.2.4",
+  "version": "2.2.6",
   "description": "Raw-JAVASCRIPT Based Matirx and AI LLM Package called Tensorloop.",
   "keywords": [
     "ai",

package/src/Softmax/index.js

@@ -119,7 +119,7 @@ export class Softmax {
         if(!this.acc) this.acc = await this.gpu.createBuffer(4 * 1)
         await this.gpu.WriteBuffer(targetBuffer, new Uint32Array(target))
         const bindGroup1 = await this.gpu.createBindGroup(this.lossPipline, 0, [probs.mainBuffer, targetBuffer, this.loss, this.paramsForward])
-        const bindGroup2 = await this.gpu.createBindGroup(this.accPipline, 0 , [this.paramsForward, probs.mainBuffer, targetBuffer, this.acc])
+        const bindGroup2 = await this.gpu.createBindGroup(this.accPipline, 0 , [this.paramsForward, probs.mainBuffer, this.acc, targetBuffer])
         await this.gpu.RunPipline(encoder, this.lossPipline, bindGroup1, Math.ceil((tokenCount / 4)))
         await this.gpu.RunPipline(encoder, this.accPipline, bindGroup2, 1)
         await this.gpu.submitQueue(encoder)

package/src/shaders/LB.wgsl

@@ -44,14 +44,14 @@ fn main(@builtin(global_invocation_id) id: vec3<u32>) {
         let dW = inputValue * gradient;
         dWeights += dW;
     }
-    let vClap = weight[index] - params.lr * clamp(dWeights, -8.0, 8.0);
+    let vClap = weight[index] - params.lr * clamp(dWeights, -1.0, 1.0);
     weight[index] = vClap;
     if (inputIndex == 0u) {
         var dBias = 0.0;
         for (var token = 0u; token < u32(params.tokenCount); token++) {
             dBias += outGradient[token * u32(params.vocabSize) + outputIndex];
         }
-        bias[outputIndex] = bias[outputIndex] - params.lr * clamp(dBias, -8.0, 8.0);
+        bias[outputIndex] = bias[outputIndex] - params.lr * clamp(dBias, -1.0, 1.0);
 
     }
 }