@mineproness/tensorloop 2.1.61 → 2.1.63

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mineproness/tensorloop",
-  "version": "2.1.61",
+  "version": "2.1.63",
   "description": "Raw-JAVASCRIPT Based Matirx and AI LLM Package called Tensorloop.",
   "keywords": [
     "ai",

package/src/Embedding/index.js

@@ -1,4 +1,5 @@
-// import { config } from "process";
+import { config } from "process";
+
 import { SaveVector, getVector } from '../getVec.js'
 import { OperationManager } from "../OperationManager.js";
 
@@ -130,7 +131,7 @@ export class Embedding {
                 }
             ]
         })
-        await this.gpu.WriteBuffer(this.tokenBuffer, new Float32Array(token));
+        await this.gpu.WriteBuffer(this.tokenBuffer, new Uint32Array(token));
         await this.gpu.RunPipline(encoder, this.backwardPipline, bind, Math.ceil((this.embeddingSize * token.length) / 256))
         this.device.queue.submit([encoder.finish()])
         inputGradient.destroy()

package/src/Softmax/index.js

@@ -152,7 +152,7 @@ export class Softmax {
                 },
             ]
         })
-        await this.gpu.RunPipline(encoder, this.lossPipline, bind, Math.ceil((this.vocabSize * tokenCount) / 256))
+        await this.gpu.RunPipline(encoder, this.lossPipline, bind, Math.ceil((tokenCount / 4)))
         const outputLoss = await this.gpu.createBuffer(4)
         const reduceBind = await this.gpu.createBindGroup(this.reduceFn, 0, [loss, outputLoss, this.paramsForward])
         await this.gpu.RunPipline(encoder, this.reduceFn, reduceBind, 1);

package/src/shaders/CalculateLoss.wgsl

@@ -5,7 +5,7 @@ struct Params {
 
 @group(0) @binding(0)
 
-var <storage, read> probs : vec2<f32>;
+var <storage, read> probs : array<f32>;
 
 @group(0) @binding(1)
 
@@ -13,14 +13,14 @@ var <storage, read> targetToken : array<f32>;
 
 @group(0) @binding(2)
 
-var <storage, read_write> loss : vec2<f32>;
+var <storage, read_write> loss : array<f32>;
 
 @group(0) @binding(3)
 
 var <uniform> params: Params;
 
 
-@compute @workgroup_size(256)
+@compute @workgroup_size(4)
 
 fn main(@builtin(global_invocation_id) id : vec3<u32>) {
       let index = id.x;
@@ -30,6 +30,6 @@ fn main(@builtin(global_invocation_id) id : vec3<u32>) {
     let end = start + params.vocabSize;
     var targetIndex = targetToken[token];
     let total: u32 = params.tokenCount * params.vocabSize;
-    let probalitils = probs[u32(targetIndex)];
+    let probalitils = probs[token * params.vocabSize + u32(targetIndex)];
     loss[index] = -log(probalitils + 1e-15);
 }

package/src/shaders/ComputeDInput.wgsl

@@ -28,5 +28,5 @@ fn main(@builtin(global_invocation_id) id: vec3<u32>) {
         let grad = outGradient[token * u32(params.vocabSize) + i];
         gradInput += grad * w;
     }
-    dInput[index] = max(-1.0, min(1.0,gradInput));
+    dInput[index] = max(-7.0, min(7.0,gradInput));
 }

package/src/shaders/EB.wgsl

@@ -1,31 +1,33 @@
-struct Params{
+struct Params {
     lr: f32
 }
 
-
 @group(0) @binding(0)
 
-var <storage, read_write> embeddings: array<f32>;
+var<storage, read_write> embeddings: array<f32>;
 
 @group(0) @binding(1)
 
-var <storage, read_write> outputGradient: array<f32>;
+var<storage, read_write> outputGradient: array<f32>;
 
 @group(0) @binding(2)
 
-var <storage, read_write> tokens: array<u32>;
+var<storage, read_write> tokens: array<u32>;
 
 @group(0) @binding(3)
-var <uniform> params : Params;
+var<uniform> params: Params;
 
 const embeddingSize = %embeddingSize%u;
 
 @compute @workgroup_size(256)
 
-fn main(@builtin(global_invocation_id) id : vec3<u32>){
+fn main(@builtin(global_invocation_id) id: vec3<u32>) {
     let dim = id.x % embeddingSize;
+    if (id.x >= arrayLength(&outputGradient)) {
+        return;
+    }
     let tokenIndex = id.x / embeddingSize;
     let tokenID = tokens[tokenIndex];
     let embeddingIDx = tokenID * embeddingSize + dim;
-    embeddings[embeddingIDx] -=  params.lr * outputGradient[id.x];
+    embeddings[embeddingIDx] -= params.lr * outputGradient[id.x];
 }

package/src/shaders/LB.wgsl

@@ -44,14 +44,14 @@ fn main(@builtin(global_invocation_id) id: vec3<u32>) {
         let dW = inputValue * gradient;
         dWeights += f32(dW);
     }
-    let vClap = weight[index] - params.lr * max(- 3.0, min(3.0, dWeights));
+    let vClap = weight[index] - params.lr * max(- 7.0, min(7.0, dWeights));
     weight[index] = vClap;
     if (inputIndex == 0u) {
         var dBias = 0.0;
         for (var token = 0u; token < u32(params.tokenCount); token++) {
             dBias += f32(outGradient[token * u32(params.vocabSize) + outputIndex]);
         }
-        bias[outputIndex] = bias[outputIndex] - params.lr * max(- 3.0, min(3.0, dBias));
+        bias[outputIndex] = bias[outputIndex] - params.lr * max(- 7.0, min(7.0, dBias));
 
     }
 }