@mineproness/tensorloop 2.1.63 → 2.1.65

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mineproness/tensorloop",
-  "version": "2.1.63",
+  "version": "2.1.65",
   "description": "Raw-JAVASCRIPT Based Matirx and AI LLM Package called Tensorloop.",
   "keywords": [
     "ai",

package/src/Embedding/index.js

@@ -98,7 +98,7 @@ export class Embedding {
             this.param = await this.gpu.createUniFormBuffer(this.oldParam.byteLength);
             this.tokenBuffer = await this.gpu.createBuffer(4 * token.length)
         }
-        if (this.oldParam[1] !== lr) {
+        if (this.oldParam[0] !== lr) {
             this.oldParam = new Float32Array([lr, 0]);
             this.gpu.WriteBuffer(this.param, this.oldParam)
         }

package/src/Softmax/index.js

@@ -123,7 +123,7 @@ export class Softmax {
         const targetBuffer = await this.gpu.createBuffer(target.length * Float32Array.BYTES_PER_ELEMENT)
         const loss = await this.gpu.createBuffer(4 * tokenCount)
         const acc = await this.gpu.createBuffer(4 * 2)
-        await this.gpu.WriteBuffer(targetBuffer, new Float32Array(target))
+        await this.gpu.WriteBuffer(targetBuffer, new Uint32Array(target))
         const bind = await this.device.createBindGroup({
             layout: await this.lossPipline.getBindGroupLayout(0),
             entries: [

package/src/Tokenizer/index.js

@@ -27,7 +27,7 @@ export class Tokenizer {
         const output = []
         for (let a = 0; a < splitedWords.length; a++) {
             const word = splitedWords[a]
-            if (this.WordtoID[word]) {
+            if (this.WordtoID[word] !== undefined) {
                 output.push(word)
             } else {
                 let max = word.length
@@ -36,7 +36,7 @@ export class Tokenizer {
                     const regPatten = new RegExp(`.{0,${max}}`, "g")
                     const wordsliced = word.match(regPatten).filter((e) => e.length > 0)
                     const matched = wordsliced.flatMap(e => {
-                        if (this.WordtoID[e]) {
+                        if (this.WordtoID[e] !== undefined) {
                             return true
                         }
                         return false

package/src/shaders/CalculateLoss.wgsl

@@ -23,13 +23,8 @@ var <uniform> params: Params;
 @compute @workgroup_size(4)
 
 fn main(@builtin(global_invocation_id) id : vec3<u32>) {
-      let index = id.x;
-    let token = index / params.vocabSize;
-    let vocabIndex = index % params.vocabSize;
-    let start = token * params.vocabSize;
-    let end = start + params.vocabSize;
-    var targetIndex = targetToken[token];
-    let total: u32 = params.tokenCount * params.vocabSize;
-    let probalitils = probs[token * params.vocabSize + u32(targetIndex)];
+    let index = id.x;
+    var targetIndex = targetToken[index];
+    let probalitils = probs[index * params.vocabSize + u32(targetIndex)];
     loss[index] = -log(probalitils + 1e-15);
 }

package/src/shaders/ComputeDInput.wgsl

@@ -28,5 +28,5 @@ fn main(@builtin(global_invocation_id) id: vec3<u32>) {
         let grad = outGradient[token * u32(params.vocabSize) + i];
         gradInput += grad * w;
     }
-    dInput[index] = max(-7.0, min(7.0,gradInput));
+    dInput[index] = gradInput;
 }

package/src/shaders/LB.wgsl

@@ -44,14 +44,14 @@ fn main(@builtin(global_invocation_id) id: vec3<u32>) {
         let dW = inputValue * gradient;
         dWeights += f32(dW);
     }
-    let vClap = weight[index] - params.lr * max(- 7.0, min(7.0, dWeights));
+    let vClap = weight[index] - params.lr * max(- 15.0, min(15.0, dWeights));
     weight[index] = vClap;
     if (inputIndex == 0u) {
         var dBias = 0.0;
         for (var token = 0u; token < u32(params.tokenCount); token++) {
             dBias += f32(outGradient[token * u32(params.vocabSize) + outputIndex]);
         }
-        bias[outputIndex] = bias[outputIndex] - params.lr * max(- 7.0, min(7.0, dBias));
+        bias[outputIndex] = bias[outputIndex] - params.lr * max(- 15.0, min(15.0, dBias));
 
     }
 }

package/src/shaders/RB.wgsl

@@ -32,6 +32,6 @@ fn main(@builtin(global_invocation_id) id: vec3<u32>) {
         output[index] = grad;
     }
     else {
-        output[index] = grad * 0.02;
+        output[index] = 0;
     }
 }

package/src/shaders/RF.wgsl

@@ -28,6 +28,6 @@ fn main(@builtin(global_invocation_id) id: vec3<u32>) {
         output[index] = x;
     }
     else {
-        output[index] = x * 0.02;
+        output[index] = 0;
     }
 }

package/src/shaders/SB.wgsl

@@ -27,6 +27,10 @@ fn main(@builtin(global_invocation_id) id: vec3<u32>) {
     let vocabIndex = index % params.vocabSize;
     let start = token * params.vocabSize;
     let end = start + params.vocabSize;
+    let total = params.tokenCount * params.vocabSize;
+    if (index >= total) {
+      return;
+    }
     var targetIndex = targetToken[token];
     let total: u32 = params.tokenCount * params.vocabSize;
     output[index] = probs[index];