@mineproness/tensorloop 2.1.70 → 2.1.72

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mineproness/tensorloop",
-  "version": "2.1.70",
+  "version": "2.1.72",
   "description": "Raw-JAVASCRIPT Based Matirx and AI LLM Package called Tensorloop.",
   "keywords": [
     "ai",

package/src/Model/index.js

@@ -46,7 +46,7 @@ export class Model {
             })
         }
         this.posEmbedding = new PositionEmbedding(this.device, this.dimSize, this.configs.contextSize, {
-            save:{
+            save: {
                 filename: join(this.configs.dirname, "posVec.bin")
             }
         })
@@ -61,7 +61,7 @@ export class Model {
         }
     }
     async forward(input, readBack) {
-        if(this.configs.contextSize < input.length) throw new Error("The Input is larger then contextSize")
+        if (this.configs.contextSize < input.length) throw new Error("The Input is larger then contextSize")
         this.tokens = input;
         await this.ClearInputCache()
         let vec = await this.embedding.forward(input)
@@ -77,7 +77,7 @@ export class Model {
         const loss = await this.softmax.crossEntropy(outputGradient, target)
         const dLogits = await this.softmax.backward(outputGradient, target);
         let DInput = await this.linear.backward(dLogits, lr);
-        for (let a = this.blockSize - 1; a !== 0; a--) {
+        for (let a = this.blockSize - 1; a >= 0; a--) {
             DInput = await this.blocks[a].backward(DInput, lr);
         }
         this.embedding.backward(this.tokens, DInput, lr);
@@ -87,7 +87,7 @@ export class Model {
     async Save() {
         await this.embedding.Save()
         await this.linear.Save()
-        for (let a = this.blockSize - 1; a !== 0; a--) {
+        for (let a = this.blockSize - 1; a >= 0; a--) {
             await this.blocks[a].Save();
         }
         await this.posEmbedding.Save()

package/src/RMSNorm/index.js

@@ -49,7 +49,7 @@ export class RMSNorm {
     const output = await this.op.createBuffer(input.size);
     const bindGroup = await this.op.createBindGroup(this.reductionShader, 0, [input, outputReduce])
     const encoder = await this.op.CreateEncoder()
-    await this.op.RunPipline(encoder, this.reductionShader, bindGroup, Math.ceil((input.size / 4) / 64))
+    await this.op.RunPipline(encoder, this.reductionShader, bindGroup, Math.ceil(input.size / (this.embeddingSize * 4)))
     const bindGroup1 = await this.op.createBindGroup(this.forwardShader, 0, [outputReduce, input, output, this.gammabuffer])
     await this.op.RunPipline(encoder, this.forwardShader, bindGroup1, Math.ceil((input.size / 4) / 64));
     await this.op.submitQueue(encoder);
@@ -68,7 +68,7 @@ export class RMSNorm {
     const bindGroup = await this.op.createBindGroup(this.backwardShader, 0, [this.input, outputGradient, this.rms, this.gammabuffer, output]);
     const bindGroup2 = await this.op.createBindGroup(this.updateGamma, 0, [this.input, outputGradient, this.rms, this.gammabuffer, params]);
     const encoder = await this.op.CreateEncoder()
-    await this.op.RunPipline(encoder, this.backwardShader, bindGroup, Math.ceil((this.input.size / 4) / 64))
+    await this.op.RunPipline(encoder, this.backwardShader, bindGroup, Math.ceil((this.input.size) / (this.embeddingSize * 4)))
     await this.op.RunPipline(encoder, this.updateGamma, bindGroup2, this.embeddingSize)
     await this.op.submitQueue(encoder)
     if (this.prevOut) this.prevOut.destroy()

package/src/TransformerBlock/index.js

@@ -79,7 +79,7 @@ export class TransformerBlock {
     async Save() {
         await this.ffn.Save();
         await this.attention.Save();
-        await tthis.rmsnorm.Save()
+        await this.rmsnorm.Save()
         await this.rmsnorm2.Save()
     }
 }

package/src/shaders/Norm/rmsBackward.wgsl

@@ -13,9 +13,9 @@ var<storage, read> gamma: array<f32>;
 @group(0) @binding(4)
 var<storage, read_write> gradInput: array<f32>;
 
-var<workgroup> partial: array<f32, %embeddingSize%>;
+var<workgroup> partial: array<f32, 64>;
 
-@compute @workgroup_size(%embeddingSize%)
+@compute @workgroup_size(64)
 fn main(@builtin(local_invocation_index) lid: u32, @builtin(workgroup_id) wid: vec3<u32>) {
     let D = %embeddingSize%u;
     let token = wid.x;