@mineproness/tensorloop 2.1.58 → 2.1.60

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mineproness/tensorloop",
-  "version": "2.1.58",
+  "version": "2.1.60",
   "description": "Raw-JAVASCRIPT Based Matirx and AI LLM Package called Tensorloop.",
   "keywords": [
     "ai",

package/src/Linear/index.js

@@ -42,7 +42,7 @@ export class Linear {
         let total = embeddingSize * vocabSize;
         if (configs && configs.save) {
             const w = getVector(configs.save.filename[0], total);
-            const b = getVector(configs.save.filename[1], total, true)
+            const b = getVector(configs.save.filename[1], vocabSize, true)
             if (!w && !b) {
                 this.bias = GenerateBias(vocabSize)
                 this.weights = GenerateVectors(embeddingSize, vocabSize)

package/src/SelfAttention/index.js

@@ -63,7 +63,6 @@ export class SelfAttention {
         const finalOutput = await this.gpu.createBuffer(tokenCount * this.embeddingSize * 4)
         const outputBind = await this.gpu.createBindGroup(this.outputForward, 0, [this.weights, this.value, finalOutput, this.asParams]);
         await this.gpu.RunPipline(encoder, this.outputForward, outputBind, Math.ceil((tokenCount * this.embeddingSize) / 256))
-
         await this.gpu.submitQueue(encoder)
         scroes.destroy()
         return finalOutput
@@ -114,7 +113,7 @@ export class SelfAttention {
     }
     async Save(){
        await this.q.Save()
-       await this.q.Save()
-       await this.q.Save()
+       await this.k.Save()
+       await this.v.Save()
     }
 }

package/src/getVec.js

@@ -5,21 +5,23 @@ export function getVector(filename, size, bias) {
     if (!fs.existsSync(filename)) {
         return null;
     }
-    const bufferFloat32Array = fs.readFileSync(filename)
-    const r = new Float32Array(bufferFloat32Array.buffer, bufferFloat32Array.byteOffset, bufferFloat32Array.byteLength / 4);
-    if (r.length < size) {
-         const output = new Float32Array(size);
-         for(let a = 0; a < size; a++){
-            let num = r[a];
-            if(num){
-                output[a] = num;
-            }else{
-                output[a] = bias ? 0 : (Math.random() * 2 - 1) * 0.02
-            }
-         }
-    } else {
-        return r
+    const buffer = fs.readFileSync(filename);
+    const r = new Float32Array(
+        buffer.buffer,
+        buffer.byteOffset,
+        buffer.byteLength / 4
+    );
+    if (r.length >= size) {
+        return r;
     }
+    const output = new Float32Array(size);
+    output.set(r);
+    if (!bias) {
+        for (let i = r.length; i < size; i++) {
+            output[i] = (Math.random() * 2 - 1) * 0.02;
+        }
+    }
+    return output;
 }
 
 export function SaveVector(filename, vectors) {

package/src/shaders/SoftmaxBackward.wgsl

@@ -27,5 +27,5 @@ fn main(@builtin(global_invocation_id) id: vec3<u32>) {
     }
     let p = P[index];
     let dp = dP[index];
-    output[index] = clamp(p * (dp - dot), -1.0, 1.0);
+    output[index] = p * (dp - dot);
 }