@mineproness/tensorloop 1.5.5 → 1.5.7

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mineproness/tensorloop",
-  "version": "1.5.5",
+  "version": "1.5.7",
   "description": "Raw-JAVASCRIPT Based Matirx and AI LLM Package called Tensorloop.",
   "keywords": [
     "ai",

package/readme.md

@@ -15,6 +15,7 @@ A high-performance JavaScript library that provides essential AI and Large Langu
 - **Text Processing**: Built-in **Tokenizer** to convert raw text into token IDs and back.
 - **Special Tokens**: Native support for `<PAD>` and `<EOS>`.
 - **FFN BLOCK**: It add a layer to understand Complex Patterns.
+- ***GPU SUPPORT** GPU Can support it without any configure
 ---
 
 ## 🏗 Core Architecture

package/src/GPU.js

@@ -23,8 +23,14 @@ export class GPULinear {
     cacheLBIas = [];
     cacheLBWeights = [];
     cacheLBInputGradent = [];
-    InitKernel(weights, size) {
+    // texures
+    Bias = [];
+    Weights = []
+    InitKernel(weights, bias, size) {
         const splitedWeights = splitMatrix(weights)
+        const splitedBias = splitMatrix(bias)
+        this.Weights = splitedWeights.data
+        this.Bias = splitedBias.data
         for (let a = 0; a < splitedWeights.data.length; a++) {
             this.cacheLF[a] = gpu.createKernel(function (input, bias, weights) {
                 let sum = bias[this.thread.x]
@@ -50,8 +56,11 @@ export class GPULinear {
                 )
 
                 return bias[this.thread.x] - learningRate * grad
+            }, {
+                pipeline: true,
+                immutable: true,
+                output: [splitedWeights.data[a].length]
             })
-                .setOutput([splitedWeights.data[a].length])
             this.cacheLBInputGradent[a] = gpu.createKernel(function (
                 weights,
                 outputGradient
@@ -68,11 +77,11 @@ export class GPULinear {
                 }
 
                 return sum
+            }, {
+                constants: { outputSize: splitedWeights.data[a].length },
+                output: [size]
             })
-                .setConstants({
-                    outputSize: splitedWeights.data[a].length
-                })
-                .setOutput([size])
+                
             this.cacheLBWeights[a] = gpu.createKernel(function (
                 weights,
                 outputGradient,
@@ -86,74 +95,52 @@ export class GPULinear {
 
                 return weights[this.thread.y][this.thread.x]
                     - learningRate * input[this.thread.x] * grad
+            }, {
+                pipeline: true,
+                immutable: true,
+                output: [size, splitedWeights.data[a].length]
             })
-                .setOutput([size, splitedWeights.data[a].length])
         }
 
 
 
-
     }
-    CalculateLinearForward(weights, bias, input) {
-        // console.log(input)
-        const splitedBias = splitMatrix(bias)
-        const splitedWeights = splitMatrix(weights)
-        const result = new Float32Array(weights.length)
+    CalculateLinearForward(input) {
+        const result = []
         let offset = 0;
-        for (let a = 0; a < splitedBias.data.length; a++) {
-            const output = this.cacheLF[a](input, splitedBias.data[a], splitedWeights.data[a])
-            result.set(output, offset)
-            offset += output.length
+        for (let a = 0; a < this.Weights.length; a++) {
+            const output = this.cacheLF[a](input, this.Bias[a], this.Weights[a])
+            result.push(...output)
+
         }
         return result
     }
-    CalculateBackward(weights, bias, out, learningRate, input) {
-        const splitedBias = splitMatrix(bias)
-        const splitedWeights = splitMatrix(weights)
-        const inputGradient = new Float32Array(input.length)
-        const biasOut = new Float32Array(bias.length)
-        const weightsOut = []
-        let offsetBias = 0
-        let offsetWeights = 0
-        let gradientOffset = 0
-
-        for (let a = 0; a < splitedWeights.data.length; a++) {
-            const weightChunk = splitedWeights.data[a]
-            const biasChunk = splitedBias.data[a]
-            const chunkOutputSize = weightChunk.length
-            const gradientChunk = out.slice(
-                gradientOffset,
-                gradientOffset + chunkOutputSize
-            )
-            gradientOffset += chunkOutputSize
-            const newBias = this.cacheLBIas[a](
-                biasChunk,
-                gradientChunk,
-                learningRate
-            )
-            biasOut.set(newBias, offsetBias)
-            offsetBias += newBias.length
-            const outputGra = this.cacheLBInputGradent[a](
-                weightChunk,
-                gradientChunk
-            )
-            for (let i = 0; i < outputGra.length; i++) {
-                inputGradient[i] += outputGra[i]
+    CalculateBackward(out, learningRate, input) {
+        let inputGradient = new Float32Array(input.length).fill(0)
+        for (let a = 0; a < this.Weights.length; a++) {
+            const outGradient = this.cacheLBInputGradent[a](this.Weights[a], out)
+            for (let i = 0; i < outGradient.length; i++) {
+                inputGradient[i] += outGradient[i]
+            }
+            const oldW = this.Weights[a]
+            const oldB = this.Bias[a]
+            this.Weights[a] = this.cacheLBWeights[a](this.Weights[a], out, input, learningRate)
+            this.Bias[a] = this.cacheLBIas[a](this.Bias[a], out, learningRate)
+            if(oldW && typeof oldW.delete === "function"){
+                oldW.delete()
+            }
+             if(oldB && typeof oldB.delete === "function"){
+                oldB.delete()
             }
-            const newWeights = this.cacheLBWeights[a](
-                weightChunk,
-                gradientChunk,
-                input,
-                learningRate
-            )
-            weightsOut.push(...newWeights)
-
-        }
 
-        return {
-            inputGradient,
-            biasOut,
-            weightsOut
         }
+        return inputGradient
+    }
+    GetWeightsAndBias(){
+        const weight = []
+        const bias = []
+        this.Weights.map((e)=> weight.push(...e.toArray()))
+        this.Bias.map((e)=> bias.push(...e.toArray()))
+        return { weights: weight, bias: new Float32Array(bias)}
     }
 }

package/src/Linear/index.js

@@ -41,11 +41,12 @@ export class Linear {
             this.Bias = Bias;
         }
         this.gpu = new GPULinear()
-        this.gpu.InitKernel(this.Weights, embeddingSize)
+        this.gpu.InitKernel(this.Weights, this.Bias, embeddingSize)
     }
     Save() {
-        SaveVectors(this.Weights, this.configs.save.filename[0])
-        SaveBias(this.Bias, this.configs.save.filename[1])
+        const {bias, weights} = this.gpu.GetWeightsAndBias()
+        SaveVectors(weights, this.configs.save.filename[0])
+        SaveBias(bias, this.configs.save.filename[1])
     }
     forward(input) {
         this.input.push(input)
@@ -54,9 +55,7 @@ export class Linear {
     backward(outputGradient, learningRate, idx) {
         const input = this.input[idx];
         const weights = this.Weights
-        const {biasOut,inputGradient, weightsOut} = this.gpu.CalculateBackward(weights, this.Bias, outputGradient, learningRate, input)
-        this.Weights = weightsOut;
-        this.Bias = biasOut
+        const inputGradient = this.gpu.CalculateBackward(outputGradient, learningRate, input)
         return inputGradient
     }
     ClearInputCache(){