@mineproness/tensorloop 1.5.4 → 1.5.6

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mineproness/tensorloop",
-  "version": "1.5.4",
+  "version": "1.5.6",
   "description": "Raw-JAVASCRIPT Based Matirx and AI LLM Package called Tensorloop.",
   "keywords": [
     "ai",

package/src/GPU.js

@@ -18,150 +18,115 @@ export function splitMatrix(matrix, chunksSizes = gl.getParameter(gl.MAX_TEXTURE
     return { split: true, data: chunks }
 
 }
-
-const cacheLF = []
-const cacheLBIas = []
-const cacheLBWeights = []
-const cacheLBInputGradent = []
-export function InitKernel(weights, size) {
-    const splitedWeights = splitMatrix(weights)
-    for (let a = 0; a < splitedWeights.data.length; a++) {
-        cacheLF[a] = gpu.createKernel(function (input, bias, weights) {
-            let sum = bias[this.thread.x]
-            for (let a = 0; a < this.constants.inputLength; a++) {
-                sum += input[a] * weights[this.thread.x][a]
-            }
-            return sum
-        }).setConstants({
-            inputLength: splitedWeights.data[a].length
-        }).setOutput([splitedWeights.data[a].length])
-        cacheLBIas[a] = gpu.createKernel(function (
-            bias,
-            outputGradient,
-            learningRate
-        ) {
-            const grad = Math.max(
-                -1,
-                Math.min(1, outputGradient[this.thread.x])
-            )
-
-            return bias[this.thread.x] - learningRate * grad
-        })
-            .setOutput([splitedWeights.data[a].length])
-        cacheLBInputGradent[a] = gpu.createKernel(function (
-            weights,
-            outputGradient
-        ) {
-            let sum = 0
-
-            for (let a = 0; a < this.constants.outputSize; a++) {
+export class GPULinear {
+    cacheLF = [];
+    cacheLBIas = [];
+    cacheLBWeights = [];
+    cacheLBInputGradent = [];
+    // texures
+    Bias = [];
+    Weights = []
+    InitKernel(weights, bias, size) {
+        const splitedWeights = splitMatrix(weights)
+        const splitedBias = splitMatrix(bias)
+        this.Weights = splitedWeights.data
+        this.Bias = splitedBias.data
+        for (let a = 0; a < splitedWeights.data.length; a++) {
+            this.cacheLF[a] = gpu.createKernel(function (input, bias, weights) {
+                let sum = bias[this.thread.x]
+                for (let a = 0; a < this.constants.inputLength; a++) {
+                    sum += input[a] * weights[this.thread.x][a]
+                }
+                return sum
+            }).setConstants({
+                inputLength: splitedWeights.data[a][0].length
+            }).setArgumentTypes({
+                input: 'Array',
+                bias: 'Array',
+                weights: 'Array'
+            }).setOutput([splitedWeights.data[a].length])
+            this.cacheLBIas[a] = gpu.createKernel(function (
+                bias,
+                outputGradient,
+                learningRate
+            ) {
                 const grad = Math.max(
                     -1,
-                    Math.min(1, outputGradient[a])
+                    Math.min(1, outputGradient[this.thread.x])
                 )
 
-                sum += grad * weights[a][this.thread.x]
-            }
-
-            return sum
-        })
-            .setConstants({
-                outputSize: splitedWeights.data[a].length
+                return bias[this.thread.x] - learningRate * grad
+            }, {
+                pipeline: true,
+                immutable: true,
+                output: [splitedWeights.data[a].length]
             })
-            .setOutput([size])
-        cacheLBWeights[a] = gpu.createKernel(function (
-            weights,
-            outputGradient,
-            input,
-            learningRate
-        ) {
-            const grad = Math.max(
-                -1,
-                Math.min(1, outputGradient[this.thread.y])
-            )
-          
-            return weights[this.thread.y][this.thread.x]
-                - learningRate * input[this.thread.x] * grad
-        })
-            .setOutput([size, splitedWeights.data[a].length])
-    }
-
-
-}
-export function CalculateLinearForward(weights, bias, input) {
-    const splitedBias = splitMatrix(bias)
-    const splitedWeights = splitMatrix(weights)
-    const result = new Float32Array(weights.length)
-    let offset = 0;
-    for (let a = 0; a < splitedBias.data.length; a++) {
-        const output = cacheLF[a](input, splitedBias.data[a], splitedWeights.data[a])
-        result.set(output, offset)
-        offset += output.length
-    }
-    return result
-}
-
-export function CalculateBackward(weights, bias, out, learningRate, input) {
-    const splitedBias = splitMatrix(bias)
-    const splitedWeights = splitMatrix(weights)
-    const inputGradient = new Float32Array(input.length)
-    const biasOut = new Float32Array(bias.length)
-    const weightsOut = []
-    let offsetBias = 0
-    let offsetWeights = 0
-    let gradientOffset = 0
-
-    for (let a = 0; a < splitedWeights.data.length; a++) {
-        const weightChunk = splitedWeights.data[a]
-        const biasChunk = splitedBias.data[a]
-
-        const chunkOutputSize = weightChunk.length
+            this.cacheLBInputGradent[a] = gpu.createKernel(function (
+                weights,
+                outputGradient
+            ) {
+                let sum = 0
+
+                for (let a = 0; a < this.constants.outputSize; a++) {
+                    const grad = Math.max(
+                        -1,
+                        Math.min(1, outputGradient[a])
+                    )
+
+                    sum += grad * weights[a][this.thread.x]
+                }
+
+                return sum
+            }, {
+                constants: { outputSize: splitedWeights.data[a].length },
+                output: [size]
+            })
+                
+            this.cacheLBWeights[a] = gpu.createKernel(function (
+                weights,
+                outputGradient,
+                input,
+                learningRate
+            ) {
+                const grad = Math.max(
+                    -1,
+                    Math.min(1, outputGradient[this.thread.y])
+                )
 
-        // Gradient belonging to this output/weight chunk
-        const gradientChunk = out.slice(
-            gradientOffset,
-            gradientOffset + chunkOutputSize
-        )
+                return weights[this.thread.y][this.thread.x]
+                    - learningRate * input[this.thread.x] * grad
+            }, {
+                pipeline: true,
+                immutable: true,
+                output: [size, splitedWeights.data[a].length]
+            })
+        }
 
-        gradientOffset += chunkOutputSize
 
-        // Bias update
-        const newBias = cacheLBIas[a](
-            biasChunk,
-            gradientChunk,
-            learningRate
-        )
 
-        biasOut.set(newBias, offsetBias)
-        offsetBias += newBias.length
 
-        // Input gradient
-        const outputGra = cacheLBInputGradent[a](
-            weightChunk,
-            gradientChunk
-        )
+    }
+    CalculateLinearForward(input) {
+        const result = []
+        let offset = 0;
+        for (let a = 0; a < this.Weights.length; a++) {
+            const output = this.cacheLF[a](input, this.Bias[a], this.Weights[a])
+            result.push(...output)
 
-        // ADD this chunk's gradient to the same input gradient
-        for (let i = 0; i < outputGra.length; i++) {
-            inputGradient[i] += outputGra[i]
         }
-
-        // Weight update
-        const newWeights = cacheLBWeights[a](
-            weightChunk,
-            gradientChunk,
-            input,
-            learningRate
-        )
-
-        // Copy chunk into final Float32Array
-        weightsOut.push(...newWeights)
-        
+        return result
     }
+    CalculateBackward(out, learningRate, input) {
+        let inputGradient = new Float32Array(input.length).fill(0)
+        for (let a = 0; a < this.Weights.length; a++) {
+            const outGradient = this.cacheLBInputGradent[a](this.Weights[a], out)
+            for (let i = 0; i < outGradient.length; i++) {
+                inputGradient[i] += outGradient[i]
+            }
+            this.Weights[a] = this.cacheLBWeights[a](this.Weights[a], out, input, learningRate)
+            this.Bias[a] = this.cacheLBIas[a](this.Bias[a], out, learningRate)
 
-    return {
-        inputGradient,
-        biasOut,
-        weightsOut
+        }
+        return inputGradient
     }
 }

package/src/Linear/index.js

@@ -1,6 +1,6 @@
 import { LoadBias, LoadVectors, SaveBias, SaveVectors } from '../GetConfigs.js'
 import { existsSync } from 'fs'
-import { CalculateBackward, CalculateLinearForward, InitKernel } from '../GPU.js'
+import { GPULinear } from '../GPU.js'
 function GenerateWeightsBias(embeddingSize, vocabSize) {
     const vectors = []
     const Bias = new Float32Array(vocabSize)
@@ -19,6 +19,7 @@ export class Linear {
     Bias = []
     configs;
     input = [];
+    gpu;
     constructor(embeddingSize, vocabSize, configs) {
         if (configs && configs.save) {
             this.configs = configs
@@ -39,7 +40,8 @@ export class Linear {
             this.Weights = vectors;
             this.Bias = Bias;
         }
-        InitKernel(this.Weights, embeddingSize)
+        this.gpu = new GPULinear()
+        this.gpu.InitKernel(this.Weights, this.Bias, embeddingSize)
     }
     Save() {
         SaveVectors(this.Weights, this.configs.save.filename[0])
@@ -47,14 +49,12 @@ export class Linear {
     }
     forward(input) {
         this.input.push(input)
-        return CalculateLinearForward(this.Weights, this.Bias, input)
+        return this.gpu.CalculateLinearForward(this.Weights, this.Bias, input)
     }
     backward(outputGradient, learningRate, idx) {
         const input = this.input[idx];
         const weights = this.Weights
-        const {biasOut,inputGradient, weightsOut} = CalculateBackward(weights, this.Bias, outputGradient, learningRate, input)
-        this.Weights = weightsOut;
-        this.Bias = biasOut
+        const inputGradient = this.gpu.CalculateBackward(outputGradient, learningRate, input)
         return inputGradient
     }
     ClearInputCache(){