@mineproness/tensorloop 1.5.3 → 1.5.5

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mineproness/tensorloop",
-  "version": "1.5.3",
+  "version": "1.5.5",
   "description": "Raw-JAVASCRIPT Based Matirx and AI LLM Package called Tensorloop.",
   "keywords": [
     "ai",
@@ -19,5 +19,8 @@
   "repository": {
     "type": "git",
     "url": "https://github.com/mineproness-org/tensorloop.git"
+  },
+  "dependencies": {
+    "gpu.js": "^2.24.0"
   }
 }

package/src/FFN/index.js

@@ -50,5 +50,6 @@ export class FFN {
         this.linear1.ClearInputCache()
         this.linear2.ClearInputCache()
         this.hidden.ClearInputCache()
+        
     }
 }

package/src/GPU.js

@@ -0,0 +1,159 @@
+import { GPU } from "gpu.js";
+
+const gpu = new GPU({
+    mode: "gpu"
+})
+const kernel = gpu.createKernel(function () {
+    return 1
+}).setOutput([1])
+
+kernel()
+const gl = kernel.context
+
+export function splitMatrix(matrix, chunksSizes = gl.getParameter(gl.MAX_TEXTURE_SIZE)) {
+    const chunks = []
+    for (let a = 0; a < matrix.length; a += chunksSizes) {
+        chunks.push(matrix.slice(a, a + chunksSizes))
+    }
+    return { split: true, data: chunks }
+
+}
+export class GPULinear {
+    cacheLF = [];
+    cacheLBIas = [];
+    cacheLBWeights = [];
+    cacheLBInputGradent = [];
+    InitKernel(weights, size) {
+        const splitedWeights = splitMatrix(weights)
+        for (let a = 0; a < splitedWeights.data.length; a++) {
+            this.cacheLF[a] = gpu.createKernel(function (input, bias, weights) {
+                let sum = bias[this.thread.x]
+                for (let a = 0; a < this.constants.inputLength; a++) {
+                    sum += input[a] * weights[this.thread.x][a]
+                }
+                return sum
+            }).setConstants({
+                inputLength: splitedWeights.data[a][0].length
+            }).setArgumentTypes({
+                input: 'Array',
+                bias: 'Array',
+                weights: 'Array'
+            }).setOutput([splitedWeights.data[a].length])
+            this.cacheLBIas[a] = gpu.createKernel(function (
+                bias,
+                outputGradient,
+                learningRate
+            ) {
+                const grad = Math.max(
+                    -1,
+                    Math.min(1, outputGradient[this.thread.x])
+                )
+
+                return bias[this.thread.x] - learningRate * grad
+            })
+                .setOutput([splitedWeights.data[a].length])
+            this.cacheLBInputGradent[a] = gpu.createKernel(function (
+                weights,
+                outputGradient
+            ) {
+                let sum = 0
+
+                for (let a = 0; a < this.constants.outputSize; a++) {
+                    const grad = Math.max(
+                        -1,
+                        Math.min(1, outputGradient[a])
+                    )
+
+                    sum += grad * weights[a][this.thread.x]
+                }
+
+                return sum
+            })
+                .setConstants({
+                    outputSize: splitedWeights.data[a].length
+                })
+                .setOutput([size])
+            this.cacheLBWeights[a] = gpu.createKernel(function (
+                weights,
+                outputGradient,
+                input,
+                learningRate
+            ) {
+                const grad = Math.max(
+                    -1,
+                    Math.min(1, outputGradient[this.thread.y])
+                )
+
+                return weights[this.thread.y][this.thread.x]
+                    - learningRate * input[this.thread.x] * grad
+            })
+                .setOutput([size, splitedWeights.data[a].length])
+        }
+
+
+
+
+    }
+    CalculateLinearForward(weights, bias, input) {
+        // console.log(input)
+        const splitedBias = splitMatrix(bias)
+        const splitedWeights = splitMatrix(weights)
+        const result = new Float32Array(weights.length)
+        let offset = 0;
+        for (let a = 0; a < splitedBias.data.length; a++) {
+            const output = this.cacheLF[a](input, splitedBias.data[a], splitedWeights.data[a])
+            result.set(output, offset)
+            offset += output.length
+        }
+        return result
+    }
+    CalculateBackward(weights, bias, out, learningRate, input) {
+        const splitedBias = splitMatrix(bias)
+        const splitedWeights = splitMatrix(weights)
+        const inputGradient = new Float32Array(input.length)
+        const biasOut = new Float32Array(bias.length)
+        const weightsOut = []
+        let offsetBias = 0
+        let offsetWeights = 0
+        let gradientOffset = 0
+
+        for (let a = 0; a < splitedWeights.data.length; a++) {
+            const weightChunk = splitedWeights.data[a]
+            const biasChunk = splitedBias.data[a]
+            const chunkOutputSize = weightChunk.length
+            const gradientChunk = out.slice(
+                gradientOffset,
+                gradientOffset + chunkOutputSize
+            )
+            gradientOffset += chunkOutputSize
+            const newBias = this.cacheLBIas[a](
+                biasChunk,
+                gradientChunk,
+                learningRate
+            )
+            biasOut.set(newBias, offsetBias)
+            offsetBias += newBias.length
+            const outputGra = this.cacheLBInputGradent[a](
+                weightChunk,
+                gradientChunk
+            )
+            for (let i = 0; i < outputGra.length; i++) {
+                inputGradient[i] += outputGra[i]
+            }
+            const newWeights = this.cacheLBWeights[a](
+                weightChunk,
+                gradientChunk,
+                input,
+                learningRate
+            )
+            weightsOut.push(...newWeights)
+
+        }
+
+        return {
+            inputGradient,
+            biasOut,
+            weightsOut
+        }
+    }
+}

package/src/Linear/index.js

@@ -1,5 +1,6 @@
 import { LoadBias, LoadVectors, SaveBias, SaveVectors } from '../GetConfigs.js'
 import { existsSync } from 'fs'
+import { GPULinear } from '../GPU.js'
 function GenerateWeightsBias(embeddingSize, vocabSize) {
     const vectors = []
     const Bias = new Float32Array(vocabSize)
@@ -18,6 +19,7 @@ export class Linear {
     Bias = []
     configs;
     input = [];
+    gpu;
     constructor(embeddingSize, vocabSize, configs) {
         if (configs && configs.save) {
             this.configs = configs
@@ -38,6 +40,8 @@ export class Linear {
             this.Weights = vectors;
             this.Bias = Bias;
         }
+        this.gpu = new GPULinear()
+        this.gpu.InitKernel(this.Weights, embeddingSize)
     }
     Save() {
         SaveVectors(this.Weights, this.configs.save.filename[0])
@@ -45,29 +49,14 @@ export class Linear {
     }
     forward(input) {
         this.input.push(input)
-        const output = new Float32Array(this.Weights.length)
-        for (let a = 0; a < this.Weights.length; a++) {
-            let sum = this.Bias[a]
-            for (let t = 0; t < input.length; t++) {
-                sum += input[t] * this.Weights[a][t]
-            }
-            output[a] = sum
-        }
-        return output
+        return this.gpu.CalculateLinearForward(this.Weights, this.Bias, input)
     }
     backward(outputGradient, learningRate, idx) {
         const input = this.input[idx];
         const weights = this.Weights
-        const inputGradient = new Float32Array(input.length)
-        for (let a = 0; a < weights.length; a++) {
-            const gran = Math.max(-1, Math.min(1, outputGradient[a]))
-            const w = weights[a]
-            for (let at = 0; at < input.length; at++) {
-                inputGradient[at] += gran * w[at]
-                w[at] -= learningRate * input[at] * gran
-            }
-            this.Bias[a] -= learningRate * gran
-        }
+        const {biasOut,inputGradient, weightsOut} = this.gpu.CalculateBackward(weights, this.Bias, outputGradient, learningRate, input)
+        this.Weights = weightsOut;
+        this.Bias = biasOut
         return inputGradient
     }
     ClearInputCache(){