@mineproness/tensorloop 2.0.0 → 2.1.11

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mineproness/tensorloop",
-  "version": "2.0.0",
+  "version": "2.1.11",
   "description": "Raw-JAVASCRIPT Based Matirx and AI LLM Package called Tensorloop.",
   "keywords": [
     "ai",

package/readme.md

@@ -9,15 +9,16 @@ A high-performance JavaScript library that provides essential AI and Large Langu
 - **Transformer Block**: Complete modular block managing full attention and forward/backward training passes.
 - **Embedding Vectors**: High-speed lookup and embedding initialization.
 - **Linear Layer**: Fully connected layers with manageable weights and biases.
-- **Activations**: Forward and backward propagation for **Softmax**, **ReLU**, and **GELU**.
+- **Activations**: Forward and backward propagation for **Softmax**, **ReLU**, **GELU**.
 - **Loss Functions**: **CrossEntropy** loss calculation for training classification tasks.
 - **Attention & Positioning**: Built-in **Self-Attention** and **Positional Embedding** matrices.
 - **Text Processing**: Built-in **Tokenizer** to convert raw text into token IDs and back.
 - **Special Tokens**: Native support for `<PAD>` and `<EOS>`.
 - **FFN BLOCK**: It add a layer to understand Complex Patterns.
-- ***GPU SUPPORT** GPU Can support it without any configure
+- **GPU SUPPORT**: Now, The tensorloop can Do Math with GPU.
 ---
 
+
 ## 🏗 Core Architecture
 
 ### Transformer Block
@@ -54,7 +55,7 @@ You can configure saving mechanisms by passing a configuration object to the `Em
 import { Embedding, Linear } from '@mineproness/tensorloop'
 
 // Initialize an Embedding layer and auto-save the weights
-const embedding = new Embedding(1024, 2000, {
+const embedding = new Embedding(1024, 2000, 16 {
     save: {
         filename: "./model/vectors.bin"
     }

package/src/Embedding/index.d.ts

@@ -1,6 +1,6 @@
 export class Embedding{
     vectors: Float32Array[]
-    constructor(embeddingSize: number, vocabSize: number, configs: {
+    constructor(embeddingSize: number, vocabSize: number, contextSize: number, configs: {
         save: {
             filename: string
         }

package/src/Embedding/index.js

@@ -1,5 +1,6 @@
 import fs, { existsSync } from 'fs'
 import { SaveVectors, LoadVectors } from '../GetConfigs.js'
+import { GPUEmbedding } from '../gpiInit.js'
 // import { GPUEmbedding } from '../GPU.js'
 // import { config } from 'process'
 
@@ -18,8 +19,8 @@ export class Embedding {
     vectors = []
     configs;
     embeddingSize = 0
-
-    constructor(embeddingSize, vocabSize, configs) {
+    gpu;
+    constructor(embeddingSize, vocabSize, contextSize, configs) {
         this.configs = configs
         this.embeddingSize = embeddingSize
         if (configs && configs.save) {
@@ -33,27 +34,20 @@ export class Embedding {
             this.vectors = GenerateVectors(embeddingSize, vocabSize)
 
         }  
+        this.gpu = new GPUEmbedding(this.vectors, vocabSize, embeddingSize, contextSize)
     }
     forward(token) {
-        if (Array.isArray(token)) {
-            return token.map((e) => {
-                return this.vectors[e]
-            })
-        } else {
-            return this.vectors[token]
-        }
+        return this.gpu.forward(token)
     }
-    backwards(token, inputGradient, learingRate) {
-        for (let a = 0; a < this.vectors[token].length; a++) {
-            this.vectors[token][a] -= learingRate * inputGradient[a]
-        }
-    }
-    backward(tokens, inputGradient, learingRate){
-         for(let a = 0; a < inputGradient.length; a++){
-            this.backwards(tokens[a], inputGradient[a], learingRate)
-         }
+    backward(inputGradient, learingRate){
+         this.gpu.backward(inputGradient, learingRate)
     }
     Save() {
-        SaveVectors(this.vectors, this.configs.save.filename)
+        const vectors = []
+        for(let i = 0; i < this.gpu.vectors.length; i++){
+            const out = this.gpu.vectors[i].toArray()
+            vectors.push(...out)
+        }
+        SaveVectors(vectors.map((e)=> new Float32Array(e)), this.configs.save.filename)
     }
 }

package/src/FFN/index.js

@@ -2,12 +2,15 @@ import { Linear } from '../Linear/index.js'
 import { GELU } from '../GELU/index.js'
 import { ReLU } from '../ReLU/index.js'
 import { join } from 'path'
+import { GPUHiddenLayer } from '../gpiInit.js';
 export class FFN {
     linear1;
     linear2;
     hidden;
+    hiddenName;
     constructor(embeddingSize, hiddenLayer = "gelu", configs) {
-        this.hidden = hiddenLayer.toLocaleLowerCase() == "gelu" ? new GELU() : new ReLU()
+        this.hiddenName = hiddenLayer
+        this.hidden = hiddenLayer.toLocaleLowerCase() == "gelu" ? new GELU() : new GPUHiddenLayer()
         if (configs) {
             this.linear1 = new Linear(embeddingSize, 4 * embeddingSize, {
                 save: {
@@ -24,32 +27,25 @@ export class FFN {
             this.linear2 = new Linear(4 * embeddingSize, embeddingSize)
         }
     }
-    forward(input){
-        const result = []
-        for(let a = 0; a < input.length; a++){
-            const out1 = this.linear1.forward(input[a])
-            const hidden = this.hidden.forward(out1, a)
-            result[a] = this.linear2.forward(hidden)
-        }
+    forward(input) {
+        const out1 = this.linear1.forward(input)
+        let hidden = this.hidden.forward(out1)
+        const result = this.linear2.forward(hidden)
         return result
     }
-    backward(dInput, LearningRate){
-        const result = []
-        for(let a = 0; a < dInput.length; a++){
-           const out2 = this.linear2.backward(dInput[a] , LearningRate, a);
-           const dHidden = this.hidden.backward(out2, a)
-           result[a] = this.linear1.backward(dHidden, LearningRate, a)
-        }
+    backward(dInput, LearningRate) {
+        const out2 = this.linear2.backward(dInput, LearningRate);
+        const dHidden = this.hidden.backward(out2)
+        const result = this.linear1.backward(dHidden, LearningRate)
         return result
     }
-    Save(){
+    Save() {
         this.linear1.Save()
         this.linear2.Save()
     }
-    ClearInputCache(){
+    ClearInputCache() {
         this.linear1.ClearInputCache()
         this.linear2.ClearInputCache()
         this.hidden.ClearInputCache()
-        
     }
 }

package/src/GELU/index.d.ts

@@ -1,6 +1,6 @@
 export class GELU{
     input : Float32Array[];
-    forward(input: Float32Array, index: number) : Float32Array;
-    backward(Doutput: Float32Array,  index: number) : Float32Array;
+    forward(input: Float32Array[]) : Float32Array[];
+    backward(Doutput: Float32Array[]) : Float32Array[];
     ClearInputCache() : void;
 }

package/src/GELU/index.js

@@ -1,40 +1,30 @@
+import { GELUGPU } from "../gpiInit.js";
+
 export class GELU {
+    gpu;
+    inputs = [];
     constructor() {
         this.inputs = []
+        
     }
 
-    forward(input, index = 0) {
-        this.inputs[index] = new Float32Array(input)
-        const output = new Float32Array(input.length)
-        const k = Math.sqrt(2 / Math.PI)
-        for (let i = 0; i < input.length; i++) {
-            const x = input[i]
-            const u = k * (x + 0.044715 * x * x * x)
-            output[i] = 0.5 * x * (1 + Math.tanh(u))
+    forward(input) {
+        this.inputs = input
+        if(!this.gpu){
+            this.gpu = new GELUGPU(input[0].length, input.length)
         }
-        return output
+        // const output = new Float32Array(input.length)
+        // const k = Math.sqrt(2 / Math.PI) 
+        // for (let i = 0; i < input.length; i++) {
+        //     const x = input[i]
+        //     const u = k * (x + 0.044715 * x * x * x)
+        //     output[i] = 0.5 * x * (1 + Math.tanh(u))
+        // }
+        return this.gpu.forward(input)
     }
 
-    backward(dOutput, index = 0) {
-        const input = this.inputs[index]
-        if (!input) throw new Error(`No GELU cache for index ${index}`);
-        const grad = new Float32Array(dOutput.length)
-        const k = Math.sqrt(2 / Math.PI)
-        for (let i = 0; i < dOutput.length; i++) {
-            const x = input[i]
-            const x3 = x * x * x
-            const u = k * (x + 0.044715 * x3)
-            const t = Math.tanh(u)
-            const sech2 = 1 - t * t
-            const duDx = k * (1 + 3 * 0.044715 * x * x)
-            const geluGrad =
-                0.5 * (1 + t) +
-                0.5 * x * sech2 * duDx
-
-            grad[i] = dOutput[i] * geluGrad
-        }
-
-        return grad
+    backward(dOutput) {  
+        return this.gpu.backward(dOutput, this.inputs)
     }
 
     ClearInputCache() {

package/src/Linear/index.d.ts

@@ -1,13 +1,13 @@
 export class Linear{
     Weights : Float32Array[]
     Bias: Float32Array
-    constructor(embeddingSize : number, vocabSize : number, contextSiz: number, configs: {
+    constructor(embeddingSize : number, vocabSize : number, configs: {
         save: {
             filename: string[]
         }
     })
     forward(input : Float32Array[]) : Float32Array[]
-    backward(outGradient : number[] , learingRate: number, idx: number) : Float32Array[]
+    backward(outGradient : number[] , learingRate: number) : Float32Array[]
     Save() : void
     ClearInputCache() : void
 }

package/src/Linear/index.js

@@ -1,6 +1,6 @@
 import { LoadBias, LoadVectors, SaveBias, SaveVectors } from '../GetConfigs.js'
 import { existsSync } from 'fs'
-import { GPULinear } from '../GPU.js'
+import { GPULinear } from '../gpiInit.js'
 function GenerateWeightsBias(embeddingSize, vocabSize) {
     const vectors = []
     const Bias = new Float32Array(vocabSize)
@@ -20,7 +20,7 @@ export class Linear {
     configs;
     input = [];
     gpu;
-    constructor(embeddingSize, vocabSize, contextSize, configs) {
+    constructor(embeddingSize, vocabSize, configs) {
         if (configs && configs.save) {
             this.configs = configs
             if (existsSync(configs.save.filename[0]) && existsSync(configs.save.filename[1])) {
@@ -40,28 +40,26 @@ export class Linear {
             this.Weights = vectors;
             this.Bias = Bias;
         }
-        this.gpu = new GPULinear()
-        this.gpu.InitKernel(this.Weights, this.Bias, embeddingSize, contextSize)
+
     }
     Save() {
-        const {bias, weights} = this.gpu.GetWeightsAndBias()
+        const {bias, weights} = this.gpu.Save()
         SaveVectors(weights, this.configs.save.filename[0])
         SaveBias(bias, this.configs.save.filename[1])
     }
     forward(input) {
-        const result = []
-        for(let a = 0;a < input.length; a++){
+        if (!this.gpu) {
+            this.gpu = new GPULinear(this.Weights, this.Bias, input.length)
+        }
+        for (let a = 0; a < input.length; a++) {
             this.input[a] = input;
-
         }
-        return this.gpu.CalculateLinearForward(input)
+        return this.gpu.forward(input)
     }
-    backward(outputGradient, learningRate) {
-        const input = this.input;
-        const inputGradient = this.gpu.CalculateBackward(outputGradient, learningRate, input)
-        return inputGradient
+    backward(out, lr){
+        return this.gpu.backward(out, this.input, lr)
     }
-    ClearInputCache(){
+    ClearInputCache() {
         this.input = []
     }
 }

package/src/PositionEmbedding/index.d.ts

@@ -8,5 +8,5 @@ export class PositionEmbedding{
     forward(position: number | number[]) : Float32Array[] | Float32Array[][]
     backward(pos: number , inputGradient: number[] | Float32Array[], learingRate: number) : void
     Save() : void;
-    addPos(PSEM, vectors : Float32Array[][]) : Float32Array 
+    addPos(vectors : Float32Array[][]) : Float32Array 
 }

package/src/PositionEmbedding/index.js

@@ -1,6 +1,7 @@
 import fs, { existsSync } from 'fs'
 import { LoadVectors, SaveVectors } from '../GetConfigs.js'
 // import { config } from 'process'
+import { GPUEmbedding } from '../gpiInit.js'
 
 function GenerateVectors(embeddingSize, vocabSize) {
     const vector = []
@@ -16,7 +17,8 @@ function GenerateVectors(embeddingSize, vocabSize) {
 export class PositionEmbedding {
     vectors = []
     configs;
-    embeddingSize = 0
+    embeddingSize = 0;
+    gpu;
     constructor(embeddingSize, vocabSize, configs) {
         this.configs = configs
         this.embeddingSize = embeddingSize
@@ -29,7 +31,7 @@ export class PositionEmbedding {
             }
         } else {
             this.vectors = GenerateVectors(embeddingSize, vocabSize)
-
+            this.gpu = new GPUEmbedding(this.vectors, vocabSize, embeddingSize, 1)
         }
     }
     forward(token) {
@@ -55,7 +57,7 @@ export class PositionEmbedding {
 
         for (let pos = 0; pos < xEM.length; pos++) {
             const tokenVector = xEM[pos]
-            const posVector = this.forward(pos)
+            const posVector = this.forward([pos])[0]
             const x = new Float32Array(xEM[0].length)
             for (let a = 0; a < tokenVector.length; a++) {
                 x[a] = tokenVector[a] + posVector[a]

package/src/ReLU/index.d.ts

@@ -1,4 +1,5 @@
 export class ReLU{
     forward(XLi : Float32Array[] | number[]) : number[] | Float32Array[];
     backward(outputGradient: number[] | Float32Array[]) : Float32Array[];
+    ClearInputCache() : void
 }

package/src/ReLU/index.js

@@ -1,14 +1,17 @@
+import { GPUHiddenLayer } from "../gpiInit.js";
+
 export class ReLU{
     input;
-    forward(XLi){
-        this.input = XLi
-        return XLi.map((e)=> e > 0 ? e : 0)
+    gpu;
+    constructor(){
+        this.gpu = new GPUHiddenLayer()
     }
-    backward(outputGradient){
-        const inputGradient = new Float32Array(this.input.length)
-        for(let i = 0; i < outputGradient.length; i++){
-            inputGradient[i] = this.input[i] > 0 ? outputGradient[i] : 0
-        }
-        return inputGradient
+    forward(logits){
+        return this.gpu.forward(logits)
     }
+    backward(outGradient){
+        return this.gpu.backward(outGradient)
+    }
+    
+    ClearInputCache(){}
 }

package/src/SelfAttention/index.js

@@ -1,4 +1,5 @@
 import { Softmax } from '../../index.js';
+import { GPUAttention } from '../gpiInit.js';
 import { Linear } from '../Linear/index.js'
 
 export class SelfAttention {
@@ -13,6 +14,7 @@ export class SelfAttention {
     score;
     softmax;
     weights;
+    gpu;
     constructor(embeddingSize, configs) {
         this.configs = configs
         this.softmax = new Softmax()
@@ -22,99 +24,30 @@ export class SelfAttention {
         this.value = new Linear(embeddingSize, embeddingSize, { save: { filename: [`${configs.dirname}/value.bin`, `${configs.dirname}/valueBias.bin`] } })
     }
     forward(vectors) {
+        if(!this.gpu) this.gpu = new GPUAttention(vectors[0].length, vectors.length);
         this.q = this.query.forward(vectors)
         this.v = this.value.forward(vectors)
         this.k = this.key.forward(vectors)
-        const attentionSocres = []
-        for (let i = 0; i < this.q.length; i++) {
-            attentionSocres[i] = new Float32Array(this.k.length);
-
-            for (let j = 0; j < this.k.length; j++) {
-                if (j > i) {
-                    attentionSocres[i][j] = -1e9; // mask future
-                    continue;
-                }
-
-                let dot = 0;
-
-                for (let d = 0; d < this.q[i].length; d++) {
-                    dot += this.q[i][d] * this.k[j][d];
-                }
-
-                attentionSocres[i][j] = dot / this.score;
-            }
-        }
-        this.softmax.forward(attentionSocres)
-        const weights = this.softmax.getSoftmaxes()
+        const attentionSocres = this.gpu.attentionScores(this.q, this.k, this.score);
+        const weights = this.softmax.forward(attentionSocres)
         this.weights = weights
-        const output = [];
-
-        for (let i = 0; i < weights.length; i++) {
-            output[i] = new Float32Array(this.v[0].length);
-
-            for (let j = 0; j < this.v.length; j++) {
-                for (let d = 0; d < this.v[j].length; d++) {
-                    output[i][d] += weights[i][j] * this.v[j][d];
-                }
-            }
-        }
-
+        const output = this.gpu.attentionOutput(weights, this.v)
         return output;
     }
-    softmaxBack(probs, dOut) {
-        const dIn = new Float32Array(probs.length);
-
-        for (let i = 0; i < probs.length; i++) {
-            let sum = 0;
 
-            for (let j = 0; j < probs.length; j++) {
-                const jacobian =
-                    i === j
-                        ? probs[i] * (1 - probs[i])
-                        : -probs[i] * probs[j];
-
-                sum += jacobian * dOut[j];
-            }
-
-            dIn[i] = sum;
-        }
-
-        return dIn;
-    }
     backward(dOutput, learningRate) {
-        const dV = this.v.map((e) => new Float32Array(e.length));
-        const dWeights = []
-        for (let a = 0; a < this.weights.length; a++) {
-            dWeights[a] = new Float32Array(this.weights[a].length)
-            for (let j = 0; j < this.v.length; j++) {
-                for (let r = 0; r < this.v[j].length; r++) {
-                    dWeights[a][j] += dOutput[a][r] * this.v[j][r]
-                    dV[j][r] += dOutput[a][r] * this.weights[a][j]
-                }
-            }
-        }
-        const dScores = []
-        for (let t = 0; t < dWeights.length; t++) {
-            dScores[t] = this.softmaxBack(this.weights[t], dWeights[t])
-        }
-        const dQ = this.q.map(q => new Float32Array(q.length));
-        const dK = this.k.map(k => new Float32Array(k.length));
-        for (let a = 0; a < this.q.length; a++) {
-            for (let j = 0; j < this.k.length; j++) {
-                if (j > a) continue; 
-                for (let r = 0; r < this.q[a].length; r++) {
-                    dQ[a][r] += dScores[a][j] * this.k[j][r] / this.score;
-                    dK[j][r] += dScores[a][j] * this.q[a][r] / this.score;
-                }
-            }
-        }
+        const dWeights = this.gpu.dWeightsKernel(dOutput, this.v)
+        const dV = this.gpu.dVKernel(dOutput, this.weights);
+        const dScores = this.gpu.softmaxBackKernel(this.weights, dWeights)
+        const dQ = this.gpu.dQKernel(dScores, this.k, this.score)
+        const dK = this.gpu.dKKernel(dScores, this.q, this.score)
         const dInput = this.q.map(() => new Float32Array(this.q[0].length))
+        const dq = this.query.backward(dQ, learningRate)
+        const dk = this.key.backward(dK, learningRate)
+        const dv = this.value.backward(dV, learningRate)
         for (let a = 0; a < this.q.length; a++) {
-            const dq = this.query.backward(dQ[a], learningRate, a)
-            const dk = this.key.backward(dK[a], learningRate, a)
-            const dv = this.value.backward(dV[a], learningRate, a)
-            for (let b = 0; b < dq.length; b++) {
-                dInput[a][b] = dq[b] + dk[b] + dv[b]
+            for (let b = 0; b < dq[a].length; b++) {
+                dInput[a][b] = dq[a][b] + dk[a][b] + dv[a][b]
             }
         }
         return dInput
@@ -124,7 +57,7 @@ export class SelfAttention {
         this.key.Save()
         this.value.Save()
     }
-    ClearInputCache(){
+    ClearInputCache() {
         this.query.ClearInputCache()
         this.key.ClearInputCache()
         this.value.ClearInputCache()

package/src/Softmax/index.d.ts

@@ -1,6 +1,5 @@
 export class Softmax{
     forward(xLi : number[]) : number
     backward(target : number[]) : Float32Array[]
-    getSoftmaxes() : Float32Array[]
     crossEntropy(dLogits: Float32Array[], target: number[])
 }

package/src/Softmax/index.js

@@ -1,20 +1,18 @@
-import { gpuSoftmax } from "../GPU.js";
+import { GPUSoftmax } from "../gpiInit.js";
+
 
 export class Softmax{
     gpu;
-    constructor(contextSize){
-        this.gpu = new gpuSoftmax(contextSize)
-    }
     forward(xLi){
-        return this.gpu.forwardSoftmax(xLi)
-    }
-    backward(target){
-        return this.gpu.softmaxBackward(target)
+        if(!this.gpu){
+            this.gpu = new GPUSoftmax(xLi, xLi.length)
+        }
+        return this.gpu.forward(xLi)
     }
-    getSoftmaxes(){
-        return this.gpu.getSoftmax()
+    backward(xLi , target){
+        return this.gpu.backward(xLi, target)
     }
     crossEntropy(probs, target){
-        return this.gpu.calculateLOSS(probs, target)
+        return this.gpu.crossEntropy(probs, target)
     }
 }

package/src/TransformerBlock/index.d.ts

@@ -1,6 +1,7 @@
 export class TransformerBlock{
     constructor(embeddingSize: number, configs:{
-        dirname: string
+        dirname: string,
+        activation: "relu" | "gelu"
     })
     Save(): void
     clearInputCache() : void

package/src/TransformerBlock/index.js

@@ -1,10 +1,12 @@
 import { SelfAttention } from '../SelfAttention/index.js'
 import { FFN } from '../FFN/index.js'
+import fs from 'fs'
 export class TransformerBlock {
     ffn;
     selfAttention;
     constructor(embeddingSize, configs) {
-        this.ffn = new FFN(embeddingSize, "gelu", configs)
+        if(!fs.existsSync(configs.dirname)) fs.mkdirSync(configs.dirname)
+        this.ffn = new FFN(embeddingSize, configs.activation, configs)
         this.selfAttention = new SelfAttention(embeddingSize, configs)
     }
     forward(input) {