@mineproness/tensorloop 1.5.8 → 2.0.0

package/index.js

@@ -2,7 +2,6 @@ import { Tokenizer as tokenizer } from "./src/Tokenizer/index.js";
 import { Embedding as embeddding2 } from "./src/Embedding/index.js";
 import { Linear as li2 } from "./src/Linear/index.js";
 import { Softmax as softmax2 } from "./src/Softmax/index.js";
-import { crossEntropy } from "./src/crossEntropy/index.js";
 import {ReLU} from './src/ReLU/index.js'
 import {PositionEmbedding as PE} from './src/PositionEmbedding/index.js'
 import {SelfAttention as ST} from './src/SelfAttention/index.js'
@@ -13,7 +12,6 @@ export const Tokenizer = tokenizer
 export const Embedding = embeddding2
 export const Linear = li2
 export const Softmax = softmax2
-export const CrossEntropy = crossEntropy
 export const ReLu = ReLU
 export const PositionEmbedding = PE
 export const SelfAttention = ST

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mineproness/tensorloop",
-  "version": "1.5.8",
+  "version": "2.0.0",
   "description": "Raw-JAVASCRIPT Based Matirx and AI LLM Package called Tensorloop.",
   "keywords": [
     "ai",

package/src/Embedding/index.js

@@ -1,6 +1,6 @@
 import fs, { existsSync } from 'fs'
 import { SaveVectors, LoadVectors } from '../GetConfigs.js'
-import { GPUEmbedding } from '../GPU.js'
+// import { GPUEmbedding } from '../GPU.js'
 // import { config } from 'process'
 
 function GenerateVectors(embeddingSize, vocabSize) {
@@ -43,11 +43,16 @@ export class Embedding {
             return this.vectors[token]
         }
     }
-    backward(token, inputGradient, learingRate) {
+    backwards(token, inputGradient, learingRate) {
         for (let a = 0; a < this.vectors[token].length; a++) {
             this.vectors[token][a] -= learingRate * inputGradient[a]
         }
     }
+    backward(tokens, inputGradient, learingRate){
+         for(let a = 0; a < inputGradient.length; a++){
+            this.backwards(tokens[a], inputGradient[a], learingRate)
+         }
+    }
     Save() {
         SaveVectors(this.vectors, this.configs.save.filename)
     }

package/src/GPU.js

@@ -18,24 +18,32 @@ export function splitMatrix(matrix, chunksSizes = maxSize) {
     return { split: true, data: chunks }
 
 }
+let outputSizes = [];
+let out = 0
 export class GPULinear {
     cacheLF = [];
     cacheLBIas = [];
     cacheLBWeights = [];
     cacheLBInputGradent = [];
     // texures
+    contextSize;
+    weightLength = 0;
     Bias = [];
     Weights = []
-    InitKernel(weights, bias, size) {
+    InitKernel(weights, bias, size, contextSize) {
         const splitedWeights = splitMatrix(weights)
         const splitedBias = splitMatrix(bias)
+        this.weightLength = weights.length
         this.Weights = splitedWeights.data
+        out = weights.length
         this.Bias = splitedBias.data
         for (let a = 0; a < splitedWeights.data.length; a++) {
+            outputSizes.push(splitedWeights.data[a].length)
+
             this.cacheLF[a] = gpu.createKernel(function (input, bias, weights) {
                 let sum = bias[this.thread.x]
                 for (let a = 0; a < this.constants.inputLength; a++) {
-                    sum += input[a] * weights[this.thread.x][a]
+                    sum += input[this.thread.y][a] * weights[this.thread.x][a]
                 }
                 return sum
             }).setConstants({
@@ -44,22 +52,25 @@ export class GPULinear {
                 input: 'Array',
                 bias: 'Array',
                 weights: 'Array'
-            }).setOutput([splitedWeights.data[a].length])
+            }).setOutput([splitedWeights.data[a].length, contextSize])
             this.cacheLBIas[a] = gpu.createKernel(function (
                 bias,
                 outputGradient,
                 learningRate
             ) {
-                const grad = Math.max(
-                    -1,
-                    Math.min(1, outputGradient[this.thread.x])
-                )
+                let grad = 0;
+                for (let a = 0; a < this.constants.contextSize; a++) {
+                    const g = outputGradient[a][this.thread.x]
+                    
+                    grad += g
+                }
 
                 return bias[this.thread.x] - learningRate * grad
             }, {
                 pipeline: true,
                 immutable: true,
-                output: [splitedWeights.data[a].length]
+                output: [splitedWeights.data[a].length, contextSize],
+                constants: { contextSize }
             })
             this.cacheLBInputGradent[a] = gpu.createKernel(function (
                 weights,
@@ -68,93 +79,201 @@ export class GPULinear {
                 let sum = 0
 
                 for (let a = 0; a < this.constants.outputSize; a++) {
-                    const grad = Math.max(
-                        -1,
-                        Math.min(1, outputGradient[a])
-                    )
-
+                    const grad = outputGradient[this.thread.y][a]
                     sum += grad * weights[a][this.thread.x]
                 }
 
                 return sum
             }, {
                 constants: { outputSize: splitedWeights.data[a].length },
-                output: [size]
+                output: [size, contextSize]
             })
-                
+
             this.cacheLBWeights[a] = gpu.createKernel(function (
                 weights,
                 outputGradient,
                 input,
                 learningRate
             ) {
-                const grad = Math.max(
-                    -1,
-                    Math.min(1, outputGradient[this.thread.y])
-                )
+                let grad = 0;
+                for (let a = 0; a < this.constants.contextSize; a++) {
+                    grad += outputGradient[a][this.thread.y] * input[a][this.thread.x]
+                }
 
                 return weights[this.thread.y][this.thread.x]
-                    - learningRate * input[this.thread.x] * grad
+                    - learningRate * grad
             }, {
                 pipeline: true,
                 immutable: true,
-                output: [size, splitedWeights.data[a].length]
+                output: [size, splitedWeights.data[a].length],
+                constants: { contextSize }
             })
         }
 
-
+        this.contextSize = contextSize
 
     }
     CalculateLinearForward(input) {
-        const result = []
+        const result = Array.from({ length: this.contextSize },
+            () => new Float32Array(this.weightLength)
+        )
         let offset = 0;
         for (let a = 0; a < this.Weights.length; a++) {
             const output = this.cacheLF[a](input, this.Bias[a], this.Weights[a])
-            result.push(...output)
+            for (let y = 0; y < this.contextSize; y++) {
+                result[y].set(output[y], offset);
+            }
+            offset += this.Weights[a].length
 
         }
         return result
     }
     CalculateBackward(out, learningRate, input) {
-        let inputGradient = new Float32Array(input.length).fill(0)
+        let inputGradient = Array.from({ length: this.contextSize } , ()=> new Float32Array(input[0].length))
         for (let a = 0; a < this.Weights.length; a++) {
             const outGradient = this.cacheLBInputGradent[a](this.Weights[a], out)
-            for (let i = 0; i < outGradient.length; i++) {
-                inputGradient[i] += outGradient[i]
+            for (let row = 0; row < this.contextSize; row++) {
+                for (let i = 0; i < input[0].length; i++) {
+                    inputGradient[row][i] += outGradient[row][i];
+                }
             }
             const oldW = this.Weights[a]
             const oldB = this.Bias[a]
             this.Weights[a] = this.cacheLBWeights[a](this.Weights[a], out, input, learningRate)
             this.Bias[a] = this.cacheLBIas[a](this.Bias[a], out, learningRate)
-            if(oldW && typeof oldW.delete === "function"){
+            if (oldW && typeof oldW.delete === "function") {
                 oldW.delete()
             }
-             if(oldB && typeof oldB.delete === "function"){
+            if (oldB && typeof oldB.delete === "function") {
                 oldB.delete()
             }
 
         }
         return inputGradient
     }
-    GetWeightsAndBias(){
+    GetWeightsAndBias() {
         const weight = []
         const bias = []
-        this.Weights.map((e)=> weight.push(...e.toArray()))
-        this.Bias.map((e)=> bias.push(...e.toArray()))
-        return { weights: weight, bias: new Float32Array(bias)}
+        this.Weights.map((e) => weight.push(...e.toArray()))
+        this.Bias.map((e) => bias.push(...e.toArray()))
+        return { weights: weight, bias: new Float32Array(bias) }
     }
 }
 
-export class GPUEmbedding{
-    cacheEF;
-    constructor(embeddingSize){
-        this.cacheEF = gpu.createKernel(function(inputGradient, learningRate, tokenVector){
-          return tokenVector[this.thread.x] - (learningRate * inputGradient[this.thread.x])
-        }).setOutput([embeddingSize])
+export class gpuSoftmax {
+    cacheSF = [];
+    cacheSB = [];
+    contextSize;
+    Probs = [];
+    LossCalculate;
+    constructor(contextSize) {
+        this.contextSize = contextSize;
+        for (let a = 0; a < outputSizes.length; a++) {
+            this.cacheSF[a] = gpu.createKernel(function (logits) {
+                let maxLogits = logits[this.thread.y][0];
+                let sum = 0;
+                for (let a = 0; a < this.constants.outputSize; a++) {
+                    if (logits[this.thread.y][a] > maxLogits) {
+                        maxLogits = logits[this.thread.y][a]
+                    }
+                }
+                for (let a = 0; a < this.constants.outputSize; a++) {
+                    sum += Math.exp(logits[this.thread.y][a] - maxLogits)
+                }
+
+                return Math.exp(logits[this.thread.y][this.thread.x] - maxLogits) / sum
+
+            }, {
+                pipeline: true,
+                immutable: true,
+                output: [outputSizes[a], contextSize],
+                constants: { outputSize: outputSizes[a] }
+            })
+            this.cacheSB[a] = gpu.createKernel(function (probs, target) {
+                const p = probs[this.thread.y][this.thread.x]
+                if (this.thread.x == target[this.thread.y]) {
+                    return p - 1.0
+                }
+                return p
+            }, {
+                output: [outputSizes[a], contextSize]
+            })
+        }
+        this.LossCalculate = gpu.createKernel(function (probs, target) {
+            const tar = target[this.thread.x]
+            return -Math.log((probs[this.thread.x][tar] + 1) + 1e-10)
+
+
+        }, {
+            output: [contextSize]
+        })
     }
-    backward(tokenVector, inputGradient, learingRate){
-        return this.cacheEF(inputGradient, learingRate, tokenVector)
+    forwardSoftmax(logits) {
+        const splitedLogits = splitMatrix(logits)
+        for (let a = 0; a < splitedLogits.data.length; a++) {
+            const oldSoftmax = this.Probs[a]
+            this.Probs[a] = this.cacheSF[a](splitedLogits.data[a])
+            if (oldSoftmax && typeof oldSoftmax.delete == "function") {
+                oldSoftmax.delete()
+            }
+        }
+        return "Completed"
     }
-  
-    
+    getSoftmax() {
+        const result = Array.from({ length: this.contextSize }, () => new Float32Array(out))
+        let offset = 0;
+        for (let a = 0; a < this.Probs.length; a++) {
+            const outputofSoftmax = this.Probs[a].toArray()
+            for (let a = 0; a < this.contextSize; a++) {
+                result[a].set(outputofSoftmax[a], offset)
+            }
+            offset += outputofSoftmax[0].length
+        }
+        return result
+    }
+    softmaxBackward(target) {
+        const result = Array.from(
+            { length: this.contextSize },
+            () => new Float32Array(out)
+        );
+        let offset = 0;
+        for (let chunk = 0; chunk < this.Probs.length; chunk++) {
+            const chunkSize = outputSizes[chunk];
+            const localTarget = new Int32Array(this.contextSize);
+            for (let row = 0; row < this.contextSize; row++) {
+                const globalTarget = target[row];
+                if (
+                    globalTarget >= offset &&
+                    globalTarget < offset + chunkSize
+                ) {
+                    localTarget[row] = globalTarget - offset;
+                } else {
+                    localTarget[row] = -1;
+                }
+            }
+            const gradient = this.cacheSB[chunk](
+                this.Probs[chunk],
+                localTarget
+            );
+            const data = gradient;
+            for (let row = 0; row < this.contextSize; row++) {
+                result[row].set(data[row], offset);
+            }
+            offset += chunkSize;
+        }
+
+        return result;
+    }
+    calculateLOSS(dLogits, target) {
+        const losses = this.LossCalculate(dLogits, target)
+        let loss = 0;
+
+        for (let i = 0; i < losses.length; i++) {
+            loss += losses[i];
+        }
+
+        loss /= losses.length;
+        return loss
+    }
+
 }

package/src/Linear/index.d.ts

@@ -1,12 +1,12 @@
 export class Linear{
     Weights : Float32Array[]
     Bias: Float32Array
-    constructor(embeddingSize : number, vocabSize : number, configs: {
+    constructor(embeddingSize : number, vocabSize : number, contextSiz: number, configs: {
         save: {
             filename: string[]
         }
     })
-    forward(input : number[]) : Float32Array[]
+    forward(input : Float32Array[]) : Float32Array[]
     backward(outGradient : number[] , learingRate: number, idx: number) : Float32Array[]
     Save() : void
     ClearInputCache() : void

package/src/Linear/index.js

@@ -20,7 +20,7 @@ export class Linear {
     configs;
     input = [];
     gpu;
-    constructor(embeddingSize, vocabSize, configs) {
+    constructor(embeddingSize, vocabSize, contextSize, configs) {
         if (configs && configs.save) {
             this.configs = configs
             if (existsSync(configs.save.filename[0]) && existsSync(configs.save.filename[1])) {
@@ -41,7 +41,7 @@ export class Linear {
             this.Bias = Bias;
         }
         this.gpu = new GPULinear()
-        this.gpu.InitKernel(this.Weights, this.Bias, embeddingSize)
+        this.gpu.InitKernel(this.Weights, this.Bias, embeddingSize, contextSize)
     }
     Save() {
         const {bias, weights} = this.gpu.GetWeightsAndBias()
@@ -49,16 +49,21 @@ export class Linear {
         SaveBias(bias, this.configs.save.filename[1])
     }
     forward(input) {
-        this.input.push(input)
-        return this.gpu.CalculateLinearForward(this.Weights, this.Bias, input)
+        const result = []
+        for(let a = 0;a < input.length; a++){
+            this.input[a] = input;
+
+        }
+        return this.gpu.CalculateLinearForward(input)
     }
-    backward(outputGradient, learningRate, idx) {
-        const input = this.input[idx];
-        const weights = this.Weights
+    backward(outputGradient, learningRate) {
+        const input = this.input;
         const inputGradient = this.gpu.CalculateBackward(outputGradient, learningRate, input)
         return inputGradient
     }
     ClearInputCache(){
         this.input = []
     }
-}
+}
+
+

package/src/SelfAttention/index.js

@@ -22,9 +22,9 @@ export class SelfAttention {
         this.value = new Linear(embeddingSize, embeddingSize, { save: { filename: [`${configs.dirname}/value.bin`, `${configs.dirname}/valueBias.bin`] } })
     }
     forward(vectors) {
-        this.q = vectors.map((e) => this.query.forward(e))
-        this.v = vectors.map((e) => this.value.forward(e))
-        this.k = vectors.map((e) => this.key.forward(e))
+        this.q = this.query.forward(vectors)
+        this.v = this.value.forward(vectors)
+        this.k = this.key.forward(vectors)
         const attentionSocres = []
         for (let i = 0; i < this.q.length; i++) {
             attentionSocres[i] = new Float32Array(this.k.length);
@@ -44,7 +44,8 @@ export class SelfAttention {
                 attentionSocres[i][j] = dot / this.score;
             }
         }
-        const weights = attentionSocres.map((e) => this.softmax.forward(e))
+        this.softmax.forward(attentionSocres)
+        const weights = this.softmax.getSoftmaxes()
         this.weights = weights
         const output = [];
 
@@ -123,4 +124,9 @@ export class SelfAttention {
         this.key.Save()
         this.value.Save()
     }
+    ClearInputCache(){
+        this.query.ClearInputCache()
+        this.key.ClearInputCache()
+        this.value.ClearInputCache()
+    }
 }

package/src/Softmax/index.d.ts

@@ -1,4 +1,6 @@
 export class Softmax{
-    forward(xLi : number[]) : number[]
-    backward(probalitils: number[] , target : number) : number[] | Float32Array[]
+    forward(xLi : number[]) : number
+    backward(target : number[]) : Float32Array[]
+    getSoftmaxes() : Float32Array[]
+    crossEntropy(dLogits: Float32Array[], target: number[])
 }

package/src/Softmax/index.js

@@ -1,13 +1,20 @@
+import { gpuSoftmax } from "../GPU.js";
+
 export class Softmax{
+    gpu;
+    constructor(contextSize){
+        this.gpu = new gpuSoftmax(contextSize)
+    }
     forward(xLi){
-        const max = Math.max(...xLi)
-        const exp = xLi.map((e)=> Math.exp(e - max))
-        const sum = exp.reduce((a,b)=> a + b , 0)
-        return new Float32Array(exp.map((e)=> e / sum))
-    }
-    backward(probs , target){
-        const pro = new Float32Array([...probs])
-        pro[target] -= 1
-        return pro
+        return this.gpu.forwardSoftmax(xLi)
+    }
+    backward(target){
+        return this.gpu.softmaxBackward(target)
+    }
+    getSoftmaxes(){
+        return this.gpu.getSoftmax()
+    }
+    crossEntropy(probs, target){
+        return this.gpu.calculateLOSS(probs, target)
     }
 }

package/src/crossEntropy/index.js

@@ -1,3 +0,0 @@
-export function crossEntropy(probs, target){
-    return -Math.log(probs[target] + 1e-10)
-}