@mineproness/tensorloop 2.1.11 → 2.1.42

package/index.js

@@ -8,6 +8,7 @@ import {SelfAttention as ST} from './src/SelfAttention/index.js'
 import {GELU as GU} from './src/GELU/index.js'
 import {FFN as ffn} from './src/FFN/index.js'
 import { TransformerBlock as TB} from './src/TransformerBlock/index.js'
+import Model from "./src/Model/index.js";
 export const Tokenizer = tokenizer
 export const Embedding = embeddding2
 export const Linear = li2
@@ -17,4 +18,5 @@ export const PositionEmbedding = PE
 export const SelfAttention = ST
 export const GELU = GU
 export const FFN = ffn
-export const TransformerBlock = TB
+export const TransformerBlock = TB
+export const model = Model

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mineproness/tensorloop",
-  "version": "2.1.11",
+  "version": "2.1.42",
   "description": "Raw-JAVASCRIPT Based Matirx and AI LLM Package called Tensorloop.",
   "keywords": [
     "ai",

package/src/PositionEmbedding/index.d.ts

@@ -5,8 +5,7 @@ export class PositionEmbedding{
             filename: string
         }
     })
-    forward(position: number | number[]) : Float32Array[] | Float32Array[][]
-    backward(pos: number , inputGradient: number[] | Float32Array[], learingRate: number) : void
+    forward(vectors: Float32Array[]) :Float32Array[]
+    backward(inputGradient: Float32Array[], learingRate: number) : void
     Save() : void;
-    addPos(vectors : Float32Array[][]) : Float32Array 
 }

package/src/PositionEmbedding/index.js

@@ -1,7 +1,7 @@
 import fs, { existsSync } from 'fs'
 import { LoadVectors, SaveVectors } from '../GetConfigs.js'
 // import { config } from 'process'
-import { GPUEmbedding } from '../gpiInit.js'
+import { GPUEmbedding, GPUPosition } from '../gpiInit.js'
 
 function GenerateVectors(embeddingSize, vocabSize) {
     const vector = []
@@ -31,39 +31,18 @@ export class PositionEmbedding {
             }
         } else {
             this.vectors = GenerateVectors(embeddingSize, vocabSize)
-            this.gpu = new GPUEmbedding(this.vectors, vocabSize, embeddingSize, 1)
+            this.gpu = new GPUPosition(this.vectors, embeddingSize, vocabSize)
         }
     }
-    forward(token) {
-        if (Array.isArray(token)) {
-            return token.map((e) => {
-                return this.vectors[e]
-            })
-        } else {
-
-            return this.vectors[token]
-        }
+    forward(vec) {
+        return this.gpu.forward(vec)
     }
-    backward(token, inputGradient, learingRate) {
-        for (let a = 0; a < this.vectors[token].length; a++) {
-            this.vectors[token][a] -= learingRate * inputGradient[a]
-        }
+    backward(inputGradient, learingRate) {
+       this.gpu.backward(inputGradient, learingRate)
     }
     Save() {
-        SaveVectors(this.vectors, this.configs.save.filename)
-    }
-    addPos(xEM) {
-        const out = []
-
-        for (let pos = 0; pos < xEM.length; pos++) {
-            const tokenVector = xEM[pos]
-            const posVector = this.forward([pos])[0]
-            const x = new Float32Array(xEM[0].length)
-            for (let a = 0; a < tokenVector.length; a++) {
-                x[a] = tokenVector[a] + posVector[a]
-            }
-            out.push(x)
-        }
-        return out
+        const save = this.gpu.Save()
+        SaveVectors(save, this.configs.save.filename)
+        
     }
 }

package/src/SelfAttention/index.js

@@ -50,6 +50,8 @@ export class SelfAttention {
                 dInput[a][b] = dq[a][b] + dk[a][b] + dv[a][b]
             }
         }
+        dWeights.delete()
+        dScores.delete()
         return dInput
     }
     Save() {

package/src/Softmax/index.d.ts

@@ -1,5 +1,5 @@
 export class Softmax{
     forward(xLi : number[]) : number
-    backward(target : number[]) : Float32Array[]
-    crossEntropy(dLogits: Float32Array[], target: number[])
+    backward(probs: Float32Array[], target : number[]) : Float32Array[]
+    crossEntropy(probs: Float32Array[], target: number[])
 }

package/src/Tokenizer/index.js

@@ -70,25 +70,6 @@ export class Tokenizer {
                 }
             }
         }
-        return Validate(output.filter(e=> e.trim().length > 0)).join(" ").replaceAll(/\s+([,.;'{}`!~\"?])/g, "$1")
+        return output.filter(e=> e.trim().length > 0).join(" ").replaceAll(/\s+([,.;'{}`!~\"?])/g, "$1")
     }
-}
-
-
-function Validate(output = ["hello", "guys"]) {
-    const bestOutput = []
-    output.map((e, idx) => {
-        if (idx == 0) {
-            bestOutput[idx] = e[0].toLocaleUpperCase() + e.slice(1, e.length)
-        } else {
-            if (bestOutput[idx - 1] == ":" || bestOutput[idx - 1] == ".") {
-
-                bestOutput[idx] = e[0].toLocaleUpperCase() + e.slice(1, e.length)
-            } else {
-                bestOutput[idx] = e
-            }
-        }
-
-    })
-    return bestOutput
 }

package/src/gpiInit.js

@@ -194,23 +194,19 @@ export class GPUSoftmax {
     cacheSF = [];
     cacheSB = []
     contextSize;
-    calculateLoss;
     avgLoss;
     constructor(logits, contextSize) {
         this.contextSize = contextSize;
-        this.avgLoss = gpu.createKernel(function (losses) {
+        this.avgLoss = gpu.createKernel(function (probs, target) {
             let sum = 0;
             for (let i = 0; i < this.constants.contextSize; i++) {
-                sum += losses[i]
+                sum += -Math.log(probs[i][target[i]] + 1e-15)
             }
             return sum / this.constants.contextSize
         }, {
             output: [1],
             constants: { contextSize }
         })
-        this.calculateLoss = gpu.createKernel(function (probs, target) {
-            return -Math.log(probs[this.thread.x][target[this.thread.x]] + 1e-9)
-        }, { output: [contextSize], immutable: true, pipeline: true })
         this.cacheSumCalculate = gpu.createKernel(function (maxLogits, logits) {
             const maxLogit = maxLogits[this.thread.x]
             let sum = 0;
@@ -261,12 +257,14 @@ export class GPUSoftmax {
         const maxLogits = this.cacheMaxCache(logits)
         const sumes = this.cacheSumCalculate(maxLogits, logits)
         const splitedLogits = split2DByColumns(logits)
-        const result = Array.from({ length: this.contextSize }, () => [])
+        const result = Array.from({ length: this.contextSize }, () => new Float32Array(logits[0].length))
+        let offset = 0
         for (let i = 0; i < splitedLogits.length; i++) {
             const out = this.cacheSF[i](splitedLogits[i], sumes, maxLogits)
             for (let a = 0; a < this.contextSize; a++) {
-                result[a].push(...out[a])
+                result[a].set(out[a], offset)
             }
+            offset += out[0].length
         }
         maxLogits.delete()
         sumes.delete()
@@ -274,19 +272,19 @@ export class GPUSoftmax {
     }
     backward(probs, target) {
         const splitedProbs = split2DByColumns(probs)
-        const result = Array.from({ length: this.contextSize }, () => [])
+        const result = Array.from({ length: this.contextSize }, () => new Float32Array(probs[0].length))
+        let offset = 0;
         for (let i = 0; i < this.cacheSB.length; i++) {
             const out = this.cacheSB[i](splitedProbs[i], target)
             for (let a = 0; a < this.contextSize; a++) {
-                result[a].push(...out[a])
+                result[a].set(out[a], offset)
             }
+            offset += out[0].length
         }
         return result
     }
     crossEntropy(probs, target) {
-        const losses = this.calculateLoss(probs, target)
-        const loss = this.avgLoss(losses)
-        losses.delete()
+        const loss = this.avgLoss(probs, target)
         return loss
     }
 
@@ -718,4 +716,44 @@ export class GPUAttention {
             }
         });
     }
+}
+
+
+export class GPUPosition {
+    cachePF;
+    cachePB;
+    posVectors
+    constructor(posVectors, embeddingSize, contextSize) {
+        const vramKernel = gpu.createKernel(function (posVec) {
+            return posVec[this.thread.y][this.thread.x]
+        }, {
+            output: [embeddingSize, contextSize],
+            immutable: true,
+            pipeline: true
+        })
+        this.posVectors = vramKernel(posVectors)
+        this.cachePF = gpu.createKernel(function (posVectors, vectors) {
+            return vectors[this.thread.y][this.thread.x] + posVectors[this.thread.y][this.thread.x]
+        }, {
+            output: [embeddingSize, contextSize]
+        })
+        this.cachePB = gpu.createKernel(function (outGradient, learingRate, oldVec) {
+            return oldVec[this.thread.y][this.thread.x] - learingRate * outGradient[this.thread.y][this.thread.x]
+        }, {
+            output: [embeddingSize, contextSize],
+            immutable: true,
+            pipeline: true,
+        })
+    }
+    forward(vectors){
+        return this.cachePF(this.posVectors, vectors)
+    }
+    backward(outGradient, learingRate){
+        const oldVec = this.posVectors;
+        this.posVectors = this.cachePB(outGradient, learingRate, oldVec)
+        oldVec.delete()
+    }
+    Save(){
+        return this.posVectors.toArray()
+    }
 }