@mineproness/tensorloop 2.1.3 → 2.1.4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mineproness/tensorloop",
-  "version": "2.1.3",
+  "version": "2.1.4",
   "description": "Raw-JAVASCRIPT Based Matirx and AI LLM Package called Tensorloop.",
   "keywords": [
     "ai",

package/src/PositionEmbedding/index.d.ts

@@ -5,8 +5,7 @@ export class PositionEmbedding{
             filename: string
         }
     })
-    forward(position: number | number[]) : Float32Array[] | Float32Array[][]
-    backward(pos: number , inputGradient: number[] | Float32Array[], learingRate: number) : void
+    forward(vectors: Float32Array[]) :Float32Array[]
+    backward(inputGradient: Float32Array[], learingRate: number) : void
     Save() : void;
-    addPos(vectors : Float32Array[][]) : Float32Array 
 }

package/src/PositionEmbedding/index.js

@@ -1,7 +1,7 @@
 import fs, { existsSync } from 'fs'
 import { LoadVectors, SaveVectors } from '../GetConfigs.js'
 // import { config } from 'process'
-import { GPUEmbedding } from '../gpiInit.js'
+import { GPUEmbedding, GPUPosition } from '../gpiInit.js'
 
 function GenerateVectors(embeddingSize, vocabSize) {
     const vector = []
@@ -31,39 +31,18 @@ export class PositionEmbedding {
             }
         } else {
             this.vectors = GenerateVectors(embeddingSize, vocabSize)
-            this.gpu = new GPUEmbedding(this.vectors, vocabSize, embeddingSize, 1)
+            this.gpu = new GPUPosition(this.vectors, embeddingSize, vocabSize)
         }
     }
-    forward(token) {
-        if (Array.isArray(token)) {
-            return token.map((e) => {
-                return this.vectors[e]
-            })
-        } else {
-
-            return this.vectors[token]
-        }
+    forward(vec) {
+        return this.gpu.forward(vec)
     }
-    backward(token, inputGradient, learingRate) {
-        for (let a = 0; a < this.vectors[token].length; a++) {
-            this.vectors[token][a] -= learingRate * inputGradient[a]
-        }
+    backward(inputGradient, learingRate) {
+       this.gpu.backward(inputGradient, learingRate)
     }
     Save() {
-        SaveVectors(this.vectors, this.configs.save.filename)
-    }
-    addPos(xEM) {
-        const out = []
-
-        for (let pos = 0; pos < xEM.length; pos++) {
-            const tokenVector = xEM[pos]
-            const posVector = this.forward([pos])[0]
-            const x = new Float32Array(xEM[0].length)
-            for (let a = 0; a < tokenVector.length; a++) {
-                x[a] = tokenVector[a] + posVector[a]
-            }
-            out.push(x)
-        }
-        return out
+        const save = this.gpu.Save()
+        SaveVectors(save, this.configs.save.filename)
+        
     }
 }

package/src/Tokenizer/index.js

@@ -70,25 +70,6 @@ export class Tokenizer {
                 }
             }
         }
-        return Validate(output.filter(e=> e.trim().length > 0)).join(" ").replaceAll(/\s+([,.;'{}`!~\"?])/g, "$1")
+        return output.filter(e=> e.trim().length > 0).join(" ").replaceAll(/\s+([,.;'{}`!~\"?])/g, "$1")
     }
-}
-
-
-function Validate(output = ["hello", "guys"]) {
-    const bestOutput = []
-    output.map((e, idx) => {
-        if (idx == 0) {
-            bestOutput[idx] = e[0].toLocaleUpperCase() + e.slice(1, e.length)
-        } else {
-            if (bestOutput[idx - 1] == ":" || bestOutput[idx - 1] == ".") {
-
-                bestOutput[idx] = e[0].toLocaleUpperCase() + e.slice(1, e.length)
-            } else {
-                bestOutput[idx] = e
-            }
-        }
-
-    })
-    return bestOutput
 }

package/src/gpiInit.js

@@ -194,23 +194,19 @@ export class GPUSoftmax {
     cacheSF = [];
     cacheSB = []
     contextSize;
-    calculateLoss;
     avgLoss;
     constructor(logits, contextSize) {
         this.contextSize = contextSize;
-        this.avgLoss = gpu.createKernel(function (losses) {
+        this.avgLoss = gpu.createKernel(function (probs, target) {
             let sum = 0;
             for (let i = 0; i < this.constants.contextSize; i++) {
-                sum += losses[i]
+                sum += -Math.log(probs[i][target[i]] + 1e-15)
             }
             return sum / this.constants.contextSize
         }, {
             output: [1],
             constants: { contextSize }
         })
-        this.calculateLoss = gpu.createKernel(function (probs, target) {
-            return -Math.log(probs[this.thread.x][target[this.thread.x]] + 1e-9)
-        }, { output: [contextSize], immutable: true, pipeline: true })
         this.cacheSumCalculate = gpu.createKernel(function (maxLogits, logits) {
             const maxLogit = maxLogits[this.thread.x]
             let sum = 0;
@@ -266,7 +262,7 @@ export class GPUSoftmax {
         for (let i = 0; i < splitedLogits.length; i++) {
             const out = this.cacheSF[i](splitedLogits[i], sumes, maxLogits)
             for (let a = 0; a < this.contextSize; a++) {
-                result[a].set(out[a] , offset)
+                result[a].set(out[a], offset)
             }
             offset += out[0].length
         }
@@ -288,9 +284,7 @@ export class GPUSoftmax {
         return result
     }
     crossEntropy(probs, target) {
-        const losses = this.calculateLoss(probs, target)
-        const loss = this.avgLoss(losses)
-        losses.delete()
+        const loss = this.avgLoss(probs, target)
         return loss
     }
 
@@ -722,4 +716,44 @@ export class GPUAttention {
             }
         });
     }
+}
+
+
+export class GPUPosition {
+    cachePF;
+    cachePB;
+    posVectors
+    constructor(posVectors, embeddingSize, contextSize) {
+        const vramKernel = gpu.createKernel(function (posVec) {
+            return posVec[this.thread.y][this.thread.x]
+        }, {
+            output: [embeddingSize, contextSize],
+            immutable: true,
+            pipeline: true
+        })
+        this.posVectors = vramKernel(posVectors)
+        this.cachePF = gpu.createKernel(function (posVectors, vectors) {
+            return vectors[this.thread.y][this.thread.x] + posVectors[this.thread.y][this.thread.x]
+        }, {
+            output: [embeddingSize, contextSize]
+        })
+        this.cachePB = gpu.createKernel(function (outGradient, learingRate, oldVec) {
+            return oldVec[this.thread.y][this.thread.x] - learingRate * outGradient[this.thread.y][this.thread.x]
+        }, {
+            output: [embeddingSize, contextSize],
+            immutable: true,
+            pipeline: true,
+        })
+    }
+    forward(vectors){
+        return this.cachePF(this.posVectors, vectors)
+    }
+    backward(outGradient, learingRate){
+        const oldVec = this.posVectors;
+        this.posVectors = this.cachePB(outGradient, learingRate, oldVec)
+        oldVec.delete()
+    }
+    Save(){
+        return this.posVectors.toArray()
+    }
 }