@mineproness/tensorloop 1.5.2 → 1.5.4

package/index.js

@@ -3,7 +3,6 @@ import { Embedding as embeddding2 } from "./src/Embedding/index.js";
 import { Linear as li2 } from "./src/Linear/index.js";
 import { Softmax as softmax2 } from "./src/Softmax/index.js";
 import { crossEntropy } from "./src/crossEntropy/index.js";
-import {tools} from './src/tools/index.js'
 import {ReLU} from './src/ReLU/index.js'
 import {PositionEmbedding as PE} from './src/PositionEmbedding/index.js'
 import {SelfAttention as ST} from './src/SelfAttention/index.js'
@@ -15,7 +14,6 @@ export const Embedding = embeddding2
 export const Linear = li2
 export const Softmax = softmax2
 export const CrossEntropy = crossEntropy
-export const Tools = tools
 export const ReLu = ReLU
 export const PositionEmbedding = PE
 export const SelfAttention = ST

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@mineproness/tensorloop",
-  "version": "1.5.2",
-  "description": "The most powerfull Machine Learning Library for AI.",
+  "version": "1.5.4",
+  "description": "Raw-JAVASCRIPT Based Matirx and AI LLM Package called Tensorloop.",
   "keywords": [
     "ai",
     "llm",
@@ -19,6 +19,8 @@
   "repository": {
     "type": "git",
     "url": "https://github.com/mineproness-org/tensorloop.git"
+  },
+  "dependencies": {
+    "gpu.js": "^2.24.0"
   }
-
 }

package/readme.md

@@ -14,7 +14,7 @@ A high-performance JavaScript library that provides essential AI and Large Langu
 - **Attention & Positioning**: Built-in **Self-Attention** and **Positional Embedding** matrices.
 - **Text Processing**: Built-in **Tokenizer** to convert raw text into token IDs and back.
 - **Special Tokens**: Native support for `<PAD>` and `<EOS>`.
-
+- **FFN BLOCK**: It add a layer to understand Complex Patterns.
 ---
 
 ## 🏗 Core Architecture
@@ -93,7 +93,7 @@ const text = "Hello, Welcome to My Channel."
 
 // Encode text to token IDs
 const { tokenIDs } = tokenizer.encoder(text) 
-console.log(tokenIDs) // Output example: ["hello", "\$12", "well", "come", "\$130"...]
+console.log(tokenIDs) // Output example: ["hello", "$12", "well", "come", "$130"...]
 
 // Decode token IDs back to text
 const decoded = tokenizer.decoder(tokenIDs) 
@@ -107,19 +107,15 @@ console.log(decoded) // Output: "Hello, Welcome to My Channel."
 ### 1. Use TypeScript
 We highly recommend using **TypeScript** with TensorLoop to enforce type safety and leverage IDE autocomplete features for all active classes and methods.
 
-### 2. Pair with Native Tools
-Maximize efficiency by processing your layers with our built-in `Tools` utility suite:
-
-```javascript
-import { Tools } from '@mineproness/tensorloop'
-
-const tools = new Tools()
+---
 
-// Optimize vector arrays for downstream tasks
-console.log(tools.betterVector(embeddingVectors))
-```
+### 2. Always use NodeJS 20 Higher
+This Package optimized for node js 20 or higher. If you use older version it Maybe Break Easliy.
 
 ---
+### 3. Use GELU Always.
+ 
+Gelu is Better Activation because it is not turn a gradient 0 so it is so smoothly works.
 
 ## ⭐ Support the Project
 

package/src/FFN/index.js

@@ -50,5 +50,6 @@ export class FFN {
         this.linear1.ClearInputCache()
         this.linear2.ClearInputCache()
         this.hidden.ClearInputCache()
+        
     }
 }

package/src/GPU.js

@@ -0,0 +1,167 @@
+import { GPU } from "gpu.js";
+
+const gpu = new GPU({
+    mode: "gpu"
+})
+const kernel = gpu.createKernel(function () {
+    return 1
+}).setOutput([1])
+
+kernel()
+const gl = kernel.context
+
+export function splitMatrix(matrix, chunksSizes = gl.getParameter(gl.MAX_TEXTURE_SIZE)) {
+    const chunks = []
+    for (let a = 0; a < matrix.length; a += chunksSizes) {
+        chunks.push(matrix.slice(a, a + chunksSizes))
+    }
+    return { split: true, data: chunks }
+
+}
+
+const cacheLF = []
+const cacheLBIas = []
+const cacheLBWeights = []
+const cacheLBInputGradent = []
+export function InitKernel(weights, size) {
+    const splitedWeights = splitMatrix(weights)
+    for (let a = 0; a < splitedWeights.data.length; a++) {
+        cacheLF[a] = gpu.createKernel(function (input, bias, weights) {
+            let sum = bias[this.thread.x]
+            for (let a = 0; a < this.constants.inputLength; a++) {
+                sum += input[a] * weights[this.thread.x][a]
+            }
+            return sum
+        }).setConstants({
+            inputLength: splitedWeights.data[a].length
+        }).setOutput([splitedWeights.data[a].length])
+        cacheLBIas[a] = gpu.createKernel(function (
+            bias,
+            outputGradient,
+            learningRate
+        ) {
+            const grad = Math.max(
+                -1,
+                Math.min(1, outputGradient[this.thread.x])
+            )
+
+            return bias[this.thread.x] - learningRate * grad
+        })
+            .setOutput([splitedWeights.data[a].length])
+        cacheLBInputGradent[a] = gpu.createKernel(function (
+            weights,
+            outputGradient
+        ) {
+            let sum = 0
+
+            for (let a = 0; a < this.constants.outputSize; a++) {
+                const grad = Math.max(
+                    -1,
+                    Math.min(1, outputGradient[a])
+                )
+
+                sum += grad * weights[a][this.thread.x]
+            }
+
+            return sum
+        })
+            .setConstants({
+                outputSize: splitedWeights.data[a].length
+            })
+            .setOutput([size])
+        cacheLBWeights[a] = gpu.createKernel(function (
+            weights,
+            outputGradient,
+            input,
+            learningRate
+        ) {
+            const grad = Math.max(
+                -1,
+                Math.min(1, outputGradient[this.thread.y])
+            )
+          
+            return weights[this.thread.y][this.thread.x]
+                - learningRate * input[this.thread.x] * grad
+        })
+            .setOutput([size, splitedWeights.data[a].length])
+    }
+
+
+}
+export function CalculateLinearForward(weights, bias, input) {
+    const splitedBias = splitMatrix(bias)
+    const splitedWeights = splitMatrix(weights)
+    const result = new Float32Array(weights.length)
+    let offset = 0;
+    for (let a = 0; a < splitedBias.data.length; a++) {
+        const output = cacheLF[a](input, splitedBias.data[a], splitedWeights.data[a])
+        result.set(output, offset)
+        offset += output.length
+    }
+    return result
+}
+
+export function CalculateBackward(weights, bias, out, learningRate, input) {
+    const splitedBias = splitMatrix(bias)
+    const splitedWeights = splitMatrix(weights)
+    const inputGradient = new Float32Array(input.length)
+    const biasOut = new Float32Array(bias.length)
+    const weightsOut = []
+    let offsetBias = 0
+    let offsetWeights = 0
+    let gradientOffset = 0
+
+    for (let a = 0; a < splitedWeights.data.length; a++) {
+        const weightChunk = splitedWeights.data[a]
+        const biasChunk = splitedBias.data[a]
+
+        const chunkOutputSize = weightChunk.length
+
+        // Gradient belonging to this output/weight chunk
+        const gradientChunk = out.slice(
+            gradientOffset,
+            gradientOffset + chunkOutputSize
+        )
+
+        gradientOffset += chunkOutputSize
+
+        // Bias update
+        const newBias = cacheLBIas[a](
+            biasChunk,
+            gradientChunk,
+            learningRate
+        )
+
+        biasOut.set(newBias, offsetBias)
+        offsetBias += newBias.length
+
+        // Input gradient
+        const outputGra = cacheLBInputGradent[a](
+            weightChunk,
+            gradientChunk
+        )
+
+        // ADD this chunk's gradient to the same input gradient
+        for (let i = 0; i < outputGra.length; i++) {
+            inputGradient[i] += outputGra[i]
+        }
+
+        // Weight update
+        const newWeights = cacheLBWeights[a](
+            weightChunk,
+            gradientChunk,
+            input,
+            learningRate
+        )
+
+        // Copy chunk into final Float32Array
+        weightsOut.push(...newWeights)
+        
+    }
+
+    return {
+        inputGradient,
+        biasOut,
+        weightsOut
+    }
+}

package/src/Linear/index.js

@@ -1,5 +1,6 @@
 import { LoadBias, LoadVectors, SaveBias, SaveVectors } from '../GetConfigs.js'
 import { existsSync } from 'fs'
+import { CalculateBackward, CalculateLinearForward, InitKernel } from '../GPU.js'
 function GenerateWeightsBias(embeddingSize, vocabSize) {
     const vectors = []
     const Bias = new Float32Array(vocabSize)
@@ -38,6 +39,7 @@ export class Linear {
             this.Weights = vectors;
             this.Bias = Bias;
         }
+        InitKernel(this.Weights, embeddingSize)
     }
     Save() {
         SaveVectors(this.Weights, this.configs.save.filename[0])
@@ -45,29 +47,14 @@ export class Linear {
     }
     forward(input) {
         this.input.push(input)
-        const output = new Float32Array(this.Weights.length)
-        for (let a = 0; a < this.Weights.length; a++) {
-            let sum = this.Bias[a]
-            for (let t = 0; t < input.length; t++) {
-                sum += input[t] * this.Weights[a][t]
-            }
-            output[a] = sum
-        }
-        return output
+        return CalculateLinearForward(this.Weights, this.Bias, input)
     }
     backward(outputGradient, learningRate, idx) {
         const input = this.input[idx];
         const weights = this.Weights
-        const inputGradient = new Float32Array(input.length)
-        for (let a = 0; a < weights.length; a++) {
-            const gran = Math.max(-1, Math.min(1, outputGradient[a]))
-            const w = weights[a]
-            for (let at = 0; at < input.length; at++) {
-                inputGradient[at] += gran * w[at]
-                w[at] -= learningRate * input[at] * gran
-            }
-            this.Bias[a] -= learningRate * gran
-        }
+        const {biasOut,inputGradient, weightsOut} = CalculateBackward(weights, this.Bias, outputGradient, learningRate, input)
+        this.Weights = weightsOut;
+        this.Bias = biasOut
         return inputGradient
     }
     ClearInputCache(){

package/src/Tokenizer/index.js

@@ -70,7 +70,7 @@ export class Tokenizer {
                 }
             }
         }
-        return Validate(output.filter(e=> e.trim().length > 0)).join(" ").replaceAll(/\s+([,.;'{}`!~\|":><?])/g, "$1")
+        return Validate(output.filter(e=> e.trim().length > 0)).join(" ").replaceAll(/\s+([,.;'{}`!~\"?])/g, "$1")
     }
 }
 
@@ -86,7 +86,6 @@ function Validate(output = ["hello", "guys"]) {
                 bestOutput[idx] = e[0].toLocaleUpperCase() + e.slice(1, e.length)
             } else {
                 bestOutput[idx] = e
-
             }
         }
 

package/src/Tokenizer/vocabs.json

@@ -19606,5 +19606,13 @@
   "?",
   "!",
   "@",
-  "$"
+  "$",
+  ":",
+  "/",
+  "+",
+  "-",
+  "=",
+  "%",
+  "*",
+  "@"
 ]

package/src/TransformerBlock/index.js

@@ -12,13 +12,8 @@ export class TransformerBlock {
         return this.ffn.forward(attention)
     }
     backward(dOutput, learningRate) {
-        const dAttention =
-            this.ffn.backward(dOutput, learningRate)
-
-        return this.selfAttention.backward(
-            dAttention,
-            learningRate
-        )
+        const dAttention = this.ffn.backward(dOutput, learningRate)
+        return this.selfAttention.backward(dAttention, learningRate)
     }
     clearInputCache() {
         this.ffn.ClearInputCache()

package/src/tools/index.d.ts

@@ -1,3 +0,0 @@
-export class tools{
-    betterVector(vectors: number[] | Float32Array[]) : string
-}

package/src/tools/index.js

@@ -1,5 +0,0 @@
-export class tools{
-    betterVector(vectors){
-        return [...vectors].map((e)=> Number(e.toFixed(2)))
-    }
-}