@mineproness/tensorloop 1.4.1 → 1.4.3

package/index.js

@@ -8,6 +8,8 @@ import {ReLU} from './src/ReLU/index.js'
 import {PositionEmbedding as PE} from './src/PositionEmbedding/index.js'
 import {SelfAttention as ST} from './src/SelfAttention/index.js'
 import {GELU as GU} from './src/GELU/index.js'
+import {FFN as ffn} from './src/FFN/index.js'
+import {LayerNorm as LN} from './src/LayerNorm/index.js'
 export const Tokenizer = tokenizer
 export const Embedding = embeddding2
 export const Linear = li2
@@ -17,4 +19,6 @@ export const Tools = tools
 export const ReLu = ReLU
 export const PositionEmbedding = PE
 export const SelfAttention = ST
-export const GELU = GU
+export const GELU = GU
+export const FFN = ffn
+export const LayerNorm = LN

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mineproness/tensorloop",
-  "version": "1.4.1",
+  "version": "1.4.3",
   "description": "The most powerfull Machine Learning Library for AI.",
   "keywords": [
     "ai",

package/readme.md

@@ -1,74 +1,126 @@
-![tensor loop logo](https://i.imgur.com/I7WW3jf_d.webp?maxwidth=760&fidelity=grand)
+# TensorLoop ➰
 
-A Library that Supply AI Tools and Large Language model Development tools like Embedding, Linear, ReLU, softmax etc...
+A high-performance JavaScript library that provides essential AI and Large Language Model (LLM) development utilities. Build, train, and run neural network components like Embeddings, Linear layers, Activations, and Tokenizers directly in JavaScript.
 
-## Feature
+---
 
-- Embedding Vectors.
-- Linear Weights and Bias. 
-- Softmax forward and Backward.
-- CrossEntroy for Loss Calculation.
-- Relu and GELU for Stable Layer.
-- Tokenizer for convert into tokens.
-- Self attention 
-- Position Embedding
-- PAD token is '<PAD>' and EOS token <EOS>
+## 🚀 Features
 
-## Configs
+- **Transformer Block**: Complete modular block managing full attention and forward/backward training passes.
+- **Embedding Vectors**: High-speed lookup and embedding initialization.
+- **Linear Layer**: Fully connected layers with manageable weights and biases.
+- **Activations**: Forward and backward propagation for **Softmax**, **ReLU**, and **GELU**.
+- **Loss Functions**: **CrossEntropy** loss calculation for training classification tasks.
+- **Attention & Positioning**: Built-in **Self-Attention** and **Positional Embedding** matrices.
+- **Text Processing**: Built-in **Tokenizer** to convert raw text into token IDs and back.
+- **Special Tokens**: Native support for `<PAD>` and `<EOS>`.
 
-You can Pass a Object in a embedding and linear function. like :-
+---
 
-```js
-import {Embedding , linear} from '@mineproness/tensorloop'
+## 🏗 Core Architecture
 
+### Transformer Block
+The `TransformerBlock` encapsulates attention, layer normalization, and feed-forward networks into a single trainable module.
+
+```typescript
+import { TransformerBlock } from '@mineproness/tensorloop'
+
+// Initialize the block with an embedding size and save directory
+const transformer = new TransformerBlock(1024, {
+    dirname: "./model/transformer_layers"
+})
+
+// Forward Pass: Processes an array of typed float arrays
+const output = transformer.forward(inputVectors)
+
+// Backward Pass: Calculates gradients and updates weights using a learning rate
+const dInput = transformer.backward(dOutput, 0.001)
+
+// Clear cache to free up system memory after an iteration
+transformer.clearInputCache()
+
+// Persist the transformer block weights to disk
+transformer.Save()
+```
+
+---
+
+## 🛠 Configurations
+
+You can configure saving mechanisms by passing a configuration object to the `Embedding` and `Linear` classes.
+
+```javascript
+import { Embedding, Linear } from '@mineproness/tensorloop'
+
+// Initialize an Embedding layer and auto-save the weights
 const embedding = new Embedding(1024, 2000, {
     save: {
         filename: "./model/vectors.bin"
     }
 })
-const linear = new Embedding(1024, 2000, {
+
+// Initialize a Linear layer (requires an array to save Weights and Biases separately)
+const linear = new Linear(1024, 2000, {
     save: {
-        filename: ["./model/Weights.bin" , "./model/Bias.bin"]
+        filename: ["./model/Weights.bin", "./model/Bias.bin"]
     }
 })
 ```
 
-Linear Layer need a filename array because it need to save the Weights and Bias and always use bin extension.
+> 📌 **Note:** The `Linear` layer requires a two-element filename array to separate the Weights and Bias files. Always use the `.bin` extension.
+
+---
 
-## Optimizations
+## ⚡ Optimizations
 
-It is Highly Fast Because of Float 32 Array and it is can handle 1024 embedding Size.
-We build that with Raw Javascript for Best Optimization.
+- **Float32Array Powered**: Built on top of native typed arrays for maximum numerical performance and low memory overhead.
+- **Heavy Workloads**: Easily handles large scaling operations, including dense 1024 embedding sizes.
+- **Vanilla JavaScript**: Written entirely in raw JavaScript to eliminate bloated dependencies and optimize V8 execution.
 
-## Tokenizer
+---
 
-It is main use to Split Words and Convert into token ids that can be again Deconvert.
+## 🔤 Tokenizer
 
+The tokenizer splits raw text strings into token IDs, which can then be fully decoded back into human-readable text.
 
-like this Exmple 
+### Example Usage:
 
 ```javascript
 import { Tokenizer } from '@mineproness/tensorloop'
 
 const tokenizer = new Tokenizer()
-const text = "Hello, Wellcome to My Channel."
+const text = "Hello, Welcome to My Channel."
 
-const { tokenIDs } = tokenizer.encoder(text) // ["hello", "$12" , "well" , "come", "$130"....]
-const decoded = tokenizer.decoder(tokenIDs) // Hello, wellcome to my channel.
+// Encode text to token IDs
+const { tokenIDs } = tokenizer.encoder(text) 
+console.log(tokenIDs) // Output example: ["hello", "\$12", "well", "come", "\$130"...]
 
+// Decode token IDs back to text
+const decoded = tokenizer.decoder(tokenIDs) 
+console.log(decoded) // Output: "Hello, Welcome to My Channel."
 ```
-## Suggestion
 
-I recoommeded that use TypeScript for Type Safety and know what function is Exist in classes.
+---
+
+## 💡 Best Practices & Suggestions
 
-And Always Pair with Our tools in Our Package like
-```js
-import {Tools} from '@mineproness/tensorloop'
+### 1. Use TypeScript
+We highly recommend using **TypeScript** with TensorLoop to enforce type safety and leverage IDE autocomplete features for all active classes and methods.
+
+### 2. Pair with Native Tools
+Maximize efficiency by processing your layers with our built-in `Tools` utility suite:
+
+```javascript
+import { Tools } from '@mineproness/tensorloop'
 
 const tools = new Tools()
 
-console.log(tools.betterVector(embeedingVectors))
+// Optimize vector arrays for downstream tasks
+console.log(tools.betterVector(embeddingVectors))
 ```
-## Thanks for Visiting this Site
 
-If You watching this, Please give me a star in Github.
+---
+
+## ⭐ Support the Project
+
+Thank you for visiting! If you find this library useful for your AI or LLM workflows, please consider giving us a star on **[GitHub](https://github.com/mineproness-org/tensorloop)**!

package/src/FFN/index.d.ts

@@ -0,0 +1,9 @@
+export class FFN{
+    constructor(embeddingSize: number, hiddenLayer: "gelu" | "relu" , configs: {
+        dirname: string
+    })
+    forward(input: Float32Array[]) : Float32Array[]
+    backward(dInput: Float32Array[], LearningRate: number) : Float32Array[]
+    Save() : void;
+    ClearInputCache() : void
+}

package/src/FFN/index.js

@@ -0,0 +1,54 @@
+import { Linear } from '../Linear/index.js'
+import { GELU } from '../GELU/index.js'
+import { ReLU } from '../ReLU/index.js'
+import { join } from 'path'
+export class FFN {
+    linear1;
+    linear2;
+    hidden;
+    constructor(embeddingSize, hiddenLayer = "gelu", configs) {
+        this.hidden = hiddenLayer.toLocaleLowerCase() == "gelu" ? new GELU() : new ReLU()
+        if (configs) {
+            this.linear1 = new Linear(embeddingSize, 4 * embeddingSize, {
+                save: {
+                    filename: [join(configs.dirname, "ffnWeights.bin"), join(configs.dirname, "ffnBias.bin")]
+                }
+            })
+            this.linear2 = new Linear(4 * embeddingSize, embeddingSize, {
+                save: {
+                    filename: [join(configs.dirname, "ffnWeights1.bin"), join(configs.dirname, "ffnBias1.bin")]
+                }
+            })
+        } else {
+            this.linear1 = new Linear(embeddingSize, 4 * embeddingSize)
+            this.linear2 = new Linear(4 * embeddingSize, embeddingSize)
+        }
+    }
+    forward(input){
+        const result = []
+        for(let a = 0; a < input.length; a++){
+            const out1 = this.linear1.forward(input[a])
+            const hidden = this.hidden.forward(out1, a)
+            result[a] = this.linear2.forward(hidden)
+        }
+        return result
+    }
+    backward(dInput, LearningRate){
+        const result = []
+        for(let a = 0; a < dInput.length; a++){
+           const out2 = this.linear2.backward(dInput[a] , LearningRate, a);
+           const dHidden = this.hidden.backward(out2, a)
+           result[a] = this.linear1.backward(dHidden, LearningRate, a)
+        }
+        return result
+    }
+    Save(){
+        this.linear1.Save()
+        this.linear2.Save()
+    }
+    ClearInputCache(){
+        this.linear1.ClearInputCache()
+        this.linear2.ClearInputCache()
+        this.hidden.ClearInputCache()
+    }
+}

package/src/LayerNorm/index.d.ts

@@ -0,0 +1,22 @@
+export class LayerNorm {
+    gamma: Float32Array;
+    beta: Float32Array;
+    eps: number;
+    cache: Array<{
+        input: Float32Array;
+        mean: number;
+        invStd: number;
+    }>;
+
+    constructor(embeddingSize: number, eps?: number);
+
+    forward(input: Float32Array, index?: number): Float32Array;
+
+    backward(
+        dOutput: Float32Array,
+        learningRate: number,
+        index?: number
+    ): Float32Array;
+
+    ClearInputCache(): void;
+}

package/src/LayerNorm/index.js

@@ -0,0 +1,82 @@
+export class LayerNorm {
+    gamma;
+    beta;
+    eps;
+    cache;
+
+    constructor(embeddingSize, eps = 1e-5) {
+        this.gamma = new Float32Array(embeddingSize);
+        this.beta = new Float32Array(embeddingSize);
+
+        this.gamma.fill(1);
+        this.beta.fill(0);
+
+        this.eps = eps;
+        this.cache = [];
+    }
+
+    forward(input, index = 0) {
+        const size = input.length;
+        let mean = 0;
+        for (let i = 0; i < size; i++) {
+            mean += input[i];
+        }
+        mean /= size;
+        let variance = 0;
+        for (let i = 0; i < size; i++) {
+            const diff = input[i] - mean;
+            variance += diff * diff;
+        }
+        variance /= size;
+        const invStd = 1 / Math.sqrt(variance + this.eps);
+        const output = new Float32Array(size);
+        for (let i = 0; i < size; i++) {
+            const normalized = (input[i] - mean) * invStd;
+            output[i] = normalized * this.gamma[i] + this.beta[i];
+        }
+        this.cache[index] = {
+            input: new Float32Array(input),
+            mean,
+            invStd
+        };
+        return output;
+    }
+
+    backward(dOutput, learningRate, index = 0) {
+        const cache = this.cache[index];
+        const input = cache.input;
+        const mean = cache.mean;
+        const invStd = cache.invStd;
+        const size = input.length;
+        const dInput = new Float32Array(size);
+        let sumDNorm = 0;
+        let sumDNormX = 0;
+        const normalized = new Float32Array(size);
+        for (let i = 0; i < size; i++) {
+            normalized[i] = (input[i] - mean) * invStd;
+            const dNorm = dOutput[i] * this.gamma[i];
+            sumDNorm += dNorm;
+            sumDNormX += dNorm * normalized[i];
+            const dGamma = dOutput[i] * normalized[i];
+            const dBeta = dOutput[i];
+
+            this.gamma[i] -= learningRate * dGamma;
+            this.beta[i] -= learningRate * dBeta;
+        }
+        for (let i = 0; i < size; i++) {
+            const dNorm = dOutput[i] * this.gamma[i];
+            dInput[i] =
+                (invStd / size) *
+                (
+                    size * dNorm -
+                    sumDNorm -
+                    normalized[i] * sumDNormX
+                );
+        }
+        return dInput;
+    }
+
+    ClearInputCache() {
+        this.cache = [];
+    }
+}

package/src/Tokenizer/vocabs.json

@@ -19597,5 +19597,14 @@
   "❤️❤️",
   "👍👍",
   "<user>",
-  "<ai>"
+  "<ai>",
+  "<",
+  ">",
+  ".",
+  "{",
+  "}",
+  "?",
+  "!",
+  "@",
+  "$"
 ]

package/src/TransformerBlock/index.d.ts

@@ -0,0 +1,9 @@
+export class TransformerBlock{
+    constructor(embeddingSize: number, configs:{
+        dirname: string
+    })
+    Save(): void
+    clearInputCache() : void
+    forward(input: Float32Array[]) : Float32Array[]
+    backward(dOutput: Float32Array[], LearningRate: number) : Float32Array[]
+}

package/src/TransformerBlock/index.js

@@ -0,0 +1,92 @@
+import { SelfAttention } from '../SelfAttention/index.js'
+import { FFN } from '../FFN/index.js'
+import { LayerNorm } from '../LayerNorm/index.js'
+export class TransformerBlock {
+    norm1;
+    norm2;
+    ffn;
+    selfAttention;
+    constructor(embeddingSize, configs) {
+        this.norm1 = new LayerNorm(embeddingSize)
+        this.norm2 = new LayerNorm(embeddingSize)
+        this.ffn = new FFN(embeddingSize, "gelu", configs)
+        this.selfAttention = new SelfAttention(embeddingSize, configs)
+    }
+    forward(input) {
+        const normal1 = []
+        for (let a = 0; a < input.length; a++) {
+            normal1[a] = this.norm1.forward(input[a], a)
+        }
+        const attention = this.selfAttention.forward(normal1)
+        const residual1 = []
+        for (let a = 0; a < input.length; a++) {
+            residual1[a] = new Float32Array(input[0].length)
+            for (let i = 0; i < input[0].length; i++) {
+                residual1[a][i] = input[a][i] + attention[a][i]
+            }
+        }
+        const normal2 = []
+        for (let a = 0; a < input.length; a++) {
+            normal2[a] = this.norm2.forward(residual1[a], a)
+        }
+        const ffnOutput = this.ffn.forward(normal2)
+        const output = []
+        for (let g = 0; g < residual1.length; g++) {
+            output[g] = new Float32Array(residual1[0].length)
+            for (let a = 0; a < output[0].length; a++) {
+                output[g][a] = residual1[g][a] + ffnOutput[g][a]
+            }
+        }
+        return output
+    }
+    backward(dOutput, learningRate) {
+        const dFFN = this.ffn.backward(
+            dOutput,
+            learningRate
+        )
+        const dResidual1 = []
+
+        for (let a = 0; a < dFFN.length; a++) {
+            dResidual1[a] = this.norm2.backward(
+                dFFN[a],
+                learningRate,
+                a
+            )
+        }
+        const dAttention = this.selfAttention.backward(
+            dResidual1,
+            learningRate
+        )
+        const dInputFromAttention = []
+        for (let a = 0; a < dAttention.length; a++) {
+            dInputFromAttention[a] = this.norm1.backward(
+                dAttention[a],
+                learningRate,
+                a
+            )
+        }
+        const dInput = []
+        for (let a = 0; a < dOutput.length; a++) {
+            dInput[a] = new Float32Array(dOutput[a].length)
+            for (let i = 0; i < dOutput[a].length; i++) {
+                dInput[a][i] =
+                    dOutput[a][i] +
+                    dInputFromAttention[a][i]
+            }
+        }
+
+        return dInput
+    }
+    clearInputCache() {
+        this.ffn.clearInputCache()
+        this.selfAttention.key.ClearInputCache()
+        this.selfAttention.query.ClearInputCache()
+        this.selfAttention.value.ClearInputCache()
+        this.norm1.ClearInputCache()
+        this.norm2.ClearInputCache()
+    }
+    Save() {
+        this.selfAttention.Save()
+        this.ffn.Save()
+    }
+}