@mineproness/tensorloop 1.4.8 → 1.4.9

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mineproness/tensorloop",
-  "version": "1.4.8",
+  "version": "1.4.9",
   "description": "The most powerfull Machine Learning Library for AI.",
   "keywords": [
     "ai",

package/src/TransformerBlock/index.js

@@ -1,70 +1,30 @@
 import { SelfAttention } from '../SelfAttention/index.js'
 import { FFN } from '../FFN/index.js'
-import { LayerNorm } from '../LayerNorm/index.js'
 export class TransformerBlock {
-    norm1;
-    norm2;
     ffn;
     selfAttention;
     constructor(embeddingSize, configs) {
-        this.norm1 = new LayerNorm(embeddingSize)
-        this.norm2 = new LayerNorm(embeddingSize)
         this.ffn = new FFN(embeddingSize, "gelu", configs)
         this.selfAttention = new SelfAttention(embeddingSize, configs)
     }
     forward(input) {
-        const normal1 = []
-        for (let a = 0; a < input.length; a++) {
-            normal1[a] = this.norm1.forward(input[a], a)
-        }
         const attention = this.selfAttention.forward(normal1)
-        const normal2 = []
-        for (let a = 0; a < attention.length; a++) {
-            normal2[a] = this.norm2.forward(attention[a], a)
-        }
         return this.ffn.forward(normal2)
     }
     backward(dOutput, learningRate) {
-        const dFFN =
-            this.ffn.backward(dOutput, learningRate)
-
-        const dAttentionInput = []
-
-        for (let a = 0; a < dFFN.length; a++) {
-            dAttentionInput[a] =
-                this.norm2.backward(
-                    dFFN[a],
-                    learningRate,
-                    a
-                )
-        }
-
         const dAttention =
-            this.selfAttention.backward(
-                dAttentionInput,
-                learningRate
-            )
-
-        const dInput = []
-
-        for (let a = 0; a < dAttention.length; a++) {
-            dInput[a] =
-                this.norm1.backward(
-                    dAttention[a],
-                    learningRate,
-                    a
-                )
-        }
+            this.ffn.backward(dOutput, learningRate)
 
-        return dInput
+        return this.selfAttention.backward(
+            dAttention,
+            learningRate
+        )
     }
     clearInputCache() {
         this.ffn.ClearInputCache()
         this.selfAttention.key.ClearInputCache()
         this.selfAttention.query.ClearInputCache()
         this.selfAttention.value.ClearInputCache()
-        this.norm1.ClearInputCache()
-        this.norm2.ClearInputCache()
     }
     Save() {
         this.selfAttention.Save()

package/src/LayerNorm/index.d.ts

@@ -1,22 +0,0 @@
-export class LayerNorm {
-    gamma: Float32Array;
-    beta: Float32Array;
-    eps: number;
-    cache: Array<{
-        input: Float32Array;
-        mean: number;
-        invStd: number;
-    }>;
-
-    constructor(embeddingSize: number, eps?: number);
-
-    forward(input: Float32Array, index?: number): Float32Array;
-
-    backward(
-        dOutput: Float32Array,
-        learningRate: number,
-        index?: number
-    ): Float32Array;
-
-    ClearInputCache(): void;
-}

package/src/LayerNorm/index.js

@@ -1,98 +0,0 @@
-export class LayerNorm {
-    gamma;
-    beta;
-    eps;
-    cache;
-
-    constructor(embeddingSize, eps = 1e-5) {
-        this.gamma = new Float32Array(embeddingSize);
-        this.beta = new Float32Array(embeddingSize);
-
-        this.gamma.fill(1);
-        this.beta.fill(0);
-
-        this.eps = eps;
-        this.cache = [];
-    }
-
-    forward(input, index = 0) {
-        const size = input.length;
-        let mean = 0;
-        for (let i = 0; i < size; i++) {
-            mean += input[i];
-        }
-        mean /= size;
-        let variance = 0;
-        for (let i = 0; i < size; i++) {
-            const diff = input[i] - mean;
-            variance += diff * diff;
-        }
-        variance /= size;
-        const invStd = 1 / Math.sqrt(variance + this.eps);
-        const output = new Float32Array(size);
-        for (let i = 0; i < size; i++) {
-            const normalized = (input[i] - mean) * invStd;
-            output[i] = normalized * this.gamma[i] + this.beta[i];
-        }
-        this.cache[index] = {
-            input: new Float32Array(input),
-            mean,
-            invStd
-        };
-        return output;
-    }
-
-    backward(dOutput, learningRate, index = 0) {
-        const cache = this.cache[index];
-        const input = cache.input;
-        const mean = cache.mean;
-        const invStd = cache.invStd;
-        const size = input.length;
-        const dInput = new Float32Array(size);
-        const normalized = new Float32Array(size);
-        let sumDNorm = 0;
-        let sumDNormX = 0;
-        for (let i = 0; i < size; i++) {
-            normalized[i] =
-                (input[i] - mean) * invStd;
-        }
-        for (let i = 0; i < size; i++) {
-            const dNorm =
-                dOutput[i] * this.gamma[i];
-
-            sumDNorm += dNorm;
-            sumDNormX +=
-                dNorm * normalized[i];
-        }
-        for (let i = 0; i < size; i++) {
-            const dNorm =
-                dOutput[i] * this.gamma[i];
-            dInput[i] =
-                (invStd / size) *
-                (
-                    size * dNorm -
-                    sumDNorm -
-                    normalized[i] * sumDNormX
-                );
-        }
-        for (let i = 0; i < size; i++) {
-            const dGamma =
-                dOutput[i] * normalized[i];
-
-            const dBeta =
-                dOutput[i];
-
-            this.gamma[i] -=
-                learningRate * dGamma;
-
-            this.beta[i] -=
-                learningRate * dBeta;
-        }
-
-        return dInput;
-    }
-
-    ClearInputCache() {
-        this.cache = [];
-    }
-}