@mineproness/tensorloop 1.4.7 → 1.4.9

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mineproness/tensorloop",
-  "version": "1.4.7",
+  "version": "1.4.9",
   "description": "The most powerfull Machine Learning Library for AI.",
   "keywords": [
     "ai",

package/src/TransformerBlock/index.js

@@ -1,90 +1,30 @@
 import { SelfAttention } from '../SelfAttention/index.js'
 import { FFN } from '../FFN/index.js'
-import { LayerNorm } from '../LayerNorm/index.js'
 export class TransformerBlock {
-    norm1;
-    norm2;
     ffn;
     selfAttention;
     constructor(embeddingSize, configs) {
-        this.norm1 = new LayerNorm(embeddingSize)
-        this.norm2 = new LayerNorm(embeddingSize)
         this.ffn = new FFN(embeddingSize, "gelu", configs)
         this.selfAttention = new SelfAttention(embeddingSize, configs)
     }
     forward(input) {
-        const normal1 = []
-        for (let a = 0; a < input.length; a++) {
-            normal1[a] = this.norm1.forward(input[a], a)
-        }
         const attention = this.selfAttention.forward(normal1)
-        const residual1 = []
-        for (let a = 0; a < input.length; a++) {
-            residual1[a] = new Float32Array(input[0].length)
-            for (let i = 0; i < input[0].length; i++) {
-                residual1[a][i] = input[a][i] + attention[a][i]
-            }
-        }
-        const normal2 = []
-        for (let a = 0; a < input.length; a++) {
-            normal2[a] = this.norm2.forward(residual1[a], a)
-        }
-        const ffnOutput = this.ffn.forward(normal2)
-        const output = []
-        for (let g = 0; g < residual1.length; g++) {
-            output[g] = new Float32Array(residual1[0].length)
-            for (let a = 0; a < output[0].length; a++) {
-                output[g][a] = residual1[g][a] + ffnOutput[g][a]
-            }
-        }
-        return output
+        return this.ffn.forward(normal2)
     }
     backward(dOutput, learningRate) {
-        const dFFN = this.ffn.backward(dOutput, learningRate)
-        const dResidual1 = []
-        for (let a = 0; a < dFFN.length; a++) {
-            const dNorm2 = this.norm2.backward(
-                dFFN[a],
-                learningRate,
-                a
-            )
-            dResidual1[a] = new Float32Array(dOutput[a].length)
-            for (let i = 0; i < dOutput[a].length; i++) {
-                dResidual1[a][i] =
-                    dOutput[a][i] + dNorm2[i]
-            }
-        }
-        const dAttention = this.selfAttention.backward(
-            dResidual1,
+        const dAttention =
+            this.ffn.backward(dOutput, learningRate)
+
+        return this.selfAttention.backward(
+            dAttention,
             learningRate
         )
-        const dInputFromAttention = []
-        for (let a = 0; a < dAttention.length; a++) {
-            dInputFromAttention[a] = this.norm1.backward(
-                dAttention[a],
-                learningRate,
-                a
-            )
-        }
-        const dInput = []
-        for (let a = 0; a < dOutput.length; a++) {
-            dInput[a] = new Float32Array(dOutput[a].length)
-            for (let i = 0; i < dOutput[a].length; i++) {
-                dInput[a][i] =
-                    dOutput[a][i] +
-                    dInputFromAttention[a][i]
-            }
-        }
-
-        return dInput
     }
     clearInputCache() {
         this.ffn.ClearInputCache()
         this.selfAttention.key.ClearInputCache()
         this.selfAttention.query.ClearInputCache()
         this.selfAttention.value.ClearInputCache()
-        this.norm1.ClearInputCache()
-        this.norm2.ClearInputCache()
     }
     Save() {
         this.selfAttention.Save()

package/src/LayerNorm/index.d.ts

@@ -1,22 +0,0 @@
-export class LayerNorm {
-    gamma: Float32Array;
-    beta: Float32Array;
-    eps: number;
-    cache: Array<{
-        input: Float32Array;
-        mean: number;
-        invStd: number;
-    }>;
-
-    constructor(embeddingSize: number, eps?: number);
-
-    forward(input: Float32Array, index?: number): Float32Array;
-
-    backward(
-        dOutput: Float32Array,
-        learningRate: number,
-        index?: number
-    ): Float32Array;
-
-    ClearInputCache(): void;
-}

package/src/LayerNorm/index.js

@@ -1,98 +0,0 @@
-export class LayerNorm {
-    gamma;
-    beta;
-    eps;
-    cache;
-
-    constructor(embeddingSize, eps = 1e-5) {
-        this.gamma = new Float32Array(embeddingSize);
-        this.beta = new Float32Array(embeddingSize);
-
-        this.gamma.fill(1);
-        this.beta.fill(0);
-
-        this.eps = eps;
-        this.cache = [];
-    }
-
-    forward(input, index = 0) {
-        const size = input.length;
-        let mean = 0;
-        for (let i = 0; i < size; i++) {
-            mean += input[i];
-        }
-        mean /= size;
-        let variance = 0;
-        for (let i = 0; i < size; i++) {
-            const diff = input[i] - mean;
-            variance += diff * diff;
-        }
-        variance /= size;
-        const invStd = 1 / Math.sqrt(variance + this.eps);
-        const output = new Float32Array(size);
-        for (let i = 0; i < size; i++) {
-            const normalized = (input[i] - mean) * invStd;
-            output[i] = normalized * this.gamma[i] + this.beta[i];
-        }
-        this.cache[index] = {
-            input: new Float32Array(input),
-            mean,
-            invStd
-        };
-        return output;
-    }
-
-    backward(dOutput, learningRate, index = 0) {
-        const cache = this.cache[index];
-        const input = cache.input;
-        const mean = cache.mean;
-        const invStd = cache.invStd;
-        const size = input.length;
-        const dInput = new Float32Array(size);
-        const normalized = new Float32Array(size);
-        let sumDNorm = 0;
-        let sumDNormX = 0;
-        for (let i = 0; i < size; i++) {
-            normalized[i] =
-                (input[i] - mean) * invStd;
-        }
-        for (let i = 0; i < size; i++) {
-            const dNorm =
-                dOutput[i] * this.gamma[i];
-
-            sumDNorm += dNorm;
-            sumDNormX +=
-                dNorm * normalized[i];
-        }
-        for (let i = 0; i < size; i++) {
-            const dNorm =
-                dOutput[i] * this.gamma[i];
-            dInput[i] =
-                (invStd / size) *
-                (
-                    size * dNorm -
-                    sumDNorm -
-                    normalized[i] * sumDNormX
-                );
-        }
-        for (let i = 0; i < size; i++) {
-            const dGamma =
-                dOutput[i] * normalized[i];
-
-            const dBeta =
-                dOutput[i];
-
-            this.gamma[i] -=
-                learningRate * dGamma;
-
-            this.beta[i] -=
-                learningRate * dBeta;
-        }
-
-        return dInput;
-    }
-
-    ClearInputCache() {
-        this.cache = [];
-    }
-}