@mineproness/tensorloop 1.4.6 → 1.4.8

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mineproness/tensorloop",
-  "version": "1.4.6",
+  "version": "1.4.8",
   "description": "The most powerfull Machine Learning Library for AI.",
   "keywords": [
     "ai",

package/src/LayerNorm/index.js

@@ -49,22 +49,24 @@ export class LayerNorm {
         const invStd = cache.invStd;
         const size = input.length;
         const dInput = new Float32Array(size);
+        const normalized = new Float32Array(size);
         let sumDNorm = 0;
         let sumDNormX = 0;
-        const normalized = new Float32Array(size);
         for (let i = 0; i < size; i++) {
-            normalized[i] = (input[i] - mean) * invStd;
-            const dNorm = dOutput[i] * this.gamma[i];
-            sumDNorm += dNorm;
-            sumDNormX += dNorm * normalized[i];
-            const dGamma = dOutput[i] * normalized[i];
-            const dBeta = dOutput[i];
+            normalized[i] =
+                (input[i] - mean) * invStd;
+        }
+        for (let i = 0; i < size; i++) {
+            const dNorm =
+                dOutput[i] * this.gamma[i];
 
-            this.gamma[i] -= learningRate * dGamma;
-            this.beta[i] -= learningRate * dBeta;
+            sumDNorm += dNorm;
+            sumDNormX +=
+                dNorm * normalized[i];
         }
         for (let i = 0; i < size; i++) {
-            const dNorm = dOutput[i] * this.gamma[i];
+            const dNorm =
+                dOutput[i] * this.gamma[i];
             dInput[i] =
                 (invStd / size) *
                 (
@@ -73,6 +75,20 @@ export class LayerNorm {
                     normalized[i] * sumDNormX
                 );
         }
+        for (let i = 0; i < size; i++) {
+            const dGamma =
+                dOutput[i] * normalized[i];
+
+            const dBeta =
+                dOutput[i];
+
+            this.gamma[i] -=
+                learningRate * dGamma;
+
+            this.beta[i] -=
+                learningRate * dBeta;
+        }
+
         return dInput;
     }
 

package/src/TransformerBlock/index.js

@@ -18,62 +18,42 @@ export class TransformerBlock {
             normal1[a] = this.norm1.forward(input[a], a)
         }
         const attention = this.selfAttention.forward(normal1)
-        const residual1 = []
-        for (let a = 0; a < input.length; a++) {
-            residual1[a] = new Float32Array(input[0].length)
-            for (let i = 0; i < input[0].length; i++) {
-                residual1[a][i] = input[a][i] + attention[a][i]
-            }
-        }
         const normal2 = []
-        for (let a = 0; a < input.length; a++) {
-            normal2[a] = this.norm2.forward(residual1[a], a)
-        }
-        const ffnOutput = this.ffn.forward(normal2)
-        const output = []
-        for (let g = 0; g < residual1.length; g++) {
-            output[g] = new Float32Array(residual1[0].length)
-            for (let a = 0; a < output[0].length; a++) {
-                output[g][a] = residual1[g][a] + ffnOutput[g][a]
-            }
+        for (let a = 0; a < attention.length; a++) {
+            normal2[a] = this.norm2.forward(attention[a], a)
         }
-        return output
+        return this.ffn.forward(normal2)
     }
     backward(dOutput, learningRate) {
-        const dFFN = this.ffn.backward(dOutput, learningRate)
-        const dResidual1 = []
+        const dFFN =
+            this.ffn.backward(dOutput, learningRate)
+
+        const dAttentionInput = []
+
         for (let a = 0; a < dFFN.length; a++) {
-            const dNorm2 = this.norm2.backward(
-                dFFN[a],
-                learningRate,
-                a
-            )
-            dResidual1[a] = new Float32Array(dOutput[a].length)
-            for (let i = 0; i < dOutput[a].length; i++) {
-                dResidual1[a][i] =
-                    dOutput[a][i] + dNorm2[i]
-            }
+            dAttentionInput[a] =
+                this.norm2.backward(
+                    dFFN[a],
+                    learningRate,
+                    a
+                )
         }
-        const dAttention = this.selfAttention.backward(
-            dResidual1,
-            learningRate
-        )
-        const dInputFromAttention = []
-        for (let a = 0; a < dAttention.length; a++) {
-            dInputFromAttention[a] = this.norm1.backward(
-                dAttention[a],
-                learningRate,
-                a
+
+        const dAttention =
+            this.selfAttention.backward(
+                dAttentionInput,
+                learningRate
             )
-        }
+
         const dInput = []
-        for (let a = 0; a < dOutput.length; a++) {
-            dInput[a] = new Float32Array(dOutput[a].length)
-            for (let i = 0; i < dOutput[a].length; i++) {
-                dInput[a][i] =
-                    dOutput[a][i] +
-                    dInputFromAttention[a][i]
-            }
+
+        for (let a = 0; a < dAttention.length; a++) {
+            dInput[a] =
+                this.norm1.backward(
+                    dAttention[a],
+                    learningRate,
+                    a
+                )
         }
 
         return dInput