@mineproness/tensorloop 1.4.7 → 1.4.8

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mineproness/tensorloop",
-  "version": "1.4.7",
+  "version": "1.4.8",
   "description": "The most powerfull Machine Learning Library for AI.",
   "keywords": [
     "ai",

package/src/TransformerBlock/index.js

@@ -18,62 +18,42 @@ export class TransformerBlock {
             normal1[a] = this.norm1.forward(input[a], a)
         }
         const attention = this.selfAttention.forward(normal1)
-        const residual1 = []
-        for (let a = 0; a < input.length; a++) {
-            residual1[a] = new Float32Array(input[0].length)
-            for (let i = 0; i < input[0].length; i++) {
-                residual1[a][i] = input[a][i] + attention[a][i]
-            }
-        }
         const normal2 = []
-        for (let a = 0; a < input.length; a++) {
-            normal2[a] = this.norm2.forward(residual1[a], a)
-        }
-        const ffnOutput = this.ffn.forward(normal2)
-        const output = []
-        for (let g = 0; g < residual1.length; g++) {
-            output[g] = new Float32Array(residual1[0].length)
-            for (let a = 0; a < output[0].length; a++) {
-                output[g][a] = residual1[g][a] + ffnOutput[g][a]
-            }
+        for (let a = 0; a < attention.length; a++) {
+            normal2[a] = this.norm2.forward(attention[a], a)
         }
-        return output
+        return this.ffn.forward(normal2)
     }
     backward(dOutput, learningRate) {
-        const dFFN = this.ffn.backward(dOutput, learningRate)
-        const dResidual1 = []
+        const dFFN =
+            this.ffn.backward(dOutput, learningRate)
+
+        const dAttentionInput = []
+
         for (let a = 0; a < dFFN.length; a++) {
-            const dNorm2 = this.norm2.backward(
-                dFFN[a],
-                learningRate,
-                a
-            )
-            dResidual1[a] = new Float32Array(dOutput[a].length)
-            for (let i = 0; i < dOutput[a].length; i++) {
-                dResidual1[a][i] =
-                    dOutput[a][i] + dNorm2[i]
-            }
+            dAttentionInput[a] =
+                this.norm2.backward(
+                    dFFN[a],
+                    learningRate,
+                    a
+                )
         }
-        const dAttention = this.selfAttention.backward(
-            dResidual1,
-            learningRate
-        )
-        const dInputFromAttention = []
-        for (let a = 0; a < dAttention.length; a++) {
-            dInputFromAttention[a] = this.norm1.backward(
-                dAttention[a],
-                learningRate,
-                a
+
+        const dAttention =
+            this.selfAttention.backward(
+                dAttentionInput,
+                learningRate
             )
-        }
+
         const dInput = []
-        for (let a = 0; a < dOutput.length; a++) {
-            dInput[a] = new Float32Array(dOutput[a].length)
-            for (let i = 0; i < dOutput[a].length; i++) {
-                dInput[a][i] =
-                    dOutput[a][i] +
-                    dInputFromAttention[a][i]
-            }
+
+        for (let a = 0; a < dAttention.length; a++) {
+            dInput[a] =
+                this.norm1.backward(
+                    dAttention[a],
+                    learningRate,
+                    a
+                )
         }
 
         return dInput