@mineproness/tensorloop 1.3.9 → 1.4.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mineproness/tensorloop",
-  "version": "1.3.9",
+  "version": "1.4.1",
   "description": "The most powerfull Machine Learning Library for AI.",
   "keywords": [
     "ai",

package/readme.md

@@ -12,6 +12,7 @@ A Library that Supply AI Tools and Large Language model Development tools like E
 - Tokenizer for convert into tokens.
 - Self attention 
 - Position Embedding
+- PAD token is '<PAD>' and EOS token <EOS>
 
 ## Configs
 

package/src/Linear/index.js

@@ -56,38 +56,21 @@ export class Linear {
         return output
     }
     backward(outputGradient, learningRate, idx) {
-        const input = this.input[idx]
+        const input = this.input[idx];
         const weights = this.Weights
         const inputGradient = new Float32Array(input.length)
-
-        const MAX_UPDATE = learningRate  * 0.1
-
         for (let a = 0; a < weights.length; a++) {
-            const gran = outputGradient[a]
+            const gran = Math.max(-1, Math.min(1, outputGradient[a]))
             const w = weights[a]
-
             for (let at = 0; at < input.length; at++) {
                 inputGradient[at] += gran * w[at]
-
-                let update = learningRate * input[at] * gran
-
-                if (update > MAX_UPDATE) update = MAX_UPDATE
-                if (update < -MAX_UPDATE) update = -MAX_UPDATE
-
-                w[at] -= update
+                w[at] -= learningRate * input[at] * gran
             }
-
-            let biasUpdate = learningRate * gran
-
-            if (biasUpdate > MAX_UPDATE) biasUpdate = MAX_UPDATE
-            if (biasUpdate < -MAX_UPDATE) biasUpdate = -MAX_UPDATE
-
-            this.Bias[a] -= biasUpdate
+            this.Bias[a] -= learningRate * gran
         }
-
         return inputGradient
     }
-    ClearInputCache() {
+    ClearInputCache(){
         this.input = []
     }
 }

package/src/Tokenizer/index.js

@@ -8,6 +8,7 @@ export class Tokenizer {
     vocabSize = 0;
     IdtoWord = {};
     WordtoID = {};
+    EOSToken = 2
     constructor() {
         this.vocabs = JSON.parse(readFileSync(join(__dirname, "vocabs.json"), "utf-8"))
         this.vocabSize = this.vocabs.length