@mineproness/tensorloop 2.1.44 → 2.1.46

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mineproness/tensorloop",
-  "version": "2.1.44",
+  "version": "2.1.46",
   "description": "Raw-JAVASCRIPT Based Matirx and AI LLM Package called Tensorloop.",
   "keywords": [
     "ai",

package/src/Embedding/index.d.ts

@@ -6,6 +6,6 @@ export class Embedding{
         }
     })
     forward(token: number | number[]) : Float32Array[] | Float32Array[][]
-    backward(token: number , inputGradient: number[] | Float32Array[], learingRate: number) : void
+    backward(inputGradient: number[] | Float32Array[], learingRate: number) : void
     Save() : void
 }

package/src/Model/index.d.ts

@@ -0,0 +1,7 @@
+export default class Model{
+    constructor(embeddingSize: number, contextSize: number, configs: {
+        dirname: string,
+        activation: "gelu" | "relu",
+    })
+    train(text: string, batchSize: number, batchExecution: number, epochs: number, learningRate: number) : void
+ }

package/src/Model/index.js

@@ -0,0 +1,89 @@
+import { Embedding, PositionEmbedding, Linear, Tokenizer, TransformerBlock, Softmax } from '../../index.js'
+import path, { join } from 'path'
+export default class Model {
+    embedding;
+    linear;
+    posEmbedding;
+    softmax;
+    tokenizer;
+    block1;
+    block2;
+    contextSize;
+    constructor(embeddingSize, contextSize, configs) {
+        this.tokenizer = new Tokenizer()
+        this.contextSize = contextSize
+        this.embedding = new Embedding(embeddingSize, this.tokenizer.vocabSize, contextSize, {
+            save: {
+                filename: path.join(configs.dirname, "vectors.bin")
+            }
+        })
+        this.posEmbedding = new PositionEmbedding(embeddingSize, contextSize, {
+            save: {
+                filename: join(configs.dirname, 'posVec.bin')
+            }
+        })
+        this.linear = new Linear(embeddingSize, this.tokenizer.vocabSize, {
+            save: {
+                filename: [join(configs.dirname, 'weights.bin'), join(configs.dirname, 'bias.bin')]
+            }
+        })
+        this.softmax = new Softmax()
+        this.block1 = new TransformerBlock(embeddingSize, {
+            dirname: join(configs.dirname, "block1"),
+            activation: configs.activation
+        })
+        this.block2 = new TransformerBlock(embeddingSize, {
+            dirname: join(configs.dirname, "block2"),
+            activation: configs.activation
+        })
+    }
+    train(text = "", epochs = 500, learningRate = 0.01) {
+        process.stdout.write("\x1Bc")
+        console.log("Preparing Training Pairs....")
+        const tokens = text.split(/\r?\n/).map((e) => e.trim()).filter((e) => e.length > 0).map((e)=> e + " <EOS>").map((e) => this.tokenizer.encoder(e).tokenIDs).flat()
+        const trainingPair = []
+        for (let a = 0; a + this.contextSize < tokens.length; a++) {
+            trainingPair.push({
+                inputIDX: tokens.slice(a, a + this.contextSize),
+                targetIDX: tokens.slice(a + 1, a + this.contextSize + 1)
+            })
+        }
+        let firstRun = true;
+        console.log(`TotalPairs: ${trainingPair.length}`)
+        if (trainingPair.length == 0 || !trainingPair) throw new Error(`The Batch Execution ${batchExecution} is doesn't exist.`);
+        for (let i = 0; i < epochs; i++) {
+            let totalLoss = 0;
+            for (let a = 0; a < trainingPair.length; a++) {
+                const input = trainingPair[a].inputIDX
+                const target = trainingPair[a].targetIDX
+                let vector = this.embedding.forward(input)
+                vector = this.posEmbedding.forward(vector)
+                vector = this.block1.forward(vector)
+                vector = this.block2.forward(vector)
+                const logits = this.linear.forward(vector)
+                const probs = this.softmax.forward(logits)
+                const loss = this.softmax.crossEntropy(probs, target)[0]
+                const dLogits = this.softmax.backward(probs, target)
+                totalLoss += loss;
+                let dOutput = this.linear.backward(dLogits, learningRate);
+                dOutput = this.block2.backward(dOutput, learningRate)
+                dOutput = this.block1.backward(dOutput, learningRate)
+                this.posEmbedding.backward(dOutput, learningRate)
+                this.embedding.backward(dOutput, learningRate)
+                if (firstRun) {
+                    firstRun = null;
+                    console.log("")
+                    console.log("")
+                }
+                const percentense = (((a + 1) / trainingPair.length) * 100).toFixed(2)
+                const filted = Math.round((percentense / 100) * 20);
+                const empty = 20 - filted
+                process.stdout.write(`\r[${"=".repeat(filted)}${" ".repeat(empty)}] ${percentense}% (${a + 1}/${trainingPair.length}) | loss ${(totalLoss / (a + 1)).toFixed(2)}`)
+            }
+            this.embedding.Save()
+            this.linear.Save()
+            this.block1.Save()
+            this.block2.Save()
+        }
+    }
+}

package/src/PositionEmbedding/index.js

@@ -27,22 +27,23 @@ export class PositionEmbedding {
                 this.vectors = LoadVectors(configs.save.filename, embeddingSize)
             } else {
                 this.vectors = GenerateVectors(embeddingSize, vocabSize)
-                this.Save()
+                SaveVectors(this.vectors, this.configs.save.filename)
+
             }
         } else {
             this.vectors = GenerateVectors(embeddingSize, vocabSize)
-            this.gpu = new GPUPosition(this.vectors, embeddingSize, vocabSize)
         }
+        this.gpu = new GPUPosition(this.vectors, embeddingSize, vocabSize)
     }
     forward(vec) {
         return this.gpu.forward(vec)
     }
     backward(inputGradient, learingRate) {
-       this.gpu.backward(inputGradient, learingRate)
+        this.gpu.backward(inputGradient, learingRate)
     }
     Save() {
         const save = this.gpu.Save()
         SaveVectors(save, this.configs.save.filename)
-        
+
     }
 }