@mineproness/tensorloop 2.1.44 → 2.1.45

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mineproness/tensorloop",
-  "version": "2.1.44",
+  "version": "2.1.45",
   "description": "Raw-JAVASCRIPT Based Matirx and AI LLM Package called Tensorloop.",
   "keywords": [
     "ai",

package/src/Model/index.d.ts

@@ -0,0 +1,7 @@
+export default class Model{
+    constructor(embeddingSize: number, contextSize: number, configs: {
+        dirname: string,
+        activation: "gelu" | "relu",
+    })
+    train(text: string, batchSize: number, batchExecution: number, epochs: number, learningRate: number) : void
+ }

package/src/Model/index.js

@@ -0,0 +1,93 @@
+import { Embedding, PositionEmbedding, Linear, Tokenizer, TransformerBlock, Softmax } from '../../index.js'
+import path, { join } from 'path'
+export default class Model {
+    embedding;
+    linear;
+    posEmbedding;
+    softmax;
+    tokenizer;
+    block1;
+    block2;
+    contextSize;
+    constructor(embeddingSize, contextSize, configs) {
+        this.tokenizer = new Tokenizer()
+        this.contextSize = contextSize
+        this.embedding = new Embedding(embeddingSize, this.tokenizer.vocabSize, contextSize, {
+            save: {
+                filename: path.join(configs.dirname, "vectors.bin")
+            }
+        })
+        this.posEmbedding = new PositionEmbedding(embeddingSize, contextSize, {
+            save: {
+                filename: join(configs.dirname, 'posVec.bin')
+            }
+        })
+        this.linear = new Linear(embeddingSize, this.tokenizer.vocabSize, {
+            save: {
+                filename: [join(configs.dirname, 'weights.bin'), join(configs.dirname, 'bias.bin')]
+            }
+        })
+        this.softmax = new Softmax()
+        this.block1 = new TransformerBlock(embeddingSize, {
+            dirname: join(configs.dirname, "block1"),
+            activation: configs.activation
+        })
+        this.block2 = new TransformerBlock(embeddingSize, {
+            dirname: join(configs.dirname, "block2"),
+            activation: configs.activation
+        })
+    }
+    train(text = "", batchSize = 5000, batchExecution = 0, epochs = 500, learningRate = 0.01) {
+        process.stdout.write("\x1Bc")
+        console.log("Preparing Training Pairs....")
+        const tokens = text.split(/\r?n/).map((e) => e.trim()).filter((e) => e.length > 0).map((e)=> e + " <EOS>").map((e) => this.tokenizer.encoder(e).tokenIDs).flat()
+        const trainingPairs2 = []
+        for (let a = 0; a + this.contextSize < tokens.length; a++) {
+            trainingPairs2.push({
+                inputIDX: tokens.slice(a, a + this.contextSize),
+                targetIDX: tokens.slice(a + 1, a + this.contextSize + 1)
+            })
+        }
+        const trainingPairs = []
+        for (let a = 0; a < trainingPairs2.length; batchSize++) {
+            trainingPairs.push(trainingPairs2.slice(a, a + batchSize))
+        }
+        const trainingPair = trainingPairs[batchExecution]
+        let firstRun = true;
+        console.log(`TotalPairs: ${trainingPair.length}`)
+        if (trainingPair.length == 0 || !trainingPair) throw new Error(`The Batch Execution ${batchExecution} is doesn't exist.`);
+        for (let i = 0; i < epochs; i++) {
+            let totalLoss = 0;
+            for (let a = 0; a < trainingPair.length; a++) {
+                const input = trainingPair[a].inputIDX
+                const target = trainingPair[a].targetIDX
+                let vector = this.embedding.forward(input)
+                vector = this.posEmbedding.forward(vector)
+                vector = this.block1.forward(vector)
+                vector = this.block2.forward(vector)
+                const logits = this.linear.forward(vector)
+                const probs = this.softmax.forward(logits)
+                const loss = this.softmax.crossEntropy(probs, target)
+                const dLogits = this.softmax.backward(probs, target)
+                totalLoss += loss;
+                let dOutput = this.linear.backward(dLogits, learningRate);
+                dOutput = this.block2.backward(dOutput, learningRate)
+                dOutput = this.block1.backward(dOutput, learningRate)
+                this.posEmbedding.backward(dOutput, learningRate)
+                this.embedding.backward(input, dOutput, learningRate)
+                if (firstRun) {
+                    firstRun = null;
+                    console.log("")
+                }
+                const percentense = (((a + 1) / trainingPair.length) * 100).toFixed(2)
+                const filted = Math.round((percentense / 100) * 20);
+                const empty = 20 - filted
+                process.stdout.write(`\r[${"=".repeat(filted)}${" ".repeat(empty)}] ${percentense}% (${a + 1}/${trainingPair.length}) | loss ${(totalLoss / (a + 1)).toFixed(2)}`)
+            }
+            this.embedding.Save()
+            this.linear.Save()
+            this.block1.Save()
+            this.block2.Save()
+        }
+    }
+}