@mineproness/tensorloop 2.1.54 → 2.1.56

package/src/Relu/index.js

@@ -0,0 +1,103 @@
+import { OperationManager } from "../OperationManager.js";
+
+export class Relu {
+    embeddingSize;
+    paramBuffer;
+    oldParam;
+    forwardPipline;
+    gpu;
+    backwardPipline;
+    input;
+    constructor(device, embeddingSize) {
+        this.embeddingSize = embeddingSize
+        this.gpu = new OperationManager(device);
+        this.device = device;
+        this.forwardPipline = this.gpu.getShaderPipline("./src/shaders/RF.wgsl")
+        this.backwardPipline = this.gpu.getShaderPipline("./src/shaders/RB.wgsl")
+    }
+    async forward(input) {
+        const encoder = this.device.createCommandEncoder()
+        const tokenCount = input.size / (this.embeddingSize * 4)
+        if (!this.oldParam) {
+            this.oldParam = new Uint32Array([0, 0, 0]);
+            this.paramBuffer = await this.gpu.createUniFormBuffer(4 * 3);
+        }
+        const output = await this.gpu.createBuffer(input.size)
+        if (this.oldParam[1] !== tokenCount) {
+            this.oldParam = new Uint32Array([this.embeddingSize, tokenCount, 0])
+            await this.gpu.WriteBuffer(this.paramBuffer, this.oldParam)
+        }
+        const bind = await this.device.createBindGroup({
+            layout: await this.forwardPipline.getBindGroupLayout(0),
+            entries: [
+                {
+                    binding: 0,
+                    resource: {
+                        buffer: input
+                    }
+                },
+                {
+                    binding: 1,
+                    resource: {
+                        buffer: this.paramBuffer
+                    }
+                }, {
+                    binding: 2,
+                    resource: {
+                        buffer: output
+                    }
+                }
+            ]
+        })
+        await this.gpu.RunPipline(encoder, this.forwardPipline, bind, Math.ceil((tokenCount * this.embeddingSize) / 256))
+        this.device.queue.submit([encoder.finish()])
+        if(this.input) this.input.destroy()
+        this.input = input
+        return output
+    }
+    async backward(inputGradient) {
+        const encoder = this.device.createCommandEncoder()
+        const tokenCount = inputGradient.size / (this.embeddingSize * 4)
+        if (!this.oldParam) {
+            this.oldParam = new Uint32Array([0, 0, 0]);
+            this.paramBuffer = await this.gpu.createUniFormBuffer(4 * 3);
+        }
+        const output = await this.gpu.createBuffer(inputGradient.size)
+        if (this.oldParam[1] !== tokenCount) {
+            this.oldParam = new Uint32Array([this.embeddingSize, tokenCount, 0])
+            await this.gpu.WriteBuffer(this.paramBuffer, this.oldParam)
+        }
+        const bind = await this.device.createBindGroup({
+            layout: await this.backwardPipline.getBindGroupLayout(0),
+            entries: [
+                {
+                    binding: 0,
+                    resource: {
+                        buffer: this.input
+                    }
+                },
+                {
+                    binding: 1,
+                    resource: {
+                        buffer: this.paramBuffer
+                    }
+                }, {
+                    binding: 2,
+                    resource: {
+                        buffer: output
+                    }
+                }, {
+                    binding: 3,
+                    resource: {
+                        buffer: inputGradient
+                    }
+                }
+            ]
+        })
+        await this.gpu.RunPipline(encoder, this.backwardPipline, bind, Math.ceil((tokenCount * this.embeddingSize) / 256))
+        await this.device.queue.submit([encoder.finish()])
+        this.input.destroy()
+        inputGradient.destroy()
+        return output
+    }
+}

package/src/SelfAttention/index.d.ts

@@ -1,15 +1,9 @@
 export class SelfAttention{
-    q : Float32Array[]
-    v : Float32Array[]
-    k : Float32Array[]
-    input;
-    constructor(embeddingSize : number, configs: {
-            dirname: string
-        
+    constructor(device : GPUAdapter,embeddingSize: number, configs : {
+        dirname: string
     })
-    weights : Float32Array[]
-    score : number
-    forward(vectors: Float32Array[][])
-    Save() : void
-    backward(dOutput, learningRate)
-}
+    async forward(vec : GPUBuffer) : Promise<GPUBuffer>
+    async backward(dO : GPUBuffer, lr : number) : Promise<GPUBuffer>
+    async ClearInputCache() : Promise<void>
+    async Save() : Promise<void>
+} 

package/src/SelfAttention/index.js

@@ -1,67 +1,120 @@
-import { Softmax } from '../../index.js';
-import { GPUAttention } from '../gpiInit.js';
-import { Linear } from '../Linear/index.js'
+import { Linear } from "../../index.js";
+import { join } from 'path'
+import { OperationManager } from "../OperationManager.js";
 
 export class SelfAttention {
     query;
-    key;
     value;
-    v;
-    k;
+    key;
     q;
-    input;
-    configs;
-    score;
-    softmax;
-    weights;
+    k;
+    v;
+    embeddingSize;
+    params;
+    attentionscoresShader;
+    softmaxForward;
     gpu;
-    constructor(embeddingSize, configs) {
-        this.configs = configs
-        this.softmax = new Softmax()
-        this.score = Math.sqrt(embeddingSize)
-        this.query = new Linear(embeddingSize, embeddingSize, { save: { filename: [`${configs.dirname}/query.bin`, `${configs.dirname}/queryBias.bin`] } })
-        this.key = new Linear(embeddingSize, embeddingSize, { save: { filename: [`${configs.dirname}/key.bin`, `${configs.dirname}/keyBias.bin`] } })
-        this.value = new Linear(embeddingSize, embeddingSize, { save: { filename: [`${configs.dirname}/value.bin`, `${configs.dirname}/valueBias.bin`] } })
-    }
-    forward(vectors) {
-        if(!this.gpu) this.gpu = new GPUAttention(vectors[0].length, vectors.length);
-        this.q = this.query.forward(vectors)
-        this.v = this.value.forward(vectors)
-        this.k = this.key.forward(vectors)
-        const attentionSocres = this.gpu.attentionScores(this.q, this.k, this.score);
-        const weights = this.softmax.forward(attentionSocres)
-        this.weights = weights
-        const output = this.gpu.attentionOutput(weights, this.v)
-        return output;
+    input;
+    constructor(device, embeddingSize, configs) {
+        this.embeddingSize = embeddingSize;
+        const dd = [1, 2, 3].map((e) => {
+            return {
+                save: {
+                    filename: [join(configs.dirname, `SELF-W-${e}.bin`), join(configs.dirname, `SELF-B-${e}.bin`)]
+                }
+            }
+        })
+        this.q = new Linear(device, embeddingSize, embeddingSize, dd[0])
+        this.k = new Linear(device, embeddingSize, embeddingSize, dd[1])
+        this.v = new Linear(device, embeddingSize, embeddingSize, dd[2])
+        this.gpu = new OperationManager(device)
+        this.attentionscoresShader = this.gpu.getShaderPipline("./src/shaders/attentionShader/attentionSocres.wgsl")
+        this.softmaxForward = this.gpu.getShaderPipline("./src/shaders/SF.wgsl")
+        this.softmaxBackward = this.gpu.getShaderPipline("./src/shaders/SoftmaxBackward.wgsl")
+        this.outputForward = this.gpu.getShaderPipline("./src/shaders/attentionShader/CompueFinalForwardOutput.wgsl")
+        this.dVShader = this.gpu.getShaderPipline("./src/shaders/attentionShader/ComputeDV.wgsl")
+        this.dAshader = this.gpu.getShaderPipline("./src/shaders/attentionShader/ComputeDA.wgsl")
+        this.dQshader = this.gpu.getShaderPipline("./src/shaders/attentionShader/ComputedQ.wgsl")
+        this.dKShader = this.gpu.getShaderPipline("./src/shaders/attentionShader/ComputeK.wgsl")
+        this.sumShader = this.gpu.getShaderPipline("./src/shaders/attentionShader/sumGradient.wgsl")
     }
+    async forward(vec) {
+        if (this.input) await this.input.destroy()
+        this.input = vec;
+        const tokenCount = vec.size / (this.embeddingSize * 4)
+        if (!this.asParams) this.asParams = await this.gpu.createUniFormBuffer(4 * 3);
+        if (!this.softmaxParams) this.softmaxParams = await this.gpu.createUniFormBuffer(4 * 3);
+        const encoder = await this.gpu.CreateEncoder()
+        const scroes = await this.gpu.createBuffer(tokenCount * tokenCount * 4)
+        await this.gpu.WriteBuffer(this.asParams, new Uint32Array([tokenCount, this.embeddingSize, 0]))
+        await this.gpu.WriteBuffer(this.softmaxParams, new Uint32Array([tokenCount, tokenCount, 0]))
+        if (this.query) this.query.destroy()
+        if (this.key) this.key.destroy()
+        if (this.value) this.value.destroy()
+        this.query = await this.q.forward(vec)
+        this.key = await this.k.forward(vec)
+        this.value = await this.v.forward(vec)
+        const bindAttention = await this.gpu.createBindGroup(this.attentionscoresShader, 0, [this.query, this.key, scroes, this.asParams])
+        await this.gpu.RunPipline(encoder, this.attentionscoresShader, bindAttention, Math.ceil(tokenCount / 16), Math.ceil(tokenCount / 16))
+        if (this.weights) this.weights.destroy()
+        this.weights = await this.gpu.createBuffer(tokenCount * tokenCount * 4)
+        const bindSF = await this.gpu.createBindGroup(this.softmaxForward, 0, [scroes, this.weights, this.softmaxParams]);
+        await this.gpu.RunPipline(encoder, this.softmaxForward, bindSF, tokenCount)
+        const finalOutput = await this.gpu.createBuffer(tokenCount * this.embeddingSize * 4)
+        const outputBind = await this.gpu.createBindGroup(this.outputForward, 0, [this.weights, this.value, finalOutput, this.asParams]);
+        await this.gpu.RunPipline(encoder, this.outputForward, outputBind, Math.ceil((tokenCount * this.embeddingSize) / 256))
 
-    backward(dOutput, learningRate) {
-        const dWeights = this.gpu.dWeightsKernel(dOutput, this.v)
-        const dV = this.gpu.dVKernel(dOutput, this.weights);
-        const dScores = this.gpu.softmaxBackKernel(this.weights, dWeights)
-        const dQ = this.gpu.dQKernel(dScores, this.k, this.score)
-        const dK = this.gpu.dKKernel(dScores, this.q, this.score)
-        const dInput = this.q.map(() => new Float32Array(this.q[0].length))
-        const dq = this.query.backward(dQ, learningRate)
-        const dk = this.key.backward(dK, learningRate)
-        const dv = this.value.backward(dV, learningRate)
-        for (let a = 0; a < this.q.length; a++) {
-            for (let b = 0; b < dq[a].length; b++) {
-                dInput[a][b] = dq[a][b] + dk[a][b] + dv[a][b]
-            }
-        }
-        dWeights.delete()
-        dScores.delete()
-        return dInput
+        await this.gpu.submitQueue(encoder)
+        scroes.destroy()
+        return finalOutput
+    }
+    async backward(dO, lr) {
+        const tokenCount = dO.size / (this.embeddingSize * 4);
+        const encoder = await this.gpu.CreateEncoder();
+        const dV = await this.gpu.createBuffer(dO.size);
+        const DvBind = await this.gpu.createBindGroup(this.dVShader, 0, [this.weights, dO, dV, this.asParams]);
+        await this.gpu.RunPipline(encoder, this.dVShader, DvBind, Math.ceil(tokenCount / 16), Math.ceil(this.embeddingSize / 16));
+        const dA = await this.gpu.createBuffer(tokenCount * tokenCount * 4)
+        const DaBind = await this.gpu.createBindGroup(this.dAshader, 0, [dO, this.value, dA, this.asParams]);
+        await this.gpu.RunPipline(encoder, this.dAshader, DaBind, Math.ceil(tokenCount / 16), Math.ceil(tokenCount / 16))
+        const dS = await this.gpu.createBuffer(tokenCount * tokenCount * 4);
+        const dSBind = await this.gpu.createBindGroup(this.softmaxBackward, 0, [this.weights, dA, dS, this.asParams])
+        await this.gpu.RunPipline(encoder, this.softmaxBackward, dSBind, Math.ceil((tokenCount * tokenCount) / 256))
+        const dQ = await this.gpu.createBuffer(tokenCount * this.embeddingSize * 4);
+        const dQbind = await this.gpu.createBindGroup(this.dQshader, 0, [dS, this.key, dQ, this.asParams]);
+        await this.gpu.RunPipline(encoder, this.dQshader, dQbind, Math.ceil(tokenCount / 16), Math.ceil(this.embeddingSize / 16))
+        const dK = await this.gpu.createBuffer(this.embeddingSize * tokenCount * 4)
+        const dKbind = await this.gpu.createBindGroup(this.dKShader, 0, [dS, this.query, dK, this.asParams]);
+        await this.gpu.RunPipline(encoder, this.dKShader, dKbind, Math.ceil(tokenCount / 16), Math.ceil(this.embeddingSize / 16))
+        await this.gpu.submitQueue(encoder);
+        const dq = await this.q.backward(dQ, lr)
+        const dk = await this.k.backward(dK, lr)
+        const dv = await this.v.backward(dV, lr)
+        const output = await this.gpu.createBuffer(this.embeddingSize * tokenCount * 4);
+        const encode = this.gpu.CreateEncoder()
+        const outputBind = await this.gpu.createBindGroup(this.sumShader, 0 , [dq, dk, dv, output, this.softmaxParams]);
+        await this.gpu.RunPipline(encode, this.sumShader, outputBind, Math.ceil((tokenCount * this.embeddingSize / 256)));
+        await this.gpu.submitQueue(encode)
+        dS.destroy()
+        dA.destroy()
+        dq.destroy()
+        dk.destroy()
+        dv.destroy()
+        dK.destroy()
+        dQ.destroy()
+        dV.destroy()
+        dO.destroy()
+        return output
+        
     }
-    Save() {
-        this.query.Save()
-        this.key.Save()
-        this.value.Save()
+    async ClearInputCache(){
+       await this.q.ClearInputCache()
+       await this.k.ClearInputCache()
+       await this.v.ClearInputCache()
     }
-    ClearInputCache() {
-        this.query.ClearInputCache()
-        this.key.ClearInputCache()
-        this.value.ClearInputCache()
+    async Save(){
+       await this.q.Save()
+       await this.q.Save()
+       await this.q.Save()
     }
 }

package/src/Softmax/index.d.ts

@@ -1,5 +1,6 @@
 export class Softmax{
-    forward(xLi : number[]) : number
-    backward(probs: Float32Array[], target : number[]) : Float32Array[]
-    crossEntropy(probs: Float32Array[], target: number[])
+    constructor(device: GPUAdapter, embeddingSize: number, vocabSize: number);
+    async forward(logit: GPUBuffer, isCpuReadBack) : Promise<Float32Array | GPUBuffer>;
+    async backward(probs : GPUBuffer, target : number[]) : Promise<GPUBuffer>
+    async crossEntropy(probs: GPUBuffer, target: number[]) : Promise<{loss: Float32Array, acc: Float32Array}>
 }

package/src/Softmax/index.js

@@ -1,18 +1,198 @@
-import { GPUSoftmax } from "../gpiInit.js";
+import { log } from 'console';
+import { OperationManager } from '../OperationManager.js'
 
-
-export class Softmax{
+export class Softmax {
+    device;
+    vocabSize;
+    forwardPipline;
+    embeddingSize;
     gpu;
-    forward(xLi){
-        if(!this.gpu){
-            this.gpu = new GPUSoftmax(xLi, xLi.length)
+    paramsForward;
+    oldParam;
+    bp
+    constructor(device, embeddingSize, vocabSize) {
+        this.vocabSize = vocabSize;
+        this.device = device;
+        this.embeddingSize = embeddingSize;
+        this.gpu = new OperationManager(device);
+        this.forwardPipline = this.gpu.getShaderPipline("./src/shaders/SF.wgsl")
+        this.backwardPipline = this.gpu.getShaderPipline("./src/shaders/SB.wgsl")
+        this.lossPipline = this.gpu.getShaderPipline("./src/shaders/CalculateLoss.wgsl")
+        this.accPipline = this.gpu.getShaderPipline("./src/shaders/CalculateAcc.wgsl")
+        this.battention = this.gpu.getShaderPipline("./src/shaders/SoftmaxBackward.wgsl")
+        this.reduceFn = this.gpu.getShaderPipline("./src/shaders/ReductionFunction.wgsl")
+    }
+    async forward(logits, cpuReadBack, encoderCPU) {
+        const encoder = encoderCPU ? encoderCPU : this.device.createCommandEncoder()
+        const tokenCount = logits.size / (this.vocabSize * 4)
+        if (!this.paramsForward) {
+            this.paramsForward = await this.gpu.createUniFormBuffer(4 * 3)
         }
-        return this.gpu.forward(xLi)
+        if (!this.oldParam) {
+            this.oldParam = new Uint32Array([this.vocabSize, 0, 0])
+        }
+
+        if (this.oldParam[1] !== tokenCount) { await this.gpu.WriteBuffer(this.paramsForward, new Uint32Array([this.vocabSize, tokenCount, 0])); this.oldParam = new Uint32Array([this.vocabSize, tokenCount, 0]); }
+        const output = await this.gpu.createBuffer(logits.size);
+        const bind = await this.device.createBindGroup({
+            layout: await this.forwardPipline.getBindGroupLayout(0),
+            entries: [
+                {
+                    binding: 0,
+                    resource: {
+                        buffer: logits
+                    }
+                },
+                {
+                    binding: 1,
+                    resource: {
+                        buffer: output
+                    }
+                }, {
+                    binding: 2,
+                    resource: {
+                        buffer: this.paramsForward
+                    }
+                }
+            ]
+        })
+        await this.gpu.RunPipline(encoder, this.forwardPipline, bind, Math.ceil((tokenCount * this.vocabSize) / 256))
+        if (!encoderCPU) {
+            await this.device.queue.submit([encoder.finish()])
+            logits.destroy()
+        }
+        if (cpuReadBack) {
+            const outputCPU = await this.gpu.readBuffer(output.size, output);
+            return outputCPU
+        }
+        return output
     }
-    backward(xLi , target){
-        return this.gpu.backward(xLi, target)
+    async backward(probs, target) {
+        const encoder = this.device.createCommandEncoder()
+        const tokenCount = probs.size / (this.vocabSize * 4)
+        if (!this.paramsForward) {
+            this.paramsForward = await this.gpu.createUniFormBuffer(4 * 3)
+        }
+        if (!this.oldParam) {
+            this.oldParam = new Uint32Array([this.vocabSize, 0, 0])
+        }
+        if (this.oldParam[1] !== tokenCount) {
+            await this.gpu.WriteBuffer(this.paramsForward, new Uint32Array([this.vocabSize, tokenCount, 0]));
+            this.oldParam = new Uint32Array([this.vocabSize, tokenCount, 0]);
+        }
+        const targetBuffer = await this.gpu.createBuffer(target.length * Float32Array.BYTES_PER_ELEMENT)
+        await this.gpu.WriteBuffer(targetBuffer, new Float32Array(target))
+        const output = await this.gpu.createBuffer(probs.size)
+        const bind = await this.device.createBindGroup({
+            layout: await this.backwardPipline.getBindGroupLayout(0),
+            entries: [
+                {
+                    binding: 0,
+                    resource: {
+                        buffer: probs
+                    }
+                },
+                {
+                    binding: 1,
+                    resource: {
+                        buffer: output
+                    }
+                }, {
+                    binding: 2,
+                    resource: {
+                        buffer: targetBuffer
+                    }
+                }, {
+                    binding: 3,
+                    resource: {
+                        buffer: this.paramsForward
+                    }
+                }
+            ]
+        })
+        await this.gpu.RunPipline(encoder, this.backwardPipline, bind, Math.ceil((tokenCount * this.vocabSize) / 256))
+        this.device.queue.submit([encoder.finish()])
+        probs.destroy()
+        targetBuffer.destroy()
+        return output
     }
-    crossEntropy(probs, target){
-        return this.gpu.crossEntropy(probs, target)
+    async crossEntropy(probs, target) {
+        const encoder = this.device.createCommandEncoder()
+
+        const tokenCount = probs.size / (this.vocabSize * 4)
+        const targetBuffer = await this.gpu.createBuffer(target.length * Float32Array.BYTES_PER_ELEMENT)
+        const loss = await this.gpu.createBuffer(4 * tokenCount)
+        const acc = await this.gpu.createBuffer(4 * 2)
+        await this.gpu.WriteBuffer(targetBuffer, new Float32Array(target))
+        const bind = await this.device.createBindGroup({
+            layout: await this.lossPipline.getBindGroupLayout(0),
+            entries: [
+                {
+                    binding: 0,
+                    resource: {
+                        buffer: probs
+                    }
+                },
+                {
+                    binding: 1,
+                    resource: {
+                        buffer: targetBuffer
+                    }
+                }, {
+                    binding: 2,
+                    resource: {
+                        buffer: loss
+                    }
+                }, {
+                    binding: 3,
+                    resource: {
+                        buffer: this.paramsForward
+                    }
+
+                },
+            ]
+        })
+        await this.gpu.RunPipline(encoder, this.lossPipline, bind, Math.ceil((this.vocabSize * tokenCount) / 256))
+        const outputLoss = await this.gpu.createBuffer(4)
+        const reduceBind = await this.gpu.createBindGroup(this.reduceFn, 0, [loss, outputLoss, this.paramsForward])
+        await this.gpu.RunPipline(encoder, this.reduceFn, reduceBind, 1);
+        const bind2 = await this.device.createBindGroup({
+            layout: await this.accPipline.getBindGroupLayout(0),
+            entries: [
+                {
+                    binding: 0,
+                    resource: {
+                        buffer: this.paramsForward
+                    }
+                },
+                {
+                    binding: 1,
+                    resource: {
+                        buffer: probs
+                    }
+                }, {
+                    binding: 3,
+                    resource: {
+                        buffer: targetBuffer
+                    }
+                },
+                {
+                    binding: 2,
+                    resource: {
+                        buffer: acc
+                    }
+                }
+            ]
+        })
+        await this.gpu.RunPipline(encoder, this.accPipline, bind2, 1)
+        this.device.queue.submit([encoder.finish()])
+        targetBuffer.destroy()
+        loss.destroy()
+        const lossCpu = await this.gpu.readBuffer(outputLoss.size, outputLoss)
+        const accCPU = await this.gpu.readBuffer(acc.size, acc)
+        acc.destroy()
+        return { loss: lossCpu[0], acc: accCPU[0] }
     }
-}
+
+}
+

package/src/Tokenizer/index.d.ts

@@ -8,4 +8,5 @@ export class Tokenizer{
         tokenIDs: number[]
     }
     decoder(tokens: number[] | number[][]) : string
+    trainTokenizer(text: string, mergeSize: number, filename: string) : void
 }

package/src/Tokenizer/index.js

@@ -1,4 +1,4 @@
-import { readFileSync } from 'fs'
+import fs, { existsSync, readFileSync } from 'fs'
 import { join, dirname, format } from 'path';
 import { fileURLToPath } from 'url';
 
@@ -9,14 +9,20 @@ export class Tokenizer {
     IdtoWord = {};
     WordtoID = {};
     EOSToken = 2
-    constructor() {
-        this.vocabs = JSON.parse(readFileSync(join(__dirname, "vocabs.json"), "utf-8"))
+    constructor(filename="") {
+        if (fs.existsSync(filename)) {
+            console.log("Warning: Tokenizer is loaded with a Custom Vocab File!")
+            console.log("")
+            this.vocabs = JSON.parse(readFileSync(filename, "utf-8"))
+        } else {
+            this.vocabs = JSON.parse(readFileSync(join(__dirname, "vocabs.json"), "utf-8"))
+        }
         this.vocabSize = this.vocabs.length
         this.IdtoWord = Object.fromEntries(this.vocabs.map((e, idx) => [idx, e]))
         this.WordtoID = Object.fromEntries(this.vocabs.map((e, idx) => [e, idx]))
     }
     encoder(text = "") {
-        const AddSpaces = text.toLocaleLowerCase().replaceAll(/([,.;'{}!`~\|":><?])/g, " $1 ")
+        const AddSpaces = text.toLocaleLowerCase().replaceAll(/([,.;'{}!`~\|":><?])/g, " $1 ").replaceAll("<" , " < ")
         const splitedWords = AddSpaces.split(/\s+/).filter(e => e.length > 0)
         const output = []
         for (let a = 0; a < splitedWords.length; a++) {
@@ -34,9 +40,9 @@ export class Tokenizer {
                             return true
                         }
                         return false
-                    
+
                     }).reduce((prev, curr) => prev && curr, true)
-                    // if(wordsliced.length == 0) console.error("Some Word Doesn't Exist in Vocabs List. so It can be undefined!")
+                    if (wordsliced.length == 0) output.push("<unk>")
                     if (matched) {
                         wordsliced.map((e) => output.push(e))
                         output.push("$130")
@@ -70,6 +76,85 @@ export class Tokenizer {
                 }
             }
         }
-        return output.filter(e=> e.trim().length > 0).join(" ").replaceAll(/\s+([,.;'{}`!~\"?])/g, "$1")
+        return output.filter(e => e.trim().length > 0).join(" ").replaceAll(/\s+([,.;'{}`!~\"?])/g, "$1")
+    }
+    trainTokenizer(text = "ejrnienieijr", mergeLength = 5, vocabFileName) {
+        console.log("Training The Tokenizer....")
+        const arrText = text.toLocaleLowerCase().replaceAll("\r\n", "").replaceAll("\n", "").replaceAll(/([,.;'{}!`~\|":><?])/g, " $1 ").split(/\s+/).map(e => e.trim()).filter((e) => e.length > 0)
+        const reg = new RegExp(`.{0,${mergeLength}}`, "g")
+        const prevVocab = existsSync(vocabFileName) ? JSON.parse(readFileSync(vocabFileName, "utf-8")) : [];
+        const specal = ["<unk>",
+            "<PAD>",
+            "<EOS>",
+            "$12",
+            "$130",
+            "😀",
+            "😃",
+            "😄",
+            "😁",
+            "😆",
+            "😅",
+            "😂",
+            "🤣",
+            "😊",
+            "😎",
+            "❤️",
+            "👍",
+            "👎",
+            "🔥",
+            "💯",
+            "🎉",
+            "🚀",
+            "⭐",
+            "😭",
+            "😍",
+            "🥰",
+            "😡",
+            "🤔",
+            "😢",
+            "😮",
+            "👋",
+            "🙏",
+            "💀",
+            "🤖",
+            "👀",
+            "💕",
+            "💔",
+            "😂😂",
+            "😂😭",
+            "🔥🔥",
+            "❤️❤️",
+            "👍👍",
+            "<",
+            ">",
+            ".",
+            "{",
+            "}",
+            "?",
+            "!",
+            "@",
+            "$",
+            ":",
+            "/",
+            "+",
+            "-",
+            "=",
+            "%",
+            "*",
+            "@"
+        ]
+        const uniArr = [...new Set([...arrText, ...specal, ...prevVocab])];
+        const output = []
+        for(let a = 0 ; a < uniArr.length; a++){
+            let text = uniArr[a].match(reg).filter((e)=> e.length > 0);
+            output.push(...text)
+        }
+        let finalOutput = [...new Set([...output])]
+        fs.writeFileSync(vocabFileName, JSON.stringify(finalOutput, null, 2), "utf-8");
+        console.log("_________________________________")
+        console.log(`New VocabSize : ${finalOutput.length - prevVocab.length}`);
+        console.log("_________________________________")
+        console.log("Operation Completed!")
+        
     }
 }