@imaduddin/fastbrain 1.1.1-patch → 2.0.0

package/src/Relu/index.js

@@ -1,108 +0,0 @@
-import { OperationManager } from "../OperationManager.js";
-
-export class Relu {
-    embeddingSize;
-    paramBuffer;
-    oldParam;
-    forwardPipline;
-    gpu;
-    backwardPipline;
-    input;
-    constructor(device, embeddingSize) {
-        this.embeddingSize = embeddingSize
-        this.gpu = new OperationManager(device);
-        this.device = device;
-        this.forwardPipline = this.gpu.getShaderPipline("./src/shaders/RF.wgsl")
-        this.backwardPipline = this.gpu.getShaderPipline("./src/shaders/RB.wgsl")
-    }
-    async forward(input, en) {
-        const encoder =  en ? en : this.device.createCommandEncoder()
-        const tokenCount = input.mainBuffer.size / (this.embeddingSize * 4)
-        if (!this.oldParam) {
-            this.oldParam = new Uint32Array([0, 0, 0]);
-            this.paramBuffer = await this.gpu.createUniFormBuffer(4 * 3);
-        }
-        if (!this.output) this.output = await this.gpu.ManageBuffer(input.mainBuffer.size);
-        if (this.output.mainBuffer.size !== input.mainBuffer.size || this.output.isDestroyed() || this.output.size == 0) {
-            this.output.destroy()
-            this.output = await this.gpu.ManageBuffer(input.mainBuffer.size);
-        }
-        if (this.oldParam[1] !== tokenCount) {
-            this.oldParam = new Uint32Array([this.embeddingSize, tokenCount, 0])
-            await this.gpu.WriteBuffer(this.paramBuffer, this.oldParam)
-        }
-        const bind = await this.device.createBindGroup({
-            layout: await this.forwardPipline.getBindGroupLayout(0),
-            entries: [
-                {
-                    binding: 0,
-                    resource: {
-                        buffer: input.mainBuffer
-                    }
-                },
-                {
-                    binding: 1,
-                    resource: {
-                        buffer: this.paramBuffer
-                    }
-                }, {
-                    binding: 2,
-                    resource: {
-                        buffer: this.output.mainBuffer
-                    }
-                }
-            ]
-        })
-        await this.gpu.RunPipline(encoder, this.forwardPipline, bind, Math.ceil((tokenCount * this.embeddingSize) / 256))
-        if(!en) this.device.queue.submit([encoder.finish()])
-        this.input = input
-        return this.output
-    }
-    async backward(inputGradient,en) {
-        const encoder = en ? en :  this.device.createCommandEncoder()
-        const tokenCount = inputGradient.size / (this.embeddingSize * 4)
-        if (!this.oldParam) {
-            this.oldParam = new Uint32Array([0, 0, 0]);
-            this.paramBuffer = await this.gpu.createUniFormBuffer(4 * 3);
-        }
-        if(!this.gradOut) this.gradOut = await this.gpu.ManageBuffer(inputGradient.size);
-        if(this.gradOut.size !== inputGradient.size || this.gradOut.isDestroyed() || this.gradOut.size == 0){
-            this.gradOut.destroy();
-            this.gradOut = await this.gpu.ManageBuffer(inputGradient.size)
-        }
-        if (this.oldParam[1] !== tokenCount) {
-            this.oldParam = new Uint32Array([this.embeddingSize, tokenCount, 0])
-            await this.gpu.WriteBuffer(this.paramBuffer, this.oldParam)
-        }
-        const bind = await this.device.createBindGroup({
-            layout: await this.backwardPipline.getBindGroupLayout(0),
-            entries: [
-                {
-                    binding: 0,
-                    resource: {
-                        buffer: this.input.mainBuffer
-                    }
-                },
-                {
-                    binding: 1,
-                    resource: {
-                        buffer: this.paramBuffer
-                    }
-                }, {
-                    binding: 2,
-                    resource: {
-                        buffer: this.gradOut.mainBuffer
-                    }
-                }, {
-                    binding: 3,
-                    resource: {
-                        buffer: inputGradient.mainBuffer
-                    }
-                }
-            ]
-        })
-        await this.gpu.RunPipline(encoder, this.backwardPipline, bind, Math.ceil((tokenCount * this.embeddingSize) / 256))
-        if(!en) await this.device.queue.submit([encoder.finish()])
-        return this.gradOut
-    }
-}

package/src/SelfAttention/index.d.ts

@@ -1,28 +0,0 @@
-export class SelfAttention {
-    constructor(device: GPUAdapter, embeddingSize: number, configs: {
-        dirname: string
-    })
-    async forward(vec: {
-        read: function(): Promise<Float32Array>,
-        mainBuffer: GPUBuffer,
-        isDestroyed: function(): Boolean,
-        destroy: function(): void
-    }): Promise<{
-        read: function(): Promise<Float32Array>,
-        mainBuffer: GPUBuffer,
-        isDestroyed: function(): Boolean,
-        destroy: function(): void
-    }>
-    async backward(dO: {
-        read: function(): Promise<Float32Array>,
-        mainBuffer: GPUBuffer,
-        isDestroyed: function(): Boolean,
-        destroy: function(): void
-    }, lr: number): Promise<{
-        read: function(): Promise<Float32Array>,
-        mainBuffer: GPUBuffer,
-        isDestroyed: function(): Boolean,
-        destroy: function(): void
-    }>
-    async Save(): Promise<void>
-} 

package/src/SelfAttention/index.js

@@ -1,138 +0,0 @@
-import { Linear } from "../../index.js";
-import { join } from 'path'
-import { OperationManager } from "../OperationManager.js";
-
-export class SelfAttention {
-    query;
-    value;
-    key;
-    q;
-    k;
-    v;
-    embeddingSize;
-    params;
-    attentionscoresShader;
-    softmaxForward;
-    gpu;
-    input;
-    constructor(device, embeddingSize, configs) {
-        this.embeddingSize = embeddingSize;
-        const dd = [1, 2, 3].map((e) => {
-            return {
-                save: {
-                    filename: [join(configs.dirname, `SELF-W-${e}.bin`), join(configs.dirname, `SELF-B-${e}.bin`)]
-                }
-            }
-        })
-        this.q = new Linear(device, embeddingSize, embeddingSize, dd[0])
-        this.k = new Linear(device, embeddingSize, embeddingSize, dd[1])
-        this.v = new Linear(device, embeddingSize, embeddingSize, dd[2])
-        this.gpu = new OperationManager(device)
-        this.attentionscoresShader = this.gpu.getShaderPipline("./src/shaders/attentionShader/attentionSocres.wgsl")
-        this.softmaxForward = this.gpu.getShaderPipline("./src/shaders/SF.wgsl")
-        this.softmaxBackward = this.gpu.getShaderPipline("./src/shaders/SoftmaxBackward.wgsl")
-        this.outputForward = this.gpu.getShaderPipline("./src/shaders/attentionShader/CompueFinalForwardOutput.wgsl")
-        this.dVShader = this.gpu.getShaderPipline("./src/shaders/attentionShader/ComputeDV.wgsl")
-        this.dAshader = this.gpu.getShaderPipline("./src/shaders/attentionShader/ComputeDA.wgsl")
-        this.dQshader = this.gpu.getShaderPipline("./src/shaders/attentionShader/ComputedQ.wgsl")
-        this.dKShader = this.gpu.getShaderPipline("./src/shaders/attentionShader/ComputeK.wgsl")
-        this.sumShader = this.gpu.getShaderPipline("./src/shaders/attentionShader/sumGradient.wgsl")
-    }
-    async forward(vec, encoder2) {
-        this.input = vec.mainBuffer;
-        const tokenCount = vec.mainBuffer.size / (this.embeddingSize * 4)
-        if (!this.asParams) this.asParams = await this.gpu.createUniFormBuffer(4 * 3);
-        if (!this.softmaxParams) this.softmaxParams = await this.gpu.createUniFormBuffer(4 * 3);
-        const encoder = encoder2 ? encoder2 :  await this.gpu.CreateEncoder()
-        if (!this.scroes) {
-            this.scroes = await this.gpu.createBuffer(tokenCount * tokenCount * 4)
-        }
-        if (this.scroes.size !== tokenCount * tokenCount * 4) {
-            this.scroes.destroy()
-            this.scroes = await this.gpu.createBuffer(tokenCount * tokenCount * 4)
-        }
-        await this.gpu.WriteBuffer(this.asParams, new Uint32Array([tokenCount, this.embeddingSize, 0]))
-        await this.gpu.WriteBuffer(this.softmaxParams, new Uint32Array([tokenCount, tokenCount, 0]))
-        this.query = await this.q.forward(vec, encoder)
-        this.key = await this.k.forward(vec, encoder)
-        this.value = await this.v.forward(vec, encoder)
-        const bindAttention = await this.gpu.createBindGroup(this.attentionscoresShader, 0, [this.query.mainBuffer, this.key.mainBuffer, this.scroes, this.asParams])
-        await this.gpu.RunPipline(encoder, this.attentionscoresShader, bindAttention, Math.ceil(tokenCount / 16), Math.ceil(tokenCount / 16))
-        if (this.weights) this.weights.destroy()
-        this.weights = await this.gpu.createBuffer(tokenCount * tokenCount * 4)
-        const bindSF = await this.gpu.createBindGroup(this.softmaxForward, 0, [this.scroes, this.weights, this.softmaxParams]);
-        await this.gpu.RunPipline(encoder, this.softmaxForward, bindSF, tokenCount)
-        if (!this.finalOutput) {
-            this.finalOutput = await this.gpu.ManageBuffer(tokenCount * this.embeddingSize * 4)
-        }
-        if (this.finalOutput.mainBuffer.size !== tokenCount * this.embeddingSize * 4 || this.finalOutput.size == 0 || this.finalOutput.isDestroyed()) {
-            this.finalOutput = await this.gpu.ManageBuffer(tokenCount * this.embeddingSize * 4)
-        }
-        await this.finalOutput.count()
-        const outputBind = await this.gpu.createBindGroup(this.outputForward, 0, [this.weights, this.value.mainBuffer, this.finalOutput.mainBuffer, this.asParams]);
-        await this.gpu.RunPipline(encoder, this.outputForward, outputBind, Math.ceil((tokenCount * this.embeddingSize) / 256))
-        if(!encoder2) await this.gpu.submitQueue(encoder)
-        return this.finalOutput
-    }
-    async backward(dO, lr,encoder2) {
-        const tokenCount = dO.size / (this.embeddingSize * 4);
-        const encoder = encoder2 ? encoder2 :  await this.gpu.CreateEncoder();
-        if (!this.dV) this.dV = await this.gpu.createBuffer(dO.size);
-        if (!this.dA) this.dA = await this.gpu.createBuffer(tokenCount * tokenCount * 4);
-        if (!this.dS) this.dS = await this.gpu.createBuffer(tokenCount * tokenCount * 4);
-        if (!this.dQ) this.dQ = await this.gpu.createBuffer(tokenCount * this.embeddingSize * 4);
-        if (!this.dK) this.dK = await this.gpu.createBuffer(this.embeddingSize * tokenCount * 4)
-        if (this.dV.size !== dO.size || this.dV.size == 0) {
-            this.dV.destroy()
-            this.dV = await this.gpu.createBuffer(dO.size);
-        }
-        if (this.dA.size !== tokenCount * tokenCount * 4 || this.dA.size == 0) {
-            this.dA.destroy()
-            this.dA = await this.gpu.createBuffer(tokenCount * tokenCount * 4);
-        }
-        if (this.dS.size !== tokenCount * tokenCount * 4 || this.dS.size == 0) {
-            this.dS.destroy()
-            this.dS = await this.gpu.createBuffer(tokenCount * tokenCount * 4);
-        }
-        if (this.dQ.size !== tokenCount * this.embeddingSize * 4 || this.dQ.size == 0) {
-            this.dQ.destroy()
-            this.dQ = await this.gpu.createBuffer(tokenCount * this.embeddingSize * 4);
-        }
-        if (this.dK.size !== tokenCount * this.embeddingSize * 4 || this.dK.size == 0) {
-            this.dK.destroy()
-            this.dK = await this.gpu.createBuffer(tokenCount * this.embeddingSize * 4);
-        }
-        const DvBind = await this.gpu.createBindGroup(this.dVShader, 0, [this.weights, dO.mainBuffer, this.dV, this.asParams]);
-        await this.gpu.RunPipline(encoder, this.dVShader, DvBind, Math.ceil(tokenCount / 16), Math.ceil(this.embeddingSize / 16));
-        const DaBind = await this.gpu.createBindGroup(this.dAshader, 0, [dO.mainBuffer, this.value.mainBuffer, this.dA, this.asParams]);
-        await this.gpu.RunPipline(encoder, this.dAshader, DaBind, Math.ceil(tokenCount / 16), Math.ceil(tokenCount / 16))
-        const dSBind = await this.gpu.createBindGroup(this.softmaxBackward, 0, [this.weights, this.dA, this.dS, this.asParams])
-        await this.gpu.RunPipline(encoder, this.softmaxBackward, dSBind, Math.ceil((tokenCount * tokenCount) / 256))
-        const dQbind = await this.gpu.createBindGroup(this.dQshader, 0, [this.dS, this.key.mainBuffer, this.dQ, this.asParams]);
-        await this.gpu.RunPipline(encoder, this.dQshader, dQbind, Math.ceil(tokenCount / 16), Math.ceil(this.embeddingSize / 16))
-        const dKbind = await this.gpu.createBindGroup(this.dKShader, 0, [this.dS, this.query.mainBuffer, this.dK, this.asParams]);
-        await this.gpu.RunPipline(encoder, this.dKShader, dKbind, Math.ceil(tokenCount / 16), Math.ceil(this.embeddingSize / 16))
-        const dq = await this.q.backward({ mainBuffer: this.dQ, size: this.dQ.size}, lr, encoder)
-        const dk = await this.k.backward({ mainBuffer: this.dK, size: this.dK.size}, lr, encoder)
-        const dv = await this.v.backward({ mainBuffer: this.dV, size: this.dV.size}, lr, encoder)
-        if(!this.gradOut) this.gradOut = await this.gpu.ManageBuffer(tokenCount * this.embeddingSize * 4)
-        if(this.gradOut.size !== tokenCount * 4 * this.embeddingSize || this.gradOut.isDestroyed() || this.gradOut.size == 0){
-            this.gradOut.destroy()
-            this.gradOut = await this.gpu.ManageBuffer(tokenCount * 4 * this.embeddingSize)
-        }
-        await this.gradOut.count()
-        if(!dq || !dk || !dq || !this.gradOut){
-            throw new Error("Something is wrong!")
-        }
-        const outputBind = await this.gpu.createBindGroup(this.sumShader, 0, [dq.mainBuffer, dk.mainBuffer, dv.mainBuffer, this.gradOut.mainBuffer, this.asParams]);
-        await this.gpu.RunPipline(encoder, this.sumShader, outputBind, Math.ceil((tokenCount * this.embeddingSize / 256)));
-        if(!encoder2) await this.gpu.submitQueue(encoder)
-        return this.gradOut
-
-    }
-    async Save() {
-        await this.q.Save()
-        await this.k.Save()
-        await this.v.Save()
-    }
-}

package/src/Softmax/index.d.ts

@@ -1,26 +0,0 @@
-export class Softmax {
-    constructor(device: GPUAdapter, embeddingSize: number, vocabSize: number);
-    async forward(logit: {
-        read: function(): Promise<Float32Array>,
-        mainBuffer: GPUBuffer,
-        isDestroyed: function(): Boolean,
-        destroy: function(): void
-    }, isCpuReadBack): Promise<{
-        read: function(): Promise<Float32Array>,
-        mainBuffer: GPUBuffer,
-        isDestroyed: function(): Boolean,
-        destroy: function(): void
-    }>;
-    async backward(probs: {
-        read: function(): Promise<Float32Array>,
-        mainBuffer: GPUBuffer,
-        isDestroyed: function(): Boolean,
-        destroy: function(): void
-    }, target: number[]): Promise<{
-        read: function(): Promise<Float32Array>,
-        mainBuffer: GPUBuffer,
-        isDestroyed: function(): Boolean,
-        destroy: function(): void
-    }>
-    async crossEntropy(probs: GPUBuffer, target: number[]): Promise<{ loss: Function, acc: Function }>
-}

package/src/Softmax/index.js

@@ -1,195 +0,0 @@
-import { log } from 'console';
-import { OperationManager } from '../OperationManager.js'
-
-export class Softmax {
-    device;
-    vocabSize;
-    forwardPipline;
-    embeddingSize;
-    gpu;
-    paramsForward;
-    oldParam;
-    bp;
-    totalLoss;
-    totalAcc;
-    totalCalculate;
-    constructor(device, embeddingSize, vocabSize) {
-        this.vocabSize = vocabSize;
-        this.device = device;
-        this.embeddingSize = embeddingSize;
-        this.gpu = new OperationManager(device);
-        this.forwardPipline = this.gpu.getShaderPipline("./src/shaders/SF.wgsl")
-        this.backwardPipline = this.gpu.getShaderPipline("./src/shaders/SB.wgsl")
-        this.lossPipline = this.gpu.getShaderPipline("./src/shaders/CalculateLoss.wgsl")
-        this.accPipline = this.gpu.getShaderPipline("./src/shaders/CalculateAcc.wgsl")
-        this.battention = this.gpu.getShaderPipline("./src/shaders/SoftmaxBackward.wgsl")
-        this.reduceFn = this.gpu.getShaderPipline("./src/shaders/ReductionFunction.wgsl")
-        this.totalCalculate = this.gpu.getShaderPipline('./src/shaders/calculateTotal.wgsl')
-    }
-    async forward(logits, en) {
-        const encoder = en ? en : this.device.createCommandEncoder()
-        const tokenCount = logits.size / (this.vocabSize * 4)
-        if (!this.paramsForward) {
-            this.paramsForward = await this.gpu.createUniFormBuffer(4 * 3)
-        }
-        if (!this.oldParam) {
-            this.oldParam = new Uint32Array([this.vocabSize, 0, 0])
-        }
-
-        if (this.oldParam[1] !== tokenCount) { await this.gpu.WriteBuffer(this.paramsForward, new Uint32Array([this.vocabSize, tokenCount, 0])); this.oldParam = new Uint32Array([this.vocabSize, tokenCount, 0]); }
-        if (!this.output) this.output = await this.gpu.ManageBuffer(logits.size);
-        if (this.output.size !== 4 * tokenCount * this.vocabSize || this.output.isDestroyed()) {
-            this.output.destroy()
-            this.output = await this.gpu.ManageBuffer(logits.size)
-        }
-        const bind = await this.device.createBindGroup({
-            layout: await this.forwardPipline.getBindGroupLayout(0),
-            entries: [
-                {
-                    binding: 0,
-                    resource: {
-                        buffer: logits.mainBuffer
-                    }
-                },
-                {
-                    binding: 1,
-                    resource: {
-                        buffer: this.output.mainBuffer
-                    }
-                }, {
-                    binding: 2,
-                    resource: {
-                        buffer: this.paramsForward
-                    }
-                }
-            ]
-        })
-        await this.gpu.RunPipline(encoder, this.forwardPipline, bind, tokenCount)
-        if(!en) await this.device.queue.submit([encoder.finish()])
-
-        return this.output
-    }
-    async backward(probs, target,encoder2) {
-        const encoder = encoder2 ? encoder2 : this.device.createCommandEncoder()
-        const tokenCount = probs.size / (this.vocabSize * 4)
-        if (!this.paramsForward) {
-            this.paramsForward = await this.gpu.createUniFormBuffer(4 * 3)
-        }
-        if (!this.oldParam) {
-            this.oldParam = new Uint32Array([this.vocabSize, 0, 0])
-        }
-        if (this.oldParam[1] !== tokenCount) {
-            await this.gpu.WriteBuffer(this.paramsForward, new Uint32Array([this.vocabSize, tokenCount, 0]));
-            this.oldParam = new Uint32Array([this.vocabSize, tokenCount, 0]);
-        }
-        if (!this.targetBuffer) this.targetBuffer = await this.gpu.createBuffer(target.length * Float32Array.BYTES_PER_ELEMENT);
-        if (this.targetBuffer.size !== target.length * 4) {
-            this.targetBuffer.destroy()
-            this.targetBuffer = await this.gpu.createBuffer(target.length * Float32Array.BYTES_PER_ELEMENT)
-        }
-        if (!this.dLogits) this.dLogits = await this.gpu.ManageBuffer(probs.size);
-        if (this.dLogits.size !== probs.size) {
-            this.dLogits.destroy()
-            this.dLogits = await this.gpu.ManageBuffer(probs.size);
-        }
-        await this.gpu.WriteBuffer(this.targetBuffer, new Float32Array(target))
-        const bind = await this.device.createBindGroup({
-            layout: await this.backwardPipline.getBindGroupLayout(0),
-            entries: [
-                {
-                    binding: 0,
-                    resource: {
-                        buffer: probs.mainBuffer
-                    }
-                }, {
-                    binding: 2,
-                    resource: {
-                        buffer: this.targetBuffer
-                    }
-                }, {
-                    binding: 3,
-                    resource: {
-                        buffer: this.paramsForward
-                    }
-                },
-                {
-                    binding: 4,
-                    resource: {
-                        buffer: this.dLogits.mainBuffer
-                    }
-                }
-            ]
-        })
-        await this.gpu.RunPipline(encoder, this.backwardPipline, bind, Math.ceil((tokenCount * this.vocabSize) / 256))
-        if(!encoder2) await this.gpu.submitQueue(encoder)
-        return this.dLogits
-    }
-    async crossEntropy(probs, target, encoderMain) {
-        const encoder = encoderMain ? encoderMain : this.device.createCommandEncoder();
-        const tokenCount = probs.size / (this.vocabSize * Float32Array.BYTES_PER_ELEMENT);
-        if (!Number.isInteger(tokenCount) || tokenCount <= 0) {
-            throw new Error("Invalid probability buffer size or vocabulary size");
-        }
-        const targetBytes = target.length * Uint32Array.BYTES_PER_ELEMENT;
-
-        if (!this.targetBuffer || this.targetBuffer.size < targetBytes) {
-            this.targetBuffer =
-                await this.gpu.createBuffer(targetBytes);
-        }
-
-        if (!this.loss || this.loss.size < tokenCount * 4) {
-            this.loss = await this.gpu.createBuffer(tokenCount * 4);
-        }
-
-        if (!this.acc) {
-            this.acc = await this.gpu.createBuffer(4);
-        }
-
-        await this.gpu.WriteBuffer(
-            this.targetBuffer,
-            new Uint32Array(target)
-        );
-
-        const bindGroup1 = await this.gpu.createBindGroup(
-            this.lossPipline,
-            0,
-            [probs.mainBuffer, this.targetBuffer, this.loss, this.paramsForward]
-        );
-
-        const bindGroup2 = await this.gpu.createBindGroup(
-            this.accPipline,
-            0,
-            [this.paramsForward, probs.mainBuffer, this.acc, this.targetBuffer]
-        );
-
-        await this.gpu.RunPipline(
-            encoder,
-            this.lossPipline,
-            bindGroup1,
-            Math.ceil(tokenCount / 4)
-        );
-
-        await this.gpu.RunPipline(
-            encoder,
-            this.accPipline,
-            bindGroup2,
-            1
-        );
-
-        if (!encoderMain) {
-            await this.gpu.submitQueue(encoder);
-        }
-
-        return {
-            loss: async () =>
-                (await this.gpu.readBuffer(this.loss.size, this.loss)),
-
-            acc: async () =>
-                (await this.gpu.readBuffer(4, this.acc))[0],
-
-
-        };
-    }
-
-}
-

package/src/Tokenizer/index.d.ts

@@ -1,12 +0,0 @@
-export class Tokenizer{
-    vocabs : string[]
-    vocabSize: number
-    IdtoWord : Record<string, string>
-    WordtoID : Record<string , number>
-    encoder(text: string) : {
-        tokens: string[]
-        tokenIDs: number[]
-    }
-    decoder(tokens: number[] | number[][]) : string
-    trainTokenizer(text: string, mergeSize: number, filename: string) : void
-}

package/src/Tokenizer/index.js

@@ -1,164 +0,0 @@
-import fs, { existsSync, readFileSync } from 'fs'
-import { join, dirname, format } from 'path';
-import { fileURLToPath } from 'url';
-
-const __dirname = dirname(fileURLToPath(import.meta.url))
-export class Tokenizer {
-    vocabs = [];
-    vocabSize = 0;
-    IdtoWord = {};
-    WordtoID = {};
-    EOSToken = 2
-    constructor(filename = "") {
-        if (fs.existsSync(filename)) {
-            console.log("Warning: Tokenizer is loaded with a Custom Vocab File!")
-            console.log("")
-            this.vocabs = JSON.parse(readFileSync(filename, "utf-8"))
-        } else {
-            this.vocabs = JSON.parse(readFileSync(join(__dirname, "vocabs.json"), "utf-8"))
-        }
-        this.vocabSize = this.vocabs.length
-        this.IdtoWord = Object.fromEntries(this.vocabs.map((e, idx) => [idx, e]))
-        this.WordtoID = Object.fromEntries(this.vocabs.map((e, idx) => [e, idx]))
-    }
-    encoder(text = "") {
-        const AddSpaces = text.replaceAll(/<EOS>/gi, " <EOS> ").toLocaleLowerCase().replaceAll(/<eos>/g, " <EOS> ").replaceAll(/([,.;'{}!`~|":?])/g, " $1 ");
-        const splitedWords = AddSpaces.split(/\s+/).filter(e => e.length > 0)
-        const output = []
-        for (let a = 0; a < splitedWords.length; a++) {
-            const word = splitedWords[a]
-            if (typeof this.WordtoID[word] !== "undefined") {
-                output.push(word)
-            } else {
-                let max = word.length
-                output.push("$12")
-                while (true) {
-                    const regPatten = new RegExp(`.{0,${max}}`, "g")
-                    const wordsliced = word.match(regPatten).filter((e) => e.length > 0)
-                    const matched = wordsliced.flatMap(e => {
-                        if (this.WordtoID[e] !== undefined) {
-                            return true
-                        }
-                        return false
-
-                    }).reduce((prev, curr) => prev && curr, true)
-                    if (wordsliced.length == 0) output.push("<unk>")
-                    if (matched) {
-                        wordsliced.map((e) => output.push(e))
-                        output.push("$130")
-                        break
-                    } else {
-                        max -= 1
-                        // break
-                    }
-
-                }
-            }
-        }
-        return { tokens: output, tokenIDs: output.map((e) => this.WordtoID[e]) }
-    }
-    decoder(tokens = [1, 2, 3]) {
-        const output = []
-        let isSpace = false
-        // let index = 0
-        for (let token of tokens) {
-            const formated = this.IdtoWord[String(token)]
-            if (formated == "$12") {
-                output[output.length] = ""
-                isSpace = true
-            } else if (formated == "$130") {
-                isSpace = false
-            } else {
-                if (isSpace) {
-                    output[output.length - 1] += formated
-                } else {
-                    output.push(formated)
-                }
-            }
-        }
-        return output.filter(e => e.trim().length > 0).join(" ").replaceAll(/\s+([,.;'{}`!~\"?])/g, "$1")
-    }
-    trainTokenizer(text = "ejrnienieijr", mergeLength = 5, vocabFileName) {
-        console.log("Training The Tokenizer....")
-        const arrText = text.toLowerCase().replace(/\r\n|\n|\r/g, " <EOS> ").toLocaleLowerCase().replaceAll(/<eos>/g, " <EOS> ").replace(/([,.;'{}!`~|":><?])/g, " $1 ").split(/\s+/).map(e => e.trim()).filter(e => e.length > 0)
-        const reg = new RegExp(`.{0,${mergeLength}}`, "g")
-        const prevVocab = existsSync(vocabFileName) ? JSON.parse(readFileSync(vocabFileName, "utf-8")) : [];
-        const specal = ["<unk>",
-            "<PAD>",
-            "<EOS>",
-            "$12",
-            "$130",
-            "😀",
-            "😃",
-            "😄",
-            "😁",
-            "😆",
-            "😅",
-            "😂",
-            "🤣",
-            "😊",
-            "😎",
-            "❤️",
-            "👍",
-            "👎",
-            "🔥",
-            "💯",
-            "🎉",
-            "🚀",
-            "⭐",
-            "😭",
-            "😍",
-            "🥰",
-            "😡",
-            "🤔",
-            "😢",
-            "😮",
-            "👋",
-            "🙏",
-            "💀",
-            "🤖",
-            "👀",
-            "💕",
-            "💔",
-            "😂😂",
-            "😂😭",
-            "🔥🔥",
-            "❤️❤️",
-            "👍👍",
-            "<",
-            ">",
-            ".",
-            "{",
-            "}",
-            "?",
-            "!",
-            "@",
-            "$",
-            ":",
-            "/",
-            "+",
-            "-",
-            "=",
-            "%",
-            "*",
-            "@"
-        ]
-        const uniArr = [...new Set([...arrText, ...specal, ...prevVocab])];
-        const output = []
-        for (let a = 0; a < uniArr.length; a++) {
-            let text = uniArr[a].match(reg).filter((e) => e.length > 0);
-            output.push(...text)
-        }
-        let finalOutput = [...new Set([...output])]
-        fs.writeFileSync(vocabFileName, JSON.stringify(finalOutput, null, 2), "utf-8");
-        console.log("_________________________________")
-        console.log(`New VocabSize : ${finalOutput.length - prevVocab.length}`);
-        console.log("_________________________________")
-        console.log("Operation Completed!")
-        this.vocabs = finalOutput;
-        this.vocabSize = finalOutput.length;
-        this.IdtoWord = Object.fromEntries(this.vocabs.map((e, idx) => [idx, e]))
-        this.WordtoID = Object.fromEntries(this.vocabs.map((e, idx) => [e, idx]))
-
-    }
-}