@mineproness/tensorloop 1.0.0

package/index.js

@@ -0,0 +1,10 @@
+import { Tokenizer as tokenizer } from "./src/Tokenizer/index.js";
+import { Embedding as embeddding2 } from "./src/Embedding/index.js";
+import { Linear as li2 } from "./src/Linear/index.js";
+import { Softmax as softmax2 } from "./src/Softmax/index.js";
+import { crossEntropy } from "./src/crossEntropy/index.js";
+export const Tokenizer = tokenizer
+export const Embedding = embeddding2
+export const Linear = li2
+export const Softmax = softmax2
+export const CrossEntropy = crossEntropy

package/package.json

@@ -0,0 +1,22 @@
+{
+  "name": "@mineproness/tensorloop",
+  "version": "1.0.0",
+  "description": "The most powerfull Machine Learing Libeary for AI.",
+  "keywords": [
+    "ai",
+    "llm",
+    "Large",
+    "Language",
+    "model",
+    "Small",
+    "Language",
+    "Model"
+  ],
+  "license": "MIT",
+  "author": "mineproness",
+  "type": "module",
+  "main": "index.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  }
+}

package/src/Embedding/index.d.ts

@@ -0,0 +1,10 @@
+export class Embedding{
+    vectors: Float32Array[]
+    constructor(embeddingSize: number, vocabSize: number, configs: {
+        save: {
+            filename: string
+        }
+    })
+    forward(token: number | number[]) : number[] | number[][]
+    backward(token: number , inputGradient: number[], learingRate: number) : void
+}

package/src/Embedding/index.js

@@ -0,0 +1,48 @@
+import fs, { existsSync } from 'fs'
+import { GetVectors, SaveVectors } from '../GetConfigs.js'
+// import { config } from 'process'
+
+function GenerateVectors(embeddingSize, vocabSize) {
+    const vector = []
+    for (let token = 0; token < vocabSize; token++) {
+        vector[token] = new Float32Array(embeddingSize)
+        for (let a = 0; a < embeddingSize; a++) {
+            vector[token][a] = (Math.random() * 2 - 1) * 0.2
+        }
+    }
+    return vector
+}
+
+export class Embedding {
+    vectors = []
+    configs;
+    constructor(embeddingSize, vocabSize, configs) {
+        this.configs = configs
+        if (configs && configs.save) {
+            if (existsSync(configs.save.filename)) {
+                 this.vectors = GetVectors(configs.save.filename, embeddingSize)
+            } else {
+                this.vectors = GenerateVectors(embeddingSize, vocabSize)
+                SaveVectors(configs.save.filename, this.vectors)
+            }
+        } else {
+            this.vectors = GenerateVectors(embeddingSize, vocabSize)
+
+        }
+    }
+    forward(token) {
+       if(Array.isArray(token)){
+        return token.map((e)=> this.vectors[e])
+       }else{
+        return this.vectors[token]
+       }
+    }
+    backward(token, inputGradient, learingRate){
+        for(let a = 0; a < this.vectors[token].length; a++){
+           this.vectors[token][a] -= learingRate * inputGradient[a]
+        }
+    }
+    Save(){
+        SaveVectors(this.configs.save.filename , this.vectors)
+    }
+}

package/src/GetConfigs.js

@@ -0,0 +1,35 @@
+import fs from "fs";
+import { join } from "path";
+
+export function SaveVectors (filename, vectors){
+    const sizeOfArray = vectors.length;
+    const sizeOfEmbedding = vectors[0].length;
+    const floatArray = vectors.map((e)=> Buffer.from(e.buffer))
+    const combinedBuffer = Buffer.concat(floatArray)
+    fs.writeFileSync(filename , combinedBuffer)
+    
+}
+
+export function GetVectors(filename, embeddingSize){
+    const fileBuffer = fs.readFileSync(filename)
+    const ELEMENTS_PER_ARRAY= embeddingSize;
+    const BYTES_PER_ELEMENT= Float32Array.BYTES_PER_ELEMENT
+    const BYTES_PER_ARRAY = ELEMENTS_PER_ARRAY * BYTES_PER_ELEMENT
+    const totalArraySaved = fileBuffer.byteLength / BYTES_PER_ARRAY;
+    const totalArray = []
+    for(let i = 0; i < totalArraySaved; i++){
+        const currentOffset = fileBuffer.byteOffset + (i * BYTES_PER_ARRAY)
+        const f32 = new Float32Array(fileBuffer.buffer, currentOffset, ELEMENTS_PER_ARRAY)
+        totalArray.push(f32)
+    }
+    return totalArray
+}
+
+export function GetVector(filename){
+    const buffer = fs.readFileSync(filename)
+    return new Float32Array(buffer.buffer, buffer.byteOffset, buffer.byteLength / Float32Array.BYTES_PER_ELEMENT)
+}
+
+export function Save(filename, vector){
+    return fs.writeFileSync(filename, Buffer.from(vector.buffer))
+}

package/src/Linear/index.d.ts

@@ -0,0 +1,11 @@
+export class Linear{
+    Weights : Float32Array[]
+    Bias: Float32Array
+    constructor(embeddingSize : number, vocabSize : number, configs: {
+        save: {
+            filename: string[]
+        }
+    })
+    forward(input : number[]) : number[]
+    backward(outGradient : number[] , learingRate: number) : number[]
+}

package/src/Linear/index.js

@@ -0,0 +1,69 @@
+import { GetVector, GetVectors, Save, SaveVectors } from '../GetConfigs.js'
+import { existsSync } from 'fs'
+function GenerateWeightsBias(embeddingSize, vocabSize) {
+    const vectors = []
+    const Bias = new Float32Array(vocabSize)
+    for (let a = 0; a < vocabSize; a++) {
+        vectors[a] = new Float32Array(embeddingSize)
+        for (let t = 0; t < embeddingSize; t++) {
+            vectors[a][t] = (Math.random() * 2 - 1) * 0.02
+        }
+        Bias[a] = 0
+    }
+    return { vectors, Bias }
+}
+
+export class Linear {
+    Weights = []
+    Bias = []
+    configs;
+    input = [];
+    constructor(embeddingSize, vocabSize, configs) {
+        if (configs && configs.save) {
+            this.configs = configs
+            if (existsSync(configs.save.filename[0]) && existsSync(configs.save.filename[1])) {
+                const Vectors = GetVectors(configs.save.filename[0], embeddingSize)
+                const Bias = GetVector(configs.save.filename[1])
+                this.Weights = Vectors;
+                this.Bias = Bias
+            } else {
+                const { vectors, Bias } = GenerateWeightsBias(embeddingSize, vocabSize)
+                this.Weights = vectors;
+                this.Bias = Bias;
+                SaveVectors(configs.save.filename[0], this.Weights)
+                Save(configs.save.filename[1], this.Bias)
+            }
+        } else {
+            const { vectors, Bias } = GenerateWeightsBias(embeddingSize, vocabSize)
+            this.Weights = vectors;
+            this.Bias = Bias;
+        }
+    }
+    Save() {
+        SaveVectors(this.configs.save.filename[0], this.Weights)
+        Save(this.configs.save.filename[1], this.Bias)
+    }
+    forward(input) {
+        this.input = input
+        const output = []
+        for (let a = 0; a < this.Weights.length; a++) {
+            let sum = this.Bias[a]
+            for (let t = 0; t < input.length; t++) {
+                sum += input[t] * this.Weights[a][t]
+            }
+            output.push(sum)
+        }
+        return output
+    }
+    backward(outputGradient, learningRate) {
+        const inputGradient = new Float32Array(this.input.length)
+        for (let a = 0; a < this.Weights.length; a++) {
+            for (let at = 0; at < this.input.length; at++) {
+                inputGradient[at] += outputGradient[a] * this.Weights[a][at]
+                this.Weights[a][at] -= learningRate * this.input[at] * outputGradient[a]
+            }
+            this.Bias[a] -= learningRate * outputGradient[a]
+        }
+        return inputGradient
+    }
+}

package/src/ReLU/index.js

@@ -0,0 +1,14 @@
+export class ReLU{
+    input;
+    forward(XLi){
+        this.input = XLi
+        return XLi.map((e)=> e > 0 ? e : 0)
+    }
+    backward(outputGradient){
+        const inputGradient = new Float32Array(this.input.length)
+        for(let i = 0; i < outputGradient.length; i++){
+            inputGradient[i] = this.input[i] > 0 ? outputGradient[i] : 0
+        }
+        return inputGradient
+    }
+}

package/src/Softmax/index.d.ts

@@ -0,0 +1,4 @@
+export class Softmax{
+    forward(xLi : number[]) : number[]
+    backward(probalitils: number[] , target : number) : number[]
+}

package/src/Softmax/index.js

@@ -0,0 +1,13 @@
+export class Softmax{
+    forward(xLi){
+        const max = Math.max(...xLi)
+        const exp = xLi.map((e)=> Math.exp(e - max))
+        const sum = exp.reduce((a,b)=> a + b , 0)
+        return exp.map((e)=> e / sum)
+    }
+    backward(probs , target){
+        const pro = [...probs]
+        pro[target] -= 1
+        return pro
+    }
+}

package/src/Tokenizer/index.d.ts

@@ -0,0 +1,11 @@
+export class Tokenizer{
+    vocabs : string[]
+    vocabSize: number
+    IdtoWord : Record<string, string>
+    WordtoID : Record<string , number>
+    encoder(text: string) : {
+        tokens: string[]
+        tokenIDs: number[]
+    }
+    decoder(tokens: number[] | number[][]) : string
+}

package/src/Tokenizer/index.js

@@ -0,0 +1,92 @@
+import { readFileSync } from 'fs'
+import { join, dirname, format } from 'path';
+import { fileURLToPath } from 'url';
+
+const __dirname = dirname(fileURLToPath(import.meta.url))
+export class Tokenizer {
+    vocabs = [];
+    vocabSize = 0;
+    IdtoWord = {};
+    WordtoID = {};
+    constructor() {
+        this.vocabs = JSON.parse(readFileSync(join(__dirname, "vocabs.json"), "utf-8"))
+        this.vocabSize = this.vocabs.length
+        this.IdtoWord = Object.fromEntries(this.vocabs.map((e, idx) => [idx, e]))
+        this.WordtoID = Object.fromEntries(this.vocabs.map((e, idx) => [e, idx]))
+    }
+    encoder(text = "") {
+        const AddSpaces = text.toLocaleLowerCase().replaceAll(/([,.;'{}`~\|":><?])/g, " $1 ")
+        const splitedWords = AddSpaces.split(/\s+/).filter(e => e.length > 0)
+        const output = []
+        for (let a = 0; a < splitedWords.length; a++) {
+            const word = splitedWords[a]
+            if (this.WordtoID[word]) {
+                output.push(word)
+            } else {
+                let max = word.length
+                output.push("$12")
+                while (true) {
+                    const regPatten = new RegExp(`.{0,${max}}`, "g")
+                    const wordsliced = word.match(regPatten).filter((e) => e.length > 0)
+                    const matched = wordsliced.flatMap(e => {
+                        if (this.WordtoID[e]) {
+                            return true
+                        }
+                        return false
+                    }).reduce((prev, curr) => prev && curr)
+                    if (matched) {
+                        wordsliced.map((e) => output.push(e))
+                        output.push("$130")
+                        break
+                    } else {
+                        max -= 1
+                        // break
+                    }
+
+                }
+            }
+        }
+        return { tokens: output, tokenIDs: output.map((e) => this.WordtoID[e]) }
+    }
+    decoder(tokens = [1, 2, 3]) {
+        const output = []
+        let isSpace = false
+        // let index = 0
+        for (let token of tokens) {
+            const formated = this.IdtoWord[String(token)]
+            if (formated == "$12") {
+                output[output.length] = ""
+                isSpace = true
+            } else if (formated == "$130") {
+                isSpace = false
+            } else {
+                if (isSpace) {
+                    output[output.length - 1] += formated
+                } else {
+                    output.push(formated)
+                }
+            }
+        }
+        return Validate(output).join(" ").replaceAll(/\s+([,.;'{}`~\|":><?])/g, "$1")
+    }
+}
+
+
+function Validate(output = ["hello", "guys"]) {
+    const bestOutput = []
+    output.map((e, idx) => {
+        if (idx == 0) {
+            bestOutput[idx] = e[0].toLocaleUpperCase() + e.slice(1, e.length)
+        } else {
+            if (bestOutput[idx - 1] == ":" || bestOutput[idx - 1] == ".") {
+
+                bestOutput[idx] = e[0].toLocaleUpperCase() + e.slice(1, e.length)
+            } else {
+                bestOutput[idx] = e
+
+            }
+        }
+
+    })
+    return bestOutput
+}