@mineproness/tensorloop 1.2.4 → 1.2.6

package/index.js

@@ -5,10 +5,14 @@ import { Softmax as softmax2 } from "./src/Softmax/index.js";
 import { crossEntropy } from "./src/crossEntropy/index.js";
 import {tools} from './src/tools/index.js'
 import {ReLU} from './src/ReLU/index.js'
+import {PositionEmbedding as PE} from './src/PositionEmbedding/index.js'
+import {SelfAttention as ST} from './src/SelfAttention/index.js'
 export const Tokenizer = tokenizer
 export const Embedding = embeddding2
 export const Linear = li2
 export const Softmax = softmax2
 export const CrossEntropy = crossEntropy
 export const Tools = tools
-export const ReLu = ReLU
+export const ReLu = ReLU
+export const PositionEmbedding = PE
+export const SelfAttention = ST

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mineproness/tensorloop",
-  "version": "1.2.4",
+  "version": "1.2.6",
   "description": "The most powerfull Machine Learning Library for AI.",
   "keywords": [
     "ai",

package/readme.md

@@ -10,6 +10,8 @@ A Library that Supply AI Tools and Large Language model Development tools like E
 - CrossEntroy for Loss Calculation.
 - Relu for Stable Layer.
 - Tokenizer for convert into tokens.
+- Self attention 
+- Position Embedding
 
 ## Configs
 

package/src/Embedding/index.js

@@ -1,5 +1,5 @@
 import fs, { existsSync } from 'fs'
-import { GetEmbeddingVectors, SaveEmbeddingVectors, SaveVectors } from '../GetConfigs.js'
+import { SaveVectors, LoadVectors } from '../GetConfigs.js'
 // import { config } from 'process'
 
 function GenerateVectors(embeddingSize, vocabSize) {
@@ -22,7 +22,7 @@ export class Embedding {
         this.embeddingSize = embeddingSize
         if (configs && configs.save) {
             if (existsSync(configs.save.filename)) {
-                this.vectors = []
+                this.vectors = LoadVectors(configs.save.filename, embeddingSize)
             } else {
                 this.vectors = GenerateVectors(embeddingSize, vocabSize)
                 this.Save()
@@ -35,15 +35,9 @@ export class Embedding {
     forward(token) {
         if (Array.isArray(token)) {
             return token.map((e) => {
-                const id = this.vectors[e]
-                if (id) return id
-                this.vectors[e] = GetEmbeddingVectors(e, this.embeddingSize, this.configs.save.filename)
                 return this.vectors[e]
             })
         } else {
-            if (!this.vectors[token]) {
-                this.vectors[token] = GetEmbeddingVectors(token, this.embeddingSize, this.configs.save.filename)
-            }
             return this.vectors[token]
         }
     }
@@ -53,6 +47,6 @@ export class Embedding {
         }
     }
     Save() {
-        SaveEmbeddingVectors(this.vectors, this.configs.save.filename)
+        SaveVectors(this.vectors, this.configs.save.filename)
     }
 }

package/src/GetConfigs.js

@@ -1,62 +1,35 @@
-import fs from "fs";
-import { join } from "path";
-
-export function SaveVectors(filename, vectors) {
-    const fd = fs.openSync(filename, "w");
-
-    try {
-        for (const vector of vectors) {
-            fs.writeSync(fd, Buffer.from(vector.buffer, vector.byteOffset, vector.byteLength));
-        }
-    } finally {
-        fs.closeSync(fd);
-    }
-}
-
-export function GetWeights(filename, embeddingSize) {
-    const fileBuffer = fs.readFileSync(filename)
-    const ELEMENTS_PER_ARRAY = embeddingSize;
-    const BYTES_PER_ELEMENT = Float32Array.BYTES_PER_ELEMENT
-    const BYTES_PER_ARRAY = ELEMENTS_PER_ARRAY * BYTES_PER_ELEMENT
-    const totalArraySaved = fileBuffer.byteLength / BYTES_PER_ARRAY;
-    const totalArray = []
-    for (let i = 0; i < totalArraySaved; i++) {
-        const currentOffset = fileBuffer.byteOffset + (i * BYTES_PER_ARRAY)
-        const f32 = new Float32Array(fileBuffer.buffer, currentOffset, ELEMENTS_PER_ARRAY)
-        totalArray.push(f32)
-    }
-    return totalArray
+import fs from 'fs'
+import { join } from 'path'
+
+export function SaveVectors(vectors, path){
+    const rows = vectors.length;
+    const cols = vectors[0].length;
+    const flatArray = new Float32Array(rows * cols)
+    vectors.forEach((e,i)=>{
+        flatArray.set(e, i * cols)
+    })
+    const bufferArray = Buffer.from(flatArray.buffer, flatArray.byteOffset, flatArray.byteLength)
+    fs.writeFileSync(path, bufferArray)
 }
 
-export function GetVector(filename) {
-    const buffer = fs.readFileSync(filename)
-    return new Float32Array(buffer.buffer, buffer.byteOffset, buffer.byteLength / Float32Array.BYTES_PER_ELEMENT)
+export function LoadVectors(path, cols){
+   const buffer = fs.readFileSync(path)
+   const flatArray = new Float32Array(buffer.buffer, buffer.byteOffset, buffer.byteLength / 4 )
+   const vectors = []
+   const rows = flatArray.length / cols;
+   for(let a = 0; a < rows; a++){
+       vectors.push(flatArray.slice(a * cols, (a+1) * cols))
+   }
+   return vectors
 }
 
-export function Save(filename, vector) {
-    return fs.writeFileSync(filename, Buffer.from(vector.buffer))
+export function SaveBias(vectors, path){
+    const bufferArray = Buffer.from(vectors.buffer, vectors.byteOffset, vectors.byteLength)
+    fs.writeFileSync(path, bufferArray)
 }
 
-export function GetEmbeddingVectors(token, embeddingSize, filename) {
-    const fd = fs.openSync(filename, "r")
-    const PER_BTYES_VECTOR = embeddingSize * Float32Array.BYTES_PER_ELEMENT
-    const buffer = Buffer.allocUnsafe(PER_BTYES_VECTOR)
-    fs.readSync(fd, buffer, 0, PER_BTYES_VECTOR, token * PER_BTYES_VECTOR)
-    fs.closeSync(fd)
-    return new Float32Array(buffer.buffer, buffer.byteOffset, embeddingSize)
-}
-
-export function SaveEmbeddingVectors(vectors = [new Float32Array([1, 2, 3])], filename) {
-    const fd = fs.openSync(filename, "w")
-    try {
-        for (let a = 0; a < vectors.length; a++) {
-            const vector = vectors[a]
-            if (vector == undefined) continue;
-            const offset = a * vector.length * Float32Array.BYTES_PER_ELEMENT;
-            fs.writeSync(fd, Buffer.from(vector.buffer, vector.byteOffset, vector.byteLength) , 0 , vector.byteLength, offset)
-
-        }
-    }finally{
-        fs.closeSync(fd)
-    }
+export function LoadBias(path){
+   const buffer = fs.readFileSync(path)
+   const flatArray = new Float32Array(buffer.buffer, buffer.byteOffset, buffer.byteLength / 4 )
+   return vectors
 }

package/src/Linear/index.d.ts

@@ -7,6 +7,6 @@ export class Linear{
         }
     })
     forward(input : number[]) : Float32Array[]
-    backward(outGradient : number[] , learingRate: number) : Float32Array[]
+    backward(outGradient : number[] , learingRate: number, idx: number) : Float32Array[]
     Save() : void
 }

package/src/Linear/index.js

@@ -1,4 +1,4 @@
-import { GetWeights, Save,GetVector ,  SaveVectors } from '../GetConfigs.js'
+import { LoadBias,LoadVectors,SaveBias,SaveVectors } from '../GetConfigs.js'
 import { existsSync } from 'fs'
 function GenerateWeightsBias(embeddingSize, vocabSize) {
     const vectors = []
@@ -22,8 +22,8 @@ export class Linear {
         if (configs && configs.save) {
             this.configs = configs
             if (existsSync(configs.save.filename[0]) && existsSync(configs.save.filename[1])) {
-                const Vectors = GetWeights(configs.save.filename[0], embeddingSize)
-                const Bias = GetVector(configs.save.filename[1])
+                const Vectors = LoadVectors(configs.save.filename[0], embeddingSize)
+                const Bias = LoadBias(configs.save.filename[1])
                 this.Weights = Vectors;
                 this.Bias = Bias
             } else {
@@ -31,7 +31,7 @@ export class Linear {
                 this.Weights = vectors;
                 this.Bias = Bias;
                 SaveVectors(configs.save.filename[0], this.Weights)
-                Save(configs.save.filename[1], this.Bias)
+                SaveBias(configs.save.filename[1], this.Bias)
             }
         } else {
             const { vectors, Bias } = GenerateWeightsBias(embeddingSize, vocabSize)
@@ -41,10 +41,10 @@ export class Linear {
     }
     Save() {
         SaveVectors(this.configs.save.filename[0], this.Weights)
-        Save(this.configs.save.filename[1], this.Bias)
+        SaveBias(this.configs.save.filename[1], this.Bias)
     }
     forward(input) {
-        this.input = input
+        this.input.push(input)
         const output = new Float32Array(this.Weights.length)
         for (let a = 0; a < this.Weights.length; a++) {
             let sum = this.Bias[a]
@@ -55,10 +55,10 @@ export class Linear {
         }
         return output
     }
-    backward(outputGradient, learningRate) {
-        const input = this.input;
+    backward(outputGradient, learningRate, idx) {
+        const input = this.input[idx];
         const weights = this.Weights
-        const inputGradient = new Float32Array(this.input.length)
+        const inputGradient = new Float32Array(input.length)
         for (let a = 0; a < weights.length; a++) {
             const gran = outputGradient[a]
             const w = weights[a]

package/src/PositionEmbedding/index.d.ts

@@ -0,0 +1,12 @@
+export class PositionEmbedding{
+    vectors: Float32Array[]
+    constructor(embeddingSize: number, contextSize: number, configs: {
+        save: {
+            filename: string
+        }
+    })
+    forward(position: number | number[]) : Float32Array[] | Float32Array[][]
+    backward(pos: number , inputGradient: number[] | Float32Array[], learingRate: number) : void
+    Save() : void;
+    addPos(PSEM, vectors : Float32Array[][]) : Float32Array 
+}

package/src/PositionEmbedding/index.js

@@ -0,0 +1,67 @@
+import fs, { existsSync } from 'fs'
+import { LoadVectors, SaveVectors } from '../GetConfigs.js'
+// import { config } from 'process'
+
+function GenerateVectors(embeddingSize, vocabSize) {
+    const vector = []
+    for (let token = 0; token < vocabSize; token++) {
+        vector[token] = new Float32Array(embeddingSize)
+        for (let a = 0; a < embeddingSize; a++) {
+            vector[token][a] = (Math.random() * 2 - 1) * 0.2
+        }
+    }
+    return vector
+}
+
+export class PositionEmbedding {
+    vectors = []
+    configs;
+    embeddingSize = 0
+    constructor(embeddingSize, vocabSize, configs) {
+        this.configs = configs
+        this.embeddingSize = embeddingSize
+        if (configs && configs.save) {
+            if (existsSync(configs.save.filename)) {
+                this.vectors = LoadVectors(onfigs.save.filename, embeddingSize)
+            } else {
+                this.vectors = GenerateVectors(embeddingSize, vocabSize)
+                this.Save()
+            }
+        } else {
+            this.vectors = GenerateVectors(embeddingSize, vocabSize)
+
+        }
+    }
+    forward(token) {
+        if (Array.isArray(token)) {
+            return token.map((e) => {
+                return this.vectors[e]
+            })
+        } else {
+
+            return this.vectors[token]
+        }
+    }
+    backward(token, inputGradient, learingRate) {
+        for (let a = 0; a < this.vectors[token].length; a++) {
+            this.vectors[token][a] -= learingRate * inputGradient[a]
+        }
+    }
+    Save() {
+        SaveVectors(this.vectors, this.configs.save.filename)
+    }
+    addPos(xEM) {
+        const out = []
+
+        for (let pos = 0; pos < xEM.length; pos++) {
+            const tokenVector = xEM[pos]
+            const posVector = this.forward(pos)
+            const x = new Float32Array(xEM[0].length)
+            for (let a = 0; a < tokenVector.length; a++) {
+                x[a] = tokenVector[a] + posVector[a]
+            }
+            out.push(x)
+        }
+        return out
+    }
+}

package/src/SelfAttention/index.d.ts

@@ -0,0 +1,16 @@
+export class SelfAttention{
+    q : Float32Array[]
+    v : Float32Array[]
+    k : Float32Array[]
+    input;
+    constructor(embeddingSize : number, configs: {
+        save: {
+            dirname: string
+        }
+    })
+    weights : Float32Array[]
+    score : number
+    forward(vectors: Float32Array[][])
+    Save() : void
+    backward(dOutput, learningRate)
+}

package/src/SelfAttention/index.js

@@ -0,0 +1,118 @@
+import { Softmax } from '../../index.js';
+import { Linear } from '../Linear/index.js'
+
+export class SelfAttention {
+    query;
+    key;
+    value;
+    v;
+    k;
+    q;
+    input;
+    configs;
+    score;
+    softmax;
+    weights;
+    constructor(embeddingSize, configs) {
+        this.configs = configs
+        this.softmax = new Softmax()
+        this.score = Math.sqrt(embeddingSize)
+        this.query = new Linear(embeddingSize, embeddingSize, { save: { filename: [`./${configs.dirname}/query.bin`, `./${configs.dirname}/queryBias.bin`] } })
+        this.key = new Linear(embeddingSize, embeddingSize, { save: { filename: [`./${configs.dirname}/key.bin`, `./${configs.dirname}/keyBias.bin`] } })
+        this.value = new Linear(embeddingSize, embeddingSize, { save: { filename: [`./${configs.dirname}/value.bin`, `./${configs.dirname}/valueBias.bin`] } })
+    }
+    forward(vectors) {
+        this.q = vectors.map((e) => this.query.forward(e))
+        this.v = vectors.map((e) => this.value.forward(e))
+        this.k = vectors.map((e) => this.key.forward(e))
+        // console.log(this.q, this.k, this.v)
+        const attentionSocres = []
+        for (let i = 0; i < this.q.length; i++) {
+            attentionSocres[i] = new Float32Array(this.k.length)
+            for (let a = 0; a < this.k.length; a++) {
+                let dot = 0;
+                for (let d = 0; d < this.q[i].length; d++) {
+                    dot += this.q[i][d] * this.k[a][d]
+                }
+                attentionSocres[i][a] = dot / this.score
+            }
+        }
+        const weights = attentionSocres.map((e) => this.softmax.forward(e))
+        this.weights = weights
+        const output = [];
+
+        for (let i = 0; i < weights.length; i++) {
+            output[i] = new Float32Array(this.v[0].length);
+
+            for (let j = 0; j < this.v.length; j++) {
+                for (let d = 0; d < this.v[j].length; d++) {
+                    output[i][d] += weights[i][j] * this.v[j][d];
+                }
+            }
+        }
+
+        return output;
+    }
+    softmaxBack(probs, dOut) {
+        const dIn = new Float32Array(probs.length);
+
+        for (let i = 0; i < probs.length; i++) {
+            let sum = 0;
+
+            for (let j = 0; j < probs.length; j++) {
+                const jacobian =
+                    i === j
+                        ? probs[i] * (1 - probs[i])
+                        : -probs[i] * probs[j];
+
+                sum += jacobian * dOut[j];
+            }
+
+            dIn[i] = sum;
+        }
+
+        return dIn;
+    }
+    backward(dOutput, learningRate) {
+        const dV = this.v.map((e) => new Float32Array(e.length));
+        const dWeights = []
+        for (let a = 0; a < this.weights.length; a++) {
+            dWeights[a] = new Float32Array(this.weights[a].length)
+            for (let j = 0; j < this.v.length; j++) {
+                for (let r = 0; r < this.v[j].length; r++) {
+                    dWeights[a][j] += dOutput[a][r] * this.v[j][r]
+                    dV[j][r] += dOutput[a][r] * this.weights[a][j]
+                }
+            }
+        }
+        const dScores = []
+        for (let t = 0; t < dWeights.length; t++) {
+            dScores[t] = this.softmaxBack(this.weights[t], dWeights[t])
+        }
+        const dQ = this.q.map(q => new Float32Array(q.length));
+        const dK = this.k.map(k => new Float32Array(k.length));
+        for (let a = 0; a < this.q.length; a++) {
+            for (let j = 0; j < this.k.length; j++) {
+                for (let r = 0; r < this.q[a].length; r++) {
+                    dQ[a][r] += dScores[a][j] * this.k[j][r] / this.score;
+                    dK[j][r] += dScores[a][j] * this.q[a][r] / this.score;
+                }
+            }
+        }
+        const dInput = this.q.map(() => new Float32Array(this.q[0].length))
+        for (let a = 0; a < this.q.length; a++) {
+            const dq = this.query.backward(dQ[a], learningRate, a)
+            const dk = this.key.backward(dK[a], learningRate, a)
+            const dv = this.value.backward(dV[a], learningRate, a)
+            for (let b = 0; b < dq.length; b++) {
+                dInput[a][b] = dq[b] + dk[b] + dv[b]
+            }
+        }
+        return dInput
+    }
+    Save() {
+        this.query.Save()
+        this.key.Save()
+        this.value.Save()
+    }
+}