@mineproness/tensorloop 1.2.5 → 1.2.6

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mineproness/tensorloop",
-  "version": "1.2.5",
+  "version": "1.2.6",
   "description": "The most powerfull Machine Learning Library for AI.",
   "keywords": [
     "ai",

package/readme.md

@@ -10,6 +10,8 @@ A Library that Supply AI Tools and Large Language model Development tools like E
 - CrossEntroy for Loss Calculation.
 - Relu for Stable Layer.
 - Tokenizer for convert into tokens.
+- Self attention 
+- Position Embedding
 
 ## Configs
 

package/src/Embedding/index.js

@@ -1,5 +1,5 @@
 import fs, { existsSync } from 'fs'
-import { GetEmbeddingVectors, SaveEmbeddingVectors, SaveVectors } from '../GetConfigs.js'
+import { SaveVectors, LoadVectors } from '../GetConfigs.js'
 // import { config } from 'process'
 
 function GenerateVectors(embeddingSize, vocabSize) {
@@ -22,7 +22,7 @@ export class Embedding {
         this.embeddingSize = embeddingSize
         if (configs && configs.save) {
             if (existsSync(configs.save.filename)) {
-                this.vectors = []
+                this.vectors = LoadVectors(configs.save.filename, embeddingSize)
             } else {
                 this.vectors = GenerateVectors(embeddingSize, vocabSize)
                 this.Save()
@@ -35,15 +35,9 @@ export class Embedding {
     forward(token) {
         if (Array.isArray(token)) {
             return token.map((e) => {
-                const id = this.vectors[e]
-                if (id) return id
-                this.vectors[e] = GetEmbeddingVectors(e, this.embeddingSize, this.configs.save.filename)
                 return this.vectors[e]
             })
         } else {
-            if (!this.vectors[token]) {
-                this.vectors[token] = GetEmbeddingVectors(token, this.embeddingSize, this.configs.save.filename)
-            }
             return this.vectors[token]
         }
     }
@@ -53,6 +47,6 @@ export class Embedding {
         }
     }
     Save() {
-        SaveEmbeddingVectors(this.vectors, this.configs.save.filename)
+        SaveVectors(this.vectors, this.configs.save.filename)
     }
 }

package/src/GetConfigs.js

@@ -1,62 +1,35 @@
-import fs from "fs";
-import { join } from "path";
-
-export function SaveVectors(filename, vectors) {
-    const fd = fs.openSync(filename, "w");
-
-    try {
-        for (const vector of vectors) {
-            fs.writeSync(fd, Buffer.from(vector.buffer, vector.byteOffset, vector.byteLength));
-        }
-    } finally {
-        fs.closeSync(fd);
-    }
-}
-
-export function GetWeights(filename, embeddingSize) {
-    const fileBuffer = fs.readFileSync(filename)
-    const ELEMENTS_PER_ARRAY = embeddingSize;
-    const BYTES_PER_ELEMENT = Float32Array.BYTES_PER_ELEMENT
-    const BYTES_PER_ARRAY = ELEMENTS_PER_ARRAY * BYTES_PER_ELEMENT
-    const totalArraySaved = fileBuffer.byteLength / BYTES_PER_ARRAY;
-    const totalArray = []
-    for (let i = 0; i < totalArraySaved; i++) {
-        const currentOffset = fileBuffer.byteOffset + (i * BYTES_PER_ARRAY)
-        const f32 = new Float32Array(fileBuffer.buffer, currentOffset, ELEMENTS_PER_ARRAY)
-        totalArray.push(f32)
-    }
-    return totalArray
+import fs from 'fs'
+import { join } from 'path'
+
+export function SaveVectors(vectors, path){
+    const rows = vectors.length;
+    const cols = vectors[0].length;
+    const flatArray = new Float32Array(rows * cols)
+    vectors.forEach((e,i)=>{
+        flatArray.set(e, i * cols)
+    })
+    const bufferArray = Buffer.from(flatArray.buffer, flatArray.byteOffset, flatArray.byteLength)
+    fs.writeFileSync(path, bufferArray)
 }
 
-export function GetVector(filename) {
-    const buffer = fs.readFileSync(filename)
-    return new Float32Array(buffer.buffer, buffer.byteOffset, buffer.byteLength / Float32Array.BYTES_PER_ELEMENT)
+export function LoadVectors(path, cols){
+   const buffer = fs.readFileSync(path)
+   const flatArray = new Float32Array(buffer.buffer, buffer.byteOffset, buffer.byteLength / 4 )
+   const vectors = []
+   const rows = flatArray.length / cols;
+   for(let a = 0; a < rows; a++){
+       vectors.push(flatArray.slice(a * cols, (a+1) * cols))
+   }
+   return vectors
 }
 
-export function Save(filename, vector) {
-    return fs.writeFileSync(filename, Buffer.from(vector.buffer))
+export function SaveBias(vectors, path){
+    const bufferArray = Buffer.from(vectors.buffer, vectors.byteOffset, vectors.byteLength)
+    fs.writeFileSync(path, bufferArray)
 }
 
-export function GetEmbeddingVectors(token, embeddingSize, filename) {
-    const fd = fs.openSync(filename, "r")
-    const PER_BTYES_VECTOR = embeddingSize * Float32Array.BYTES_PER_ELEMENT
-    const buffer = Buffer.allocUnsafe(PER_BTYES_VECTOR)
-    fs.readSync(fd, buffer, 0, PER_BTYES_VECTOR, token * PER_BTYES_VECTOR)
-    fs.closeSync(fd)
-    return new Float32Array(buffer.buffer, buffer.byteOffset, embeddingSize)
-}
-
-export function SaveEmbeddingVectors(vectors = [new Float32Array([1, 2, 3])], filename) {
-    const fd = fs.openSync(filename, "w")
-    try {
-        for (let a = 0; a < vectors.length; a++) {
-            const vector = vectors[a]
-            if (vector == undefined) continue;
-            const offset = a * vector.length * Float32Array.BYTES_PER_ELEMENT;
-            fs.writeSync(fd, Buffer.from(vector.buffer, vector.byteOffset, vector.byteLength) , 0 , vector.byteLength, offset)
-
-        }
-    }finally{
-        fs.closeSync(fd)
-    }
+export function LoadBias(path){
+   const buffer = fs.readFileSync(path)
+   const flatArray = new Float32Array(buffer.buffer, buffer.byteOffset, buffer.byteLength / 4 )
+   return vectors
 }

package/src/Linear/index.js

@@ -1,4 +1,4 @@
-import { GetWeights, Save,GetVector ,  SaveVectors } from '../GetConfigs.js'
+import { LoadBias,LoadVectors,SaveBias,SaveVectors } from '../GetConfigs.js'
 import { existsSync } from 'fs'
 function GenerateWeightsBias(embeddingSize, vocabSize) {
     const vectors = []
@@ -22,8 +22,8 @@ export class Linear {
         if (configs && configs.save) {
             this.configs = configs
             if (existsSync(configs.save.filename[0]) && existsSync(configs.save.filename[1])) {
-                const Vectors = GetWeights(configs.save.filename[0], embeddingSize)
-                const Bias = GetVector(configs.save.filename[1])
+                const Vectors = LoadVectors(configs.save.filename[0], embeddingSize)
+                const Bias = LoadBias(configs.save.filename[1])
                 this.Weights = Vectors;
                 this.Bias = Bias
             } else {
@@ -31,7 +31,7 @@ export class Linear {
                 this.Weights = vectors;
                 this.Bias = Bias;
                 SaveVectors(configs.save.filename[0], this.Weights)
-                Save(configs.save.filename[1], this.Bias)
+                SaveBias(configs.save.filename[1], this.Bias)
             }
         } else {
             const { vectors, Bias } = GenerateWeightsBias(embeddingSize, vocabSize)
@@ -41,7 +41,7 @@ export class Linear {
     }
     Save() {
         SaveVectors(this.configs.save.filename[0], this.Weights)
-        Save(this.configs.save.filename[1], this.Bias)
+        SaveBias(this.configs.save.filename[1], this.Bias)
     }
     forward(input) {
         this.input.push(input)

package/src/PositionEmbedding/index.js

@@ -1,5 +1,5 @@
 import fs, { existsSync } from 'fs'
-import { GetEmbeddingVectors, SaveEmbeddingVectors, SaveVectors } from '../GetConfigs.js'
+import { LoadVectors, SaveVectors } from '../GetConfigs.js'
 // import { config } from 'process'
 
 function GenerateVectors(embeddingSize, vocabSize) {
@@ -22,7 +22,7 @@ export class PositionEmbedding {
         this.embeddingSize = embeddingSize
         if (configs && configs.save) {
             if (existsSync(configs.save.filename)) {
-                this.vectors = []
+                this.vectors = LoadVectors(onfigs.save.filename, embeddingSize)
             } else {
                 this.vectors = GenerateVectors(embeddingSize, vocabSize)
                 this.Save()
@@ -35,15 +35,10 @@ export class PositionEmbedding {
     forward(token) {
         if (Array.isArray(token)) {
             return token.map((e) => {
-                const id = this.vectors[e]
-                if (id) return id
-                this.vectors[e] = GetEmbeddingVectors(e, this.embeddingSize, this.configs.save.filename)
                 return this.vectors[e]
             })
         } else {
-            if (!this.vectors[token]) {
-                this.vectors[token] = GetEmbeddingVectors(token, this.embeddingSize, this.configs.save.filename)
-            }
+
             return this.vectors[token]
         }
     }
@@ -53,14 +48,14 @@ export class PositionEmbedding {
         }
     }
     Save() {
-        SaveEmbeddingVectors(this.vectors, this.configs.save.filename)
+        SaveVectors(this.vectors, this.configs.save.filename)
     }
-    addPos(PSEM, xEM) {
+    addPos(xEM) {
         const out = []
 
         for (let pos = 0; pos < xEM.length; pos++) {
             const tokenVector = xEM[pos]
-            const posVector = PSEM.forward(pos)
+            const posVector = this.forward(pos)
             const x = new Float32Array(xEM[0].length)
             for (let a = 0; a < tokenVector.length; a++) {
                 x[a] = tokenVector[a] + posVector[a]

package/src/SelfAttention/index.d.ts

@@ -3,9 +3,14 @@ export class SelfAttention{
     v : Float32Array[]
     k : Float32Array[]
     input;
+    constructor(embeddingSize : number, configs: {
+        save: {
+            dirname: string
+        }
+    })
     weights : Float32Array[]
     score : number
-    forward(vectors: Float32Array[])
+    forward(vectors: Float32Array[][])
     Save() : void
     backward(dOutput, learningRate)
 }