@mineproness/tensorloop 1.2.3 → 1.2.5

package/index.js

@@ -5,10 +5,14 @@ import { Softmax as softmax2 } from "./src/Softmax/index.js";
 import { crossEntropy } from "./src/crossEntropy/index.js";
 import {tools} from './src/tools/index.js'
 import {ReLU} from './src/ReLU/index.js'
+import {PositionEmbedding as PE} from './src/PositionEmbedding/index.js'
+import {SelfAttention as ST} from './src/SelfAttention/index.js'
 export const Tokenizer = tokenizer
 export const Embedding = embeddding2
 export const Linear = li2
 export const Softmax = softmax2
 export const CrossEntropy = crossEntropy
 export const Tools = tools
-export const ReLu = ReLU
+export const ReLu = ReLU
+export const PositionEmbedding = PE
+export const SelfAttention = ST

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mineproness/tensorloop",
-  "version": "1.2.3",
+  "version": "1.2.5",
   "description": "The most powerfull Machine Learning Library for AI.",
   "keywords": [
     "ai",
@@ -20,4 +20,5 @@
     "type": "git",
     "url": "https://github.com/mineproness-org/tensorloop.git"
   }
+
 }

package/src/Linear/index.d.ts

@@ -7,6 +7,6 @@ export class Linear{
         }
     })
     forward(input : number[]) : Float32Array[]
-    backward(outGradient : number[] , learingRate: number) : Float32Array[]
+    backward(outGradient : number[] , learingRate: number, idx: number) : Float32Array[]
     Save() : void
 }

package/src/Linear/index.js

@@ -44,7 +44,7 @@ export class Linear {
         Save(this.configs.save.filename[1], this.Bias)
     }
     forward(input) {
-        this.input = input
+        this.input.push(input)
         const output = new Float32Array(this.Weights.length)
         for (let a = 0; a < this.Weights.length; a++) {
             let sum = this.Bias[a]
@@ -55,10 +55,10 @@ export class Linear {
         }
         return output
     }
-    backward(outputGradient, learningRate) {
-        const input = this.input;
+    backward(outputGradient, learningRate, idx) {
+        const input = this.input[idx];
         const weights = this.Weights
-        const inputGradient = new Float32Array(this.input.length)
+        const inputGradient = new Float32Array(input.length)
         for (let a = 0; a < weights.length; a++) {
             const gran = outputGradient[a]
             const w = weights[a]

package/src/PositionEmbedding/index.d.ts

@@ -0,0 +1,12 @@
+export class PositionEmbedding{
+    vectors: Float32Array[]
+    constructor(embeddingSize: number, contextSize: number, configs: {
+        save: {
+            filename: string
+        }
+    })
+    forward(position: number | number[]) : Float32Array[] | Float32Array[][]
+    backward(pos: number , inputGradient: number[] | Float32Array[], learingRate: number) : void
+    Save() : void;
+    addPos(PSEM, vectors : Float32Array[][]) : Float32Array 
+}

package/src/PositionEmbedding/index.js

@@ -0,0 +1,72 @@
+import fs, { existsSync } from 'fs'
+import { GetEmbeddingVectors, SaveEmbeddingVectors, SaveVectors } from '../GetConfigs.js'
+// import { config } from 'process'
+
+function GenerateVectors(embeddingSize, vocabSize) {
+    const vector = []
+    for (let token = 0; token < vocabSize; token++) {
+        vector[token] = new Float32Array(embeddingSize)
+        for (let a = 0; a < embeddingSize; a++) {
+            vector[token][a] = (Math.random() * 2 - 1) * 0.2
+        }
+    }
+    return vector
+}
+
+export class PositionEmbedding {
+    vectors = []
+    configs;
+    embeddingSize = 0
+    constructor(embeddingSize, vocabSize, configs) {
+        this.configs = configs
+        this.embeddingSize = embeddingSize
+        if (configs && configs.save) {
+            if (existsSync(configs.save.filename)) {
+                this.vectors = []
+            } else {
+                this.vectors = GenerateVectors(embeddingSize, vocabSize)
+                this.Save()
+            }
+        } else {
+            this.vectors = GenerateVectors(embeddingSize, vocabSize)
+
+        }
+    }
+    forward(token) {
+        if (Array.isArray(token)) {
+            return token.map((e) => {
+                const id = this.vectors[e]
+                if (id) return id
+                this.vectors[e] = GetEmbeddingVectors(e, this.embeddingSize, this.configs.save.filename)
+                return this.vectors[e]
+            })
+        } else {
+            if (!this.vectors[token]) {
+                this.vectors[token] = GetEmbeddingVectors(token, this.embeddingSize, this.configs.save.filename)
+            }
+            return this.vectors[token]
+        }
+    }
+    backward(token, inputGradient, learingRate) {
+        for (let a = 0; a < this.vectors[token].length; a++) {
+            this.vectors[token][a] -= learingRate * inputGradient[a]
+        }
+    }
+    Save() {
+        SaveEmbeddingVectors(this.vectors, this.configs.save.filename)
+    }
+    addPos(PSEM, xEM) {
+        const out = []
+
+        for (let pos = 0; pos < xEM.length; pos++) {
+            const tokenVector = xEM[pos]
+            const posVector = PSEM.forward(pos)
+            const x = new Float32Array(xEM[0].length)
+            for (let a = 0; a < tokenVector.length; a++) {
+                x[a] = tokenVector[a] + posVector[a]
+            }
+            out.push(x)
+        }
+        return out
+    }
+}

package/src/SelfAttention/index.d.ts

@@ -0,0 +1,11 @@
+export class SelfAttention{
+    q : Float32Array[]
+    v : Float32Array[]
+    k : Float32Array[]
+    input;
+    weights : Float32Array[]
+    score : number
+    forward(vectors: Float32Array[])
+    Save() : void
+    backward(dOutput, learningRate)
+}

package/src/SelfAttention/index.js

@@ -0,0 +1,118 @@
+import { Softmax } from '../../index.js';
+import { Linear } from '../Linear/index.js'
+
+export class SelfAttention {
+    query;
+    key;
+    value;
+    v;
+    k;
+    q;
+    input;
+    configs;
+    score;
+    softmax;
+    weights;
+    constructor(embeddingSize, configs) {
+        this.configs = configs
+        this.softmax = new Softmax()
+        this.score = Math.sqrt(embeddingSize)
+        this.query = new Linear(embeddingSize, embeddingSize, { save: { filename: [`./${configs.dirname}/query.bin`, `./${configs.dirname}/queryBias.bin`] } })
+        this.key = new Linear(embeddingSize, embeddingSize, { save: { filename: [`./${configs.dirname}/key.bin`, `./${configs.dirname}/keyBias.bin`] } })
+        this.value = new Linear(embeddingSize, embeddingSize, { save: { filename: [`./${configs.dirname}/value.bin`, `./${configs.dirname}/valueBias.bin`] } })
+    }
+    forward(vectors) {
+        this.q = vectors.map((e) => this.query.forward(e))
+        this.v = vectors.map((e) => this.value.forward(e))
+        this.k = vectors.map((e) => this.key.forward(e))
+        // console.log(this.q, this.k, this.v)
+        const attentionSocres = []
+        for (let i = 0; i < this.q.length; i++) {
+            attentionSocres[i] = new Float32Array(this.k.length)
+            for (let a = 0; a < this.k.length; a++) {
+                let dot = 0;
+                for (let d = 0; d < this.q[i].length; d++) {
+                    dot += this.q[i][d] * this.k[a][d]
+                }
+                attentionSocres[i][a] = dot / this.score
+            }
+        }
+        const weights = attentionSocres.map((e) => this.softmax.forward(e))
+        this.weights = weights
+        const output = [];
+
+        for (let i = 0; i < weights.length; i++) {
+            output[i] = new Float32Array(this.v[0].length);
+
+            for (let j = 0; j < this.v.length; j++) {
+                for (let d = 0; d < this.v[j].length; d++) {
+                    output[i][d] += weights[i][j] * this.v[j][d];
+                }
+            }
+        }
+
+        return output;
+    }
+    softmaxBack(probs, dOut) {
+        const dIn = new Float32Array(probs.length);
+
+        for (let i = 0; i < probs.length; i++) {
+            let sum = 0;
+
+            for (let j = 0; j < probs.length; j++) {
+                const jacobian =
+                    i === j
+                        ? probs[i] * (1 - probs[i])
+                        : -probs[i] * probs[j];
+
+                sum += jacobian * dOut[j];
+            }
+
+            dIn[i] = sum;
+        }
+
+        return dIn;
+    }
+    backward(dOutput, learningRate) {
+        const dV = this.v.map((e) => new Float32Array(e.length));
+        const dWeights = []
+        for (let a = 0; a < this.weights.length; a++) {
+            dWeights[a] = new Float32Array(this.weights[a].length)
+            for (let j = 0; j < this.v.length; j++) {
+                for (let r = 0; r < this.v[j].length; r++) {
+                    dWeights[a][j] += dOutput[a][r] * this.v[j][r]
+                    dV[j][r] += dOutput[a][r] * this.weights[a][j]
+                }
+            }
+        }
+        const dScores = []
+        for (let t = 0; t < dWeights.length; t++) {
+            dScores[t] = this.softmaxBack(this.weights[t], dWeights[t])
+        }
+        const dQ = this.q.map(q => new Float32Array(q.length));
+        const dK = this.k.map(k => new Float32Array(k.length));
+        for (let a = 0; a < this.q.length; a++) {
+            for (let j = 0; j < this.k.length; j++) {
+                for (let r = 0; r < this.q[a].length; r++) {
+                    dQ[a][r] += dScores[a][j] * this.k[j][r] / this.score;
+                    dK[j][r] += dScores[a][j] * this.q[a][r] / this.score;
+                }
+            }
+        }
+        const dInput = this.q.map(() => new Float32Array(this.q[0].length))
+        for (let a = 0; a < this.q.length; a++) {
+            const dq = this.query.backward(dQ[a], learningRate, a)
+            const dk = this.key.backward(dK[a], learningRate, a)
+            const dv = this.value.backward(dV[a], learningRate, a)
+            for (let b = 0; b < dq.length; b++) {
+                dInput[a][b] = dq[b] + dk[b] + dv[b]
+            }
+        }
+        return dInput
+    }
+    Save() {
+        this.query.Save()
+        this.key.Save()
+        this.value.Save()
+    }
+}