@mineproness/tensorloop 2.1.55 → 2.1.56

package/src/shaders/attentionShader/ComputeDV.wgsl

@@ -0,0 +1,37 @@
+struct Params {
+    contextSize: u32,
+    embeddingSize: u32,
+}
+
+@group(0) @binding(0)
+var<storage, read> A: array<f32>;
+
+@group(0) @binding(1)
+var<storage, read> dOutput: array<f32>;
+
+@group(0) @binding(2)
+var<storage, read_write> dV: array<f32>;
+
+@group(0) @binding(3)
+var<uniform> params: Params;
+
+
+@compute @workgroup_size(16, 16)
+fn main(@builtin(global_invocation_id) id: vec3<u32>)
+ {
+    let row = id.x;
+    let col = id.y;
+    let T = params.contextSize;
+    let D = params.embeddingSize;
+    if (row >= T || col >= D) {
+        return;
+    }
+    var sum = 0.0;
+     for (var i = 0u; i < T; i++) {
+        let aIndex = i * T + row;
+        let gradIndex = i * D + col;
+        sum += A[aIndex] * dOutput[gradIndex];
+    }
+    let outputIndex = row * D + col;
+    dV[outputIndex] = sum;
+}

package/src/shaders/attentionShader/ComputeK.wgsl

@@ -0,0 +1,30 @@
+struct Params {
+    contextSize: u32,
+    embeddingSize: u32,
+}
+@group(0) @binding(0)
+var<storage, read> dS: array<f32>;
+@group(0) @binding(1)
+var<storage, read> Q: array<f32>;
+@group(0) @binding(2)
+var<storage, read_write> dK: array<f32>;
+@group(0) @binding(3)
+var<uniform> params: Params;
+@compute @workgroup_size(16, 16)
+fn main(@builtin(global_invocation_id) id: vec3<u32>) {
+    let row = id.x;
+    let col = id.y;
+    let T = params.contextSize;
+    let D = params.embeddingSize;
+    if (row >= T || col >= D) {
+        return;
+    }
+    var sum = 0.0;
+    for (var i = 0u; i < T; i++) {
+        let dsIndex = i * T + row;
+        let qIndex = i * D + col;
+        sum += dS[dsIndex] * Q[qIndex];
+    }
+    let scale = sqrt(f32(D));
+    dK[row * D + col] = sum / scale;
+}

package/src/shaders/attentionShader/ComputedQ.wgsl

@@ -0,0 +1,31 @@
+struct Params {
+    contextSize: u32,
+    embeddingSize: u32,
+}
+@group(0) @binding(0)
+var<storage, read> dS: array<f32>;
+@group(0) @binding(1)
+var<storage, read> K: array<f32>;
+@group(0) @binding(2)
+var<storage, read_write> dQ: array<f32>;
+@group(0) @binding(3)
+var<uniform> params: Params;
+@compute @workgroup_size(16, 16)
+fn main(@builtin(global_invocation_id) id: vec3<u32>) {
+    let row = id.x;
+    let col = id.y;
+    let T = params.contextSize;
+    let D = params.embeddingSize;
+    if (row >= T || col >= D) {
+        return;
+    }
+    var sum = 0.0;
+    for (var i = 0u; i < T; i++) {
+        let dsIndex = row * T + i;
+        let kIndex = i * D + col;
+
+        sum += dS[dsIndex] * K[kIndex];
+    }
+    let scale = sqrt(f32(D));
+    dQ[row * D + col] = sum / scale;
+}

package/src/shaders/attentionShader/attentionSocres.wgsl

@@ -0,0 +1,40 @@
+struct Params {
+    contextSize: u32,
+    embeddingSize: u32,
+}
+
+@group(0) @binding(0)
+var<storage, read> Q: array<f32>;
+
+@group(0) @binding(1)
+var<storage, read> K: array<f32>;
+
+@group(0) @binding(2)
+var<storage, read_write> scores: array<f32>;
+
+@group(0) @binding(3)
+var<uniform> params: Params;
+
+@compute @workgroup_size(16, 16)
+fn main(@builtin(global_invocation_id) id: vec3<u32>) {
+    let row = id.x;
+    let col = id.y;
+
+    if (row >= params.contextSize || col >= params.contextSize) {
+        return;
+    }
+    var sum = 0.0;
+    for (var k = 0u; k < params.embeddingSize; k++) {
+        let qIndex = row * params.embeddingSize + k;
+        let kIndex = col * params.embeddingSize + k;
+        sum += Q[qIndex] * K[kIndex];
+    }
+
+    let outIndex = row * params.contextSize + col;
+    if (col > row) {
+        scores[outIndex] = - 1e9;
+        return;
+    }
+    let scale = sqrt(f32(params.embeddingSize));
+    scores[outIndex] = sum / scale;
+}

package/src/shaders/attentionShader/sumGradient.wgsl

@@ -0,0 +1,24 @@
+struct Params {
+    contextSize: u32,
+    embeddingSize: u32,
+}
+
+@group(0) @binding(0)
+var<storage, read> dXq: array<f32>;
+@group(0) @binding(1)
+var<storage, read> dXk: array<f32>;
+@group(0) @binding(2)
+var<storage, read> dXv: array<f32>;
+@group(0) @binding(3)
+var<storage, read_write> dX: array<f32>;
+@group(0) @binding(4)
+var<uniform> params: Params;
+@compute @workgroup_size(256)
+fn main(@builtin(global_invocation_id) id: vec3<u32>) {
+    let index = id.x;
+    let total = params.contextSize * params.embeddingSize;
+    if (index >= total) {
+        return;
+    }
+    dX[index] =dXq[index] +dXk[index] +dXv[index];
+}

package/src/GELU/index.d.ts

@@ -1,6 +0,0 @@
-export class GELU{
-    input : Float32Array[];
-    forward(input: Float32Array[]) : Float32Array[];
-    backward(Doutput: Float32Array[]) : Float32Array[];
-    ClearInputCache() : void;
-}

package/src/GELU/index.js

@@ -1,33 +0,0 @@
-import { GELUGPU } from "../gpiInit.js";
-
-export class GELU {
-    gpu;
-    inputs = [];
-    constructor() {
-        this.inputs = []
-        
-    }
-
-    forward(input) {
-        this.inputs = input
-        if(!this.gpu){
-            this.gpu = new GELUGPU(input[0].length, input.length)
-        }
-        // const output = new Float32Array(input.length)
-        // const k = Math.sqrt(2 / Math.PI) 
-        // for (let i = 0; i < input.length; i++) {
-        //     const x = input[i]
-        //     const u = k * (x + 0.044715 * x * x * x)
-        //     output[i] = 0.5 * x * (1 + Math.tanh(u))
-        // }
-        return this.gpu.forward(input)
-    }
-
-    backward(dOutput) {  
-        return this.gpu.backward(dOutput, this.inputs)
-    }
-
-    ClearInputCache() {
-        this.inputs = []
-    }
-}

package/src/GetConfigs.js

@@ -1,35 +0,0 @@
-import fs from 'fs'
-import { join } from 'path'
-
-export function SaveVectors(vectors, path){
-    const rows = vectors.length;
-    const cols = vectors[0].length;
-    const flatArray = new Float32Array(rows * cols)
-    vectors.forEach((e,i)=>{
-        flatArray.set(e, i * cols)
-    })
-    const bufferArray = Buffer.from(flatArray.buffer, flatArray.byteOffset, flatArray.byteLength)
-    fs.writeFileSync(path, bufferArray)
-}
-
-export function LoadVectors(path, cols){
-   const buffer = fs.readFileSync(path)
-   const flatArray = new Float32Array(buffer.buffer, buffer.byteOffset, buffer.byteLength / 4 )
-   const vectors = []
-   const rows = flatArray.length / cols;
-   for(let a = 0; a < rows; a++){
-       vectors.push(flatArray.slice(a * cols, (a+1) * cols))
-   }
-   return vectors
-}
-
-export function SaveBias(vectors, path){
-    const bufferArray = Buffer.from(vectors.buffer, vectors.byteOffset, vectors.byteLength)
-    fs.writeFileSync(path, bufferArray)
-}
-
-export function LoadBias(path){
-   const buffer = fs.readFileSync(path)
-   const flatArray = new Float32Array(buffer.buffer, buffer.byteOffset, buffer.byteLength / 4 )
-   return flatArray
-}

package/src/Model/index.d.ts

@@ -1,10 +0,0 @@
-export default class Model {
-    constructor(embeddingSize: number, contextSize: number, configs: {
-        dirname: string,
-        activation: "gelu" | "relu",
-    })
-    train(pair: {
-        inputIDX: number[],
-        targetIDX: number[]
-    }[], batchSize: number, batchExecution: number, epochs: number, learningRate: number): void
-}

package/src/Model/index.js

@@ -1,88 +0,0 @@
-import { Embedding, PositionEmbedding, Linear, Tokenizer, TransformerBlock, Softmax } from '../../index.js'
-import path, { join } from 'path'
-
-export default class Model {
-    embedding;
-    linear;
-    posEmbedding;
-    softmax;
-    tokenizer;
-    block1;
-    block2;
-    contextSize;
-    constructor(embeddingSize, contextSize, configs) {
-        this.tokenizer = new Tokenizer()
-        this.contextSize = contextSize
-        this.embedding = new Embedding(embeddingSize, this.tokenizer.vocabSize, contextSize, {
-            save: {
-                filename: path.join(configs.dirname, "vectors.bin")
-            }
-        })
-        this.posEmbedding = new PositionEmbedding(embeddingSize, contextSize, {
-            save: {
-                filename: join(configs.dirname, 'posVec.bin')
-            }
-        })
-        this.linear = new Linear(embeddingSize, this.tokenizer.vocabSize, {
-            save: {
-                filename: [join(configs.dirname, 'weights.bin'), join(configs.dirname, 'bias.bin')]
-            }
-        })
-        this.softmax = new Softmax()
-        this.block1 = new TransformerBlock(embeddingSize, {
-            dirname: join(configs.dirname, "block1"),
-            activation: configs.activation
-        })
-        this.block2 = new TransformerBlock(embeddingSize, {
-            dirname: join(configs.dirname, "block2"),
-            activation: configs.activation
-        })
-    }
-    train(pair = {
-        inputIDX: [],
-        targetIDX: []
-    }, epochs = 500, learningRate = 0.01) {
-        process.stdout.write("\x1Bc")
-        console.log("Preparing Training Pairs....")
-        const trainingPair = pair
-        let firstRun = true;
-        console.log(`TotalPairs: ${trainingPair.length}`)
-        if (trainingPair.length == 0 || !trainingPair) throw new Error(`The Batch Execution ${batchExecution} is doesn't exist.`);
-        for (let i = 0; i < epochs; i++) {
-            let totalLoss = 0;
-            for (let a = 0; a < trainingPair.length; a++) {
-                const input = trainingPair[a].inputIDX
-                const target = trainingPair[a].targetIDX
-                let vector = this.embedding.forward(input)
-                vector = this.posEmbedding.forward(vector)
-                vector = this.block1.forward(vector)
-                vector = this.block2.forward(vector)
-                const logits = this.linear.forward(vector)
-                const probs = this.softmax.forward(logits)
-                const loss = this.softmax.crossEntropy(probs, target)[0]
-                const dLogits = this.softmax.backward(probs, target)
-                totalLoss += loss;
-                let dOutput = this.linear.backward(dLogits, learningRate);
-                dOutput = this.block2.backward(dOutput, learningRate)
-                dOutput = this.block1.backward(dOutput, learningRate)
-                this.posEmbedding.backward(dOutput, learningRate)
-                this.embedding.backward(dOutput, learningRate)
-                if (firstRun) {
-                    firstRun = null;
-                    console.log("")
-                    console.log("")
-                }
-                const percentense = (((a + 1) / trainingPair.length) * 100).toFixed(2)
-                const filted = Math.round((percentense / 100) * 20);
-                const empty = 20 - filted
-        
-                process.stdout.write(`\rEPOCHS: ${i} [${"=".repeat(filted)}${" ".repeat(empty)}] ${percentense}% (${a + 1}/${trainingPair.length}) | loss ${(totalLoss / (a + 1)).toFixed(2)}`)
-            }
-            console.log("")
-            this.embedding.Save()
-            this.linear.Save()
-            this.block1.Save()
-            this.block2.Save()
-        }
-    }
-}

package/src/PositionEmbedding/index.d.ts

@@ -1,11 +0,0 @@
-export class PositionEmbedding{
-    vectors: Float32Array[]
-    constructor(embeddingSize: number, contextSize: number, configs: {
-        save: {
-            filename: string
-        }
-    })
-    forward(vectors: Float32Array[]) :Float32Array[]
-    backward(inputGradient: Float32Array[], learingRate: number) : void
-    Save() : void;
-}

package/src/PositionEmbedding/index.js

@@ -1,49 +0,0 @@
-import fs, { existsSync } from 'fs'
-import { LoadVectors, SaveVectors } from '../GetConfigs.js'
-// import { config } from 'process'
-import { GPUEmbedding, GPUPosition } from '../gpiInit.js'
-
-function GenerateVectors(embeddingSize, vocabSize) {
-    const vector = []
-    for (let token = 0; token < vocabSize; token++) {
-        vector[token] = new Float32Array(embeddingSize)
-        for (let a = 0; a < embeddingSize; a++) {
-            vector[token][a] = (Math.random() * 2 - 1) * 0.2
-        }
-    }
-    return vector
-}
-
-export class PositionEmbedding {
-    vectors = []
-    configs;
-    embeddingSize = 0;
-    gpu;
-    constructor(embeddingSize, vocabSize, configs) {
-        this.configs = configs
-        this.embeddingSize = embeddingSize
-        if (configs && configs.save) {
-            if (existsSync(configs.save.filename)) {
-                this.vectors = LoadVectors(configs.save.filename, embeddingSize)
-            } else {
-                this.vectors = GenerateVectors(embeddingSize, vocabSize)
-                SaveVectors(this.vectors, this.configs.save.filename)
-
-            }
-        } else {
-            this.vectors = GenerateVectors(embeddingSize, vocabSize)
-        }
-        this.gpu = new GPUPosition(this.vectors, embeddingSize, vocabSize)
-    }
-    forward(vec) {
-        return this.gpu.forward(vec)
-    }
-    backward(inputGradient, learingRate) {
-        this.gpu.backward(inputGradient, learingRate)
-    }
-    Save() {
-        const save = this.gpu.Save()
-        SaveVectors(save, this.configs.save.filename)
-
-    }
-}

package/src/ReLU/index.d.ts

@@ -1,5 +0,0 @@
-export class ReLU{
-    forward(XLi : Float32Array[] | number[]) : number[] | Float32Array[];
-    backward(outputGradient: number[] | Float32Array[]) : Float32Array[];
-    ClearInputCache() : void
-}

package/src/ReLU/index.js

@@ -1,17 +0,0 @@
-import { GPUHiddenLayer } from "../gpiInit.js";
-
-export class ReLU{
-    input;
-    gpu;
-    constructor(){
-        this.gpu = new GPUHiddenLayer()
-    }
-    forward(logits){
-        return this.gpu.forward(logits)
-    }
-    backward(outGradient){
-        return this.gpu.backward(outGradient)
-    }
-    
-    ClearInputCache(){}
-}