@mineproness/tensorloop 2.1.54 → 2.1.56

package/src/Embedding/index.d.ts

@@ -1,11 +1,11 @@
 export class Embedding{
-    vectors: Float32Array[]
-    constructor(embeddingSize: number, vocabSize: number, contextSize: number, configs: {
+    constructor(device: GPUBuffer ,embedding : number, vocabSize: number, config:{
+        cpuReadBack: boolean,
         save: {
             filename: string
         }
     })
-    forward(token: number | number[]) : Float32Array[] | Float32Array[][]
-    backward(inputGradient: number[] | Float32Array[], learingRate: number) : void
-    Save() : void
+    async forward(tokens: number[]) : Promise<GPUBuffer>
+    async Save() : void
+    async backward(tokens: number[], inputGradient: GPUBuffer, lr: number) : Promise<void>
 }

package/src/Embedding/index.js

@@ -1,53 +1,138 @@
-import fs, { existsSync } from 'fs'
-import { SaveVectors, LoadVectors } from '../GetConfigs.js'
-import { GPUEmbedding } from '../gpiInit.js'
-// import { GPUEmbedding } from '../GPU.js'
-// import { config } from 'process'
+// import { config } from "process";
+import { SaveVector, getVector } from '../getVec.js'
+import { OperationManager } from "../OperationManager.js";
 
 function GenerateVectors(embeddingSize, vocabSize) {
-    const vector = []
-    for (let token = 0; token < vocabSize; token++) {
-        vector[token] = new Float32Array(embeddingSize)
-        for (let a = 0; a < embeddingSize; a++) {
-            vector[token][a] = (Math.random() * 2 - 1) * 0.2
-        }
-    }
-    return vector
+    const vec = new Float32Array(embeddingSize * vocabSize).map(() => (Math.random() * 2 - 1) * 0.02)
+    return vec
 }
 
 export class Embedding {
-    vectors = []
-    configs;
-    embeddingSize = 0
+    embeddingSize;
+    vocabSize;
     gpu;
-    constructor(embeddingSize, vocabSize, contextSize, configs) {
+    firstTrue;
+    vectors;
+    configs;
+    vectorsBuffer;
+    forwardPipline;
+    constructor(device, embeddingSize, vocabSize, configs) {
         this.configs = configs
-        this.embeddingSize = embeddingSize
+        this.embeddingSize = embeddingSize;
+        this.vocabSize = vocabSize;
+        this.device = device
+        let total = embeddingSize * vocabSize;
         if (configs && configs.save) {
-            if (existsSync(configs.save.filename)) {
-                this.vectors = LoadVectors(configs.save.filename, embeddingSize)
+            const vec = getVector(configs.save.filename, total);
+            if (vec) {
+                this.vectors = vec;
+
             } else {
                 this.vectors = GenerateVectors(embeddingSize, vocabSize)
-                SaveVectors(this.vectors, this.configs.save.filename)
             }
+
         } else {
             this.vectors = GenerateVectors(embeddingSize, vocabSize)
 
-        }  
-        this.gpu = new GPUEmbedding(this.vectors, vocabSize, embeddingSize, contextSize)
+        }
+        this.gpu = new OperationManager(device)
+        this.forwardPipline = this.gpu.getShaderPipline("./src/shaders/EF.wgsl", {
+            embeddingSize: embeddingSize
+        })
+        this.backwardPipline = this.gpu.getShaderPipline("./src/shaders/EB.wgsl", {
+            embeddingSize: embeddingSize
+        })
+        SaveVector(this.configs.save.filename, this.vectors)
     }
-    forward(token) {
-        return this.gpu.forward(token)
+    async forward(input) {
+        const encoder = this.device.createCommandEncoder()
+        if (!this.vectorsBuffer) {
+            this.vectorsBuffer = await this.gpu.createBuffer(Float32Array.BYTES_PER_ELEMENT * this.vocabSize * this.embeddingSize);
+            await this.gpu.WriteBuffer(this.vectorsBuffer, this.vectors)
+        }
+        const tokenBuffer = await this.gpu.createBuffer(Uint32Array.BYTES_PER_ELEMENT * input.length);
+        await this.gpu.WriteBuffer(tokenBuffer, new Uint32Array(input))
+        const outputBuffer = await this.gpu.createBuffer(Float32Array.BYTES_PER_ELEMENT * this.embeddingSize * input.length);
+        const bind = await this.device.createBindGroup({
+            layout: await this.forwardPipline.getBindGroupLayout(0),
+            entries: [
+                {
+                    binding: 0,
+                    resource: {
+                        buffer: this.vectorsBuffer
+                    }
+                },
+                {
+                    binding: 1,
+                    resource: {
+                        buffer: outputBuffer
+                    }
+                }, {
+                    binding: 2,
+                    resource: {
+                        buffer: tokenBuffer
+                    }
+                }
+            ]
+        })
+        await this.gpu.RunPipline(encoder,this.forwardPipline, bind, Math.ceil((this.embeddingSize * input.length) / 256))
+        this.device.queue.submit([encoder.finish()])
+        tokenBuffer.destroy()
+        if (this.configs.cpuReadBack) {
+            const output = await this.gpu.readBuffer(Float32Array.BYTES_PER_ELEMENT * this.embeddingSize * input.length, outputBuffer);
+            outputBuffer.destroy()
+            return output
+        }
+        return outputBuffer
+
     }
-    backward(inputGradient, learingRate){
-         this.gpu.backward(inputGradient, learingRate)
+    async Save() {
+        const vec = await this.gpu.readBuffer(Float32Array.BYTES_PER_ELEMENT * this.vocabSize * this.embeddingSize, this.vectorsBuffer)
+        SaveVector(this.configs.save.filename, vec)
     }
-    Save() {
-        const vectors = []
-        for(let i = 0; i < this.gpu.vectors.length; i++){
-            const out = this.gpu.vectors[i].toArray()
-            vectors.push(...out)
+    async backward(token, inputGradient, lr) {
+        const encoder = this.device.createCommandEncoder()
+        if (!this.oldParam) {
+            this.oldParam = new Float32Array([0, 0]);
+            this.param = await this.gpu.createUniFormBuffer(this.oldParam.byteLength);
+            this.tokenBuffer = await this.gpu.createBuffer(4 * token.length)
         }
-        SaveVectors(vectors.map((e)=> new Float32Array(e)), this.configs.save.filename)
+        if (this.oldParam[1] !== lr) {
+            this.oldParam = new Float32Array([lr, 0]);
+            this.gpu.WriteBuffer(this.param, this.oldParam)
+        }
+        const bind = await this.device.createBindGroup({
+            layout: await this.backwardPipline.getBindGroupLayout(0),
+            entries: [
+                {
+                    binding: 0,
+                    resource: {
+                        buffer: this.vectorsBuffer
+                    }
+                },
+                {
+                    binding: 1,
+                    resource: {
+                        buffer: inputGradient
+                    }
+                }, {
+                    binding: 2,
+                    resource: {
+                        buffer: this.tokenBuffer
+                    }
+
+                }, {
+                    binding: 3,
+                    resource: {
+                        buffer: this.param
+                    }
+
+                }
+            ]
+        })
+        await this.gpu.WriteBuffer(this.tokenBuffer, new Float32Array(token));
+        await this.gpu.RunPipline(encoder, this.backwardPipline, bind, Math.ceil((this.embeddingSize * token.length) / 256))
+        this.device.queue.submit([encoder.finish()])
+        inputGradient.destroy()
     }
 }

package/src/FFN/index.d.ts

@@ -1,9 +1,9 @@
 export class FFN{
-    constructor(embeddingSize: number, hiddenLayer: "gelu" | "relu" , configs: {
+    constructor(device: GPUAdapter, embeddingSize: number, configs: {
         dirname: string
     })
-    forward(input: Float32Array[]) : Float32Array[]
-    backward(dInput: Float32Array[], LearningRate: number) : Float32Array[]
-    Save() : void;
-    ClearInputCache() : void
+    async forward(vec: GPUBuffer) : Promise<GPUBuffer>
+    async backward(inputGradient: GPUBuffer, lr: number) : Promise<GPUBuffer>
+      async Save() : Promise<void>
+      async ClearInputCache() : Promise<void>
 }

package/src/FFN/index.js

@@ -1,51 +1,46 @@
-import { Linear } from '../Linear/index.js'
-import { GELU } from '../GELU/index.js'
-import { ReLU } from '../ReLU/index.js'
+import { RELU, Linear } from "../../index.js";
 import { join } from 'path'
-import { GPUHiddenLayer } from '../gpiInit.js';
+import { OperationManager } from "../OperationManager.js";
+
+
 export class FFN {
     linear1;
     linear2;
-    hidden;
-    hiddenName;
-    constructor(embeddingSize, hiddenLayer = "gelu", configs) {
-        this.hiddenName = hiddenLayer
-        this.hidden = hiddenLayer.toLocaleLowerCase() == "gelu" ? new GELU() : new GPUHiddenLayer()
-        if (configs) {
-            this.linear1 = new Linear(embeddingSize, 4 * embeddingSize, {
-                save: {
-                    filename: [join(configs.dirname, "ffnWeights.bin"), join(configs.dirname, "ffnBias.bin")]
-                }
-            })
-            this.linear2 = new Linear(4 * embeddingSize, embeddingSize, {
-                save: {
-                    filename: [join(configs.dirname, "ffnWeights1.bin"), join(configs.dirname, "ffnBias1.bin")]
-                }
-            })
-        } else {
-            this.linear1 = new Linear(embeddingSize, 4 * embeddingSize)
-            this.linear2 = new Linear(4 * embeddingSize, embeddingSize)
-        }
+    hiddenLayer;
+    Op
+    constructor(device, embeddingSize, configs) {
+        this.Op = new OperationManager(device)
+        this.linear1 = new Linear(device, embeddingSize, embeddingSize * 4, {
+            save: {
+                filename: [join(configs.dirname, "FFN_weights1.bin"), join(configs.dirname, "FFN_bias1.bin")]
+            }
+        });
+        this.hiddenLayer = new RELU(device, embeddingSize * 4);
+        this.linear2 = new Linear(device, embeddingSize * 4, embeddingSize, {
+            save: {
+                filename: [join(configs.dirname, "FFN_weights2.bin"), join(configs.dirname, "FFN_bias2.bin")]
+            }
+        })
     }
-    forward(input) {
-        const out1 = this.linear1.forward(input)
-        let hidden = this.hidden.forward(out1)
-        const result = this.linear2.forward(hidden)
-        return result
+    async forward(vec) {
+        const logit1 = await this.linear1.forward(vec)
+        const hidden = await this.hiddenLayer.forward(logit1)
+        const final = await this.linear2.forward(hidden)
+        return final
     }
-    backward(dInput, LearningRate) {
-        const out2 = this.linear2.backward(dInput, LearningRate);
-        const dHidden = this.hidden.backward(out2)
-        const result = this.linear1.backward(dHidden, LearningRate)
-        return result
+    async backward(inputGradient, lr) {
+        const logit2 = await this.linear2.backward(inputGradient, lr)
+        const hidden = await this.hiddenLayer.backward(logit2)
+        const final = await this.linear1.backward(hidden, lr)
+        inputGradient.destroy()
+        return final
     }
-    Save() {
-        this.linear1.Save()
-        this.linear2.Save()
+    async Save(){
+        await this.linear1.Save()
+        await this.linear2.Save()
     }
-    ClearInputCache() {
+    async ClearInputCache(){
         this.linear1.ClearInputCache()
         this.linear2.ClearInputCache()
-        this.hidden.ClearInputCache()
     }
 }

package/src/Linear/index.d.ts

@@ -1,13 +1,12 @@
-export class Linear{
-    Weights : Float32Array[]
-    Bias: Float32Array
-    constructor(embeddingSize : number, vocabSize : number, configs: {
+export class Linear {
+    constructor(device: GPUAdapter, embeddingSize: number, vocabSize: number, configs: {
+        cpuReadBack: boolean,
         save: {
             filename: string[]
         }
     })
-    forward(input : Float32Array[]) : Float32Array[]
-    backward(outGradient : number[] , learingRate: number) : Float32Array[]
-    Save() : void
-    ClearInputCache() : void
+    async forward(vec: GPUBuffer): Promise<GPUBuffer | Float32Array>
+    async backward(dOutput: GPUBuffer, lr: number): Promise<GPUBuffer>
+    async Save(): Promise<void>
+    async ClearInputCache() : Promise<void>
 }

package/src/Linear/index.js

@@ -1,67 +1,191 @@
-import { LoadBias, LoadVectors, SaveBias, SaveVectors } from '../GetConfigs.js'
-import { existsSync } from 'fs'
-import { GPULinear } from '../gpiInit.js'
-function GenerateWeightsBias(embeddingSize, vocabSize) {
-    const vectors = []
-    const Bias = new Float32Array(vocabSize)
-    for (let a = 0; a < vocabSize; a++) {
-        vectors[a] = new Float32Array(embeddingSize)
-        for (let t = 0; t < embeddingSize; t++) {
-            vectors[a][t] = (Math.random() * 2 - 1) * 0.02
-        }
-        Bias[a] = 0
-    }
-    return { vectors, Bias }
+
+import { create, globals } from "webgpu";
+import fs from 'fs'
+import { buffer } from "stream/consumers";
+import { SaveVector, getVector } from '../getVec.js'
+import { OperationManager } from "../OperationManager.js";
+Object.assign(globalThis, globals)
+const navigator = { gpu: create([]) }
+
+
+function GenerateVectors(embeddingSize, vocabSize) {
+    const vec = new Float32Array(embeddingSize * vocabSize).map(() => (Math.random() * 2 - 1) * 0.02)
+    return vec
+}
+
+function GenerateBias(vocabSize) {
+    const vec = new Float32Array(vocabSize)
+    return vec
 }
 
 export class Linear {
-    Weights = []
-    Bias = []
+    device;
+    embeddingSize;
+    vocabSize;
+    configs;
+    bias;
+    weights;
+    forwardPipline;
     configs;
-    input = [];
+    input;
+    wBuffer;
+    bBuffer;
     gpu;
-    constructor(embeddingSize, vocabSize, configs) {
+    paramForward;
+    oldParam;
+    constructor(device, embeddingSize, vocabSize, configs) {
+        this.configs = configs
+        this.embeddingSize = embeddingSize;
+        this.vocabSize = vocabSize;
+        this.configs = configs;
+        this.device = device
+        let total = embeddingSize * vocabSize;
         if (configs && configs.save) {
-            this.configs = configs
-            if (existsSync(configs.save.filename[0]) && existsSync(configs.save.filename[1])) {
-                const Vectors = LoadVectors(configs.save.filename[0], embeddingSize)
-                const Bias = LoadBias(configs.save.filename[1])
-                this.Weights = Vectors;
-                this.Bias = Bias
+            const w = getVector(configs.save.filename[0], total);
+            const b = getVector(configs.save.filename[1], total, true)
+            if (!w && !b) {
+                this.bias = GenerateBias(vocabSize)
+                this.weights = GenerateVectors(embeddingSize, vocabSize)
+                SaveVector(configs.save.filename[0], this.weights)
+                SaveVector(configs.save.filename[1], this.bias)
             } else {
-                const { vectors, Bias } = GenerateWeightsBias(embeddingSize, vocabSize)
-                this.Weights = vectors;
-                this.Bias = Bias;
-                SaveVectors(this.Weights, configs.save.filename[0])
-                SaveBias(this.Bias, configs.save.filename[1])
+                this.bias = b;
+                this.weights = w;
             }
         } else {
-            const { vectors, Bias } = GenerateWeightsBias(embeddingSize, vocabSize)
-            this.Weights = vectors;
-            this.Bias = Bias;
+            this.bias = GenerateBias(vocabSize)
+            this.weights = GenerateVectors(embeddingSize, vocabSize)
         }
-
+        this.gpu = new OperationManager(device);
+        this.forwardPipline = this.gpu.getShaderPipline("./src/shaders/LF.wgsl")
+        this.backwardPipline = this.gpu.getShaderPipline("./src/shaders/LB.wgsl")
+        this.dInputBackward = this.gpu.getShaderPipline("./src/shaders/ComputeDInput.wgsl")
     }
-    Save() {
-        const {bias, weights} = this.gpu.Save()
-        SaveVectors(weights, this.configs.save.filename[0])
-        SaveBias(bias, this.configs.save.filename[1])
+    async forward(vec) {
+        const encoder = this.device.createCommandEncoder()
+        const tokenCount = vec.size / (this.embeddingSize * 4)
+        if (!this.wBuffer) {
+
+            this.wBuffer = await this.gpu.createBuffer(this.embeddingSize * this.vocabSize * 4)
+            await this.gpu.WriteBuffer(this.wBuffer, this.weights)
+            this.bBuffer = await this.gpu.createBuffer(this.vocabSize * 4)
+            await this.gpu.WriteBuffer(this.bBuffer, this.bias)
+            this.paramForward = await this.gpu.createUniFormBuffer(4 * 4)
+            this.paramBackward = await this.gpu.createUniFormBuffer(4 * 5)
+        }
+        if (!this.oldParam) {
+            this.oldParam = new Uint32Array([this.embeddingSize, this.vocabSize, 0, 0])
+        }
+        if (this.oldParam[2] !== tokenCount) {
+            await this.gpu.WriteBuffer(this.paramForward, new Uint32Array([this.embeddingSize, this.vocabSize, tokenCount, 0]))
+            this.oldParam = new Uint32Array([this.embeddingSize, this.vocabSize, tokenCount, 0]);
+        }
+        const outputBuffer = await this.gpu.createBuffer(4 * this.vocabSize * tokenCount)
+        const bind = await this.device.createBindGroup({
+            layout: await this.forwardPipline.getBindGroupLayout(0),
+            entries: [
+                {
+                    binding: 0,
+                    resource: this.wBuffer
+                },
+                {
+                    binding: 1,
+                    resource: outputBuffer
+                },
+                {
+                    binding: 2,
+                    resource: vec
+                },
+                {
+                    binding: 3,
+                    resource: this.bBuffer
+                },
+                {
+                    binding: 4,
+                    resource: this.paramForward
+                }
+            ]
+        })
+        await this.gpu.RunPipline(encoder, this.forwardPipline, bind, Math.ceil((this.vocabSize * tokenCount) / 256))
+        if (this.input) this.input.destroy()
+        this.input = await this.gpu.MakeACopyBuffer(encoder, vec);
+        this.device.queue.submit([encoder.finish()])
+        return outputBuffer
     }
-    forward(input) {
-        if (!this.gpu) {
-            this.gpu = new GPULinear(this.Weights, this.Bias, input.length)
+    async backward(dOutput, learningRate) {
+        const encoder = this.device.createCommandEncoder()
+        const tokenCount = dOutput.size / (this.vocabSize * 4)
+        if (!this.oldParamb) {
+            this.oldParamb = new Float32Array([this.vocabSize, 0, 0, this.embeddingSize, 0])
         }
-        for (let a = 0; a < input.length; a++) {
-            this.input[a] = input;
+        if (this.oldParamb[1] !== tokenCount || this.oldParamb[2] !== learningRate) {
+            await this.gpu.WriteBuffer(this.paramBackward, new Float32Array([this.vocabSize, tokenCount, learningRate, this.embeddingSize, 0]));
+            this.oldParamb = new Float32Array([this.vocabSize, tokenCount, learningRate, this.embeddingSize, 0]);
         }
-        return this.gpu.forward(input)
+        const outputBuffer = await this.gpu.createBuffer(this.embeddingSize * tokenCount * 4);
+        const bind = await this.device.createBindGroup({
+            layout: await this.backwardPipline.getBindGroupLayout(0),
+            entries: [
+                {
+                    binding: 0,
+                    resource: this.bBuffer
+                },
+                {
+                    binding: 1,
+                    resource: this.wBuffer
+                },
+                {
+                    binding: 2,
+                    resource: dOutput
+                },
+                {
+                    binding: 3,
+                    resource: this.input
+                },
+                {
+                    binding: 4,
+                    resource: this.paramBackward
+                }
+            ]
+        })
+        const bind2 = await this.device.createBindGroup({
+            layout: await this.dInputBackward.getBindGroupLayout(0),
+            entries: [
+                {
+                    binding: 0,
+                    resource: this.wBuffer
+                },
+                {
+                    binding: 1,
+                    resource: dOutput
+                },
+                {
+                    binding: 3,
+                    resource: this.paramBackward
+                },
+                {
+                    binding: 4,
+                    resource: outputBuffer
+                }
+            ]
+        })
+        // console.log((await this.gpu.readBuffer(this.input.size, this.input))[0])
+        await this.gpu.RunPipline(encoder, this.dInputBackward, bind2, Math.ceil((this.embeddingSize * tokenCount) / 256));
+        await this.gpu.RunPipline(encoder, this.backwardPipline, bind, Math.ceil((this.vocabSize * tokenCount) / 256));
+        await this.device.queue.submit([encoder.finish()])
+        await dOutput.destroy()
+        this.input.destroy()
+        return outputBuffer
+
     }
-    backward(out, lr){
-        return this.gpu.backward(out, this.input, lr)
+    async Save() {
+        const b = await this.gpu.readBuffer(this.bBuffer.size, this.bBuffer);
+        const w = await this.gpu.readBuffer(this.wBuffer.size, this.wBuffer);
+        SaveVector(this.configs.save.filename[0], w)
+        SaveVector(this.configs.save.filename[1], b)
     }
-    ClearInputCache() {
-        this.input = []
+    async ClearInputCache() {
+        if (this.input) await this.input.destroy();
     }
-}
-
 
+}

package/src/OperationManager.js

@@ -0,0 +1,96 @@
+import fs from 'fs'
+
+export class OperationManager {
+    device;
+    matrixMatSHhaders;
+    params;
+    matrixParams;
+    constructor(device) {
+        this.device = device;
+    }
+    async createBuffer(size) {
+        return this.device.createBuffer({
+            size,
+            usage: GPUBufferUsage.COPY_DST | GPUBufferUsage.COPY_SRC | GPUBufferUsage.STORAGE
+        })
+    }
+    async createUniFormBuffer(size) {
+        return this.device.createBuffer({
+            size,
+            usage: GPUBufferUsage.COPY_DST | GPUBufferUsage.COPY_SRC | GPUBufferUsage.UNIFORM
+        })
+    }
+    async readBuffer(size, buffer) {
+        const readBuffer = this.device.createBuffer({
+            size: size,
+            usage: GPUBufferUsage.COPY_DST | GPUBufferUsage.MAP_READ
+        })
+        const encoder = this.device.createCommandEncoder()
+        encoder.copyBufferToBuffer(buffer, 0, readBuffer, 0, size);
+        this.device.queue.submit([await encoder.finish()])
+        await readBuffer.mapAsync(GPUMapMode.READ);
+        const mapped = readBuffer.getMappedRange();
+        const result = new Float32Array(
+            mapped.slice(0)
+        );
+        readBuffer.unmap()
+        readBuffer.destroy()
+        return result
+    }
+    getShaderPipline(filename, value = { "embedding": 123 }, entryPoint = "main") {
+        let readShaderCode = fs.readFileSync(filename, "utf-8");
+        if (value) {
+            for (const a in value) {
+                readShaderCode = readShaderCode.replaceAll(`%${a}%`, value[a])
+            }
+        };
+        const compiledKernel = this.device.createShaderModule({
+            code: readShaderCode
+        });
+        const compiledPipLine = this.device.createComputePipeline({
+            layout: "auto",
+            compute: {
+                module: compiledKernel,
+                entryPoint: entryPoint
+            }
+        });
+        return compiledPipLine;
+    }
+    async WriteBuffer(buffer, data) {
+        this.device.queue.writeBuffer(buffer, 0, data)
+    }
+    async RunPipline(encoder, pipline, binding, dispatchWorkgroups, y = 1, z = 1) {
+        const pass = encoder.beginComputePass();
+        pass.setPipeline(pipline)
+        pass.setBindGroup(0, binding)
+        pass.dispatchWorkgroups(dispatchWorkgroups, y , z)
+        pass.end()
+    }
+   
+    async MakeACopyBuffer(encoder, sourceBuffer) {
+        const buffer = this.device.createBuffer({
+            size: sourceBuffer.size,
+            usage: GPUBufferUsage.COPY_DST | GPUBufferUsage.COPY_SRC | GPUBufferUsage.STORAGE
+        })
+        encoder.copyBufferToBuffer(sourceBuffer, 0, buffer, 0, sourceBuffer.size);
+        return buffer
+    }
+    CreateEncoder(){
+        return this.device.createCommandEncoder()
+    }
+    createBindGroup(shader,groupNum,ent){
+        return this.device.createBindGroup({
+            layout: shader.getBindGroupLayout(groupNum),
+            entries: ent.map((e, x)=>{
+                return {
+                    binding : x,
+                    resource: e 
+                }
+            })
+        })
+    }
+    async submitQueue(encoder){
+         this.device.queue.submit([encoder.finish()])
+    }
+
+}   

package/src/Relu/index.d.ts

@@ -0,0 +1,5 @@
+export class Relu{
+    constructor(device : GPUAdapter, embeddingSize : number);
+    async forward(input: GPUBuffer) : Promise<GPUBuffer>
+    async backward(outGradinet: GPUBuffer) : Promise<GPUBuffer>
+}