@mineproness/tensorloop 1.5.9 → 2.1.0

package/src/GPU.js

@@ -1,279 +0,0 @@
-import { GPU } from "gpu.js";
-
-const gpu = new GPU({
-    mode: "gpu"
-})
-const kernel = gpu.createKernel(function () {
-    return 1
-}).setOutput([1])
-
-kernel()
-const gl = kernel.context
-const maxSize = gl ? gl.getParameter(gl.MAX_TEXTURE_SIZE) : 1200
-export function splitMatrix(matrix, chunksSizes = maxSize) {
-    const chunks = []
-    for (let a = 0; a < matrix.length; a += chunksSizes) {
-        chunks.push(matrix.slice(a, a + chunksSizes))
-    }
-    return { split: true, data: chunks }
-
-}
-let outputSizes = [];
-let out = 0
-export class GPULinear {
-    cacheLF = [];
-    cacheLBIas = [];
-    cacheLBWeights = [];
-    cacheLBInputGradent = [];
-    // texures
-    contextSize;
-    weightLength = 0;
-    Bias = [];
-    Weights = []
-    InitKernel(weights, bias, size, contextSize) {
-        const splitedWeights = splitMatrix(weights)
-        const splitedBias = splitMatrix(bias)
-        this.weightLength = weights.length
-        this.Weights = splitedWeights.data
-        out = weights.length
-        this.Bias = splitedBias.data
-        for (let a = 0; a < splitedWeights.data.length; a++) {
-            outputSizes.push(splitedWeights.data[a].length)
-
-            this.cacheLF[a] = gpu.createKernel(function (input, bias, weights) {
-                let sum = bias[this.thread.x]
-                for (let a = 0; a < this.constants.inputLength; a++) {
-                    sum += input[this.thread.y][a] * weights[this.thread.x][a]
-                }
-                return sum
-            }).setConstants({
-                inputLength: splitedWeights.data[a][0].length
-            }).setArgumentTypes({
-                input: 'Array',
-                bias: 'Array',
-                weights: 'Array'
-            }).setOutput([splitedWeights.data[a].length, contextSize])
-            this.cacheLBIas[a] = gpu.createKernel(function (
-                bias,
-                outputGradient,
-                learningRate
-            ) {
-                let grad = 0;
-                for (let a = 0; a < this.constants.contextSize; a++) {
-                    const g = outputGradient[a][this.thread.x]
-                    
-                    grad += g
-                }
-
-                return bias[this.thread.x] - learningRate * grad
-            }, {
-                pipeline: true,
-                immutable: true,
-                output: [splitedWeights.data[a].length, contextSize],
-                constants: { contextSize }
-            })
-            this.cacheLBInputGradent[a] = gpu.createKernel(function (
-                weights,
-                outputGradient
-            ) {
-                let sum = 0
-
-                for (let a = 0; a < this.constants.outputSize; a++) {
-                    const grad = outputGradient[this.thread.y][a]
-                    sum += grad * weights[a][this.thread.x]
-                }
-
-                return sum
-            }, {
-                constants: { outputSize: splitedWeights.data[a].length },
-                output: [size, contextSize]
-            })
-
-            this.cacheLBWeights[a] = gpu.createKernel(function (
-                weights,
-                outputGradient,
-                input,
-                learningRate
-            ) {
-                let grad = 0;
-                for (let a = 0; a < this.constants.contextSize; a++) {
-                    grad += outputGradient[a][this.thread.y] * input[a][this.thread.x]
-                }
-
-                return weights[this.thread.y][this.thread.x]
-                    - learningRate * grad
-            }, {
-                pipeline: true,
-                immutable: true,
-                output: [size, splitedWeights.data[a].length],
-                constants: { contextSize }
-            })
-        }
-
-        this.contextSize = contextSize
-
-    }
-    CalculateLinearForward(input) {
-        const result = Array.from({ length: this.contextSize },
-            () => new Float32Array(this.weightLength)
-        )
-        let offset = 0;
-        for (let a = 0; a < this.Weights.length; a++) {
-            const output = this.cacheLF[a](input, this.Bias[a], this.Weights[a])
-            for (let y = 0; y < this.contextSize; y++) {
-                result[y].set(output[y], offset);
-            }
-            offset += this.Weights[a].length
-
-        }
-        return result
-    }
-    CalculateBackward(out, learningRate, input) {
-        let inputGradient = Array.from({ length: this.contextSize } , ()=> new Float32Array(input[0].length))
-        for (let a = 0; a < this.Weights.length; a++) {
-            const outGradient = this.cacheLBInputGradent[a](this.Weights[a], out)
-            for (let row = 0; row < this.contextSize; row++) {
-                for (let i = 0; i < input[0].length; i++) {
-                    inputGradient[row][i] += outGradient[row][i];
-                }
-            }
-            const oldW = this.Weights[a]
-            const oldB = this.Bias[a]
-            this.Weights[a] = this.cacheLBWeights[a](this.Weights[a], out, input, learningRate)
-            this.Bias[a] = this.cacheLBIas[a](this.Bias[a], out, learningRate)
-            if (oldW && typeof oldW.delete === "function") {
-                oldW.delete()
-            }
-            if (oldB && typeof oldB.delete === "function") {
-                oldB.delete()
-            }
-
-        }
-        return inputGradient
-    }
-    GetWeightsAndBias() {
-        const weight = []
-        const bias = []
-        this.Weights.map((e) => weight.push(...e.toArray()))
-        this.Bias.map((e) => bias.push(...e.toArray()))
-        return { weights: weight, bias: new Float32Array(bias) }
-    }
-}
-
-export class gpuSoftmax {
-    cacheSF = [];
-    cacheSB = [];
-    contextSize;
-    Probs = [];
-    LossCalculate;
-    constructor(contextSize) {
-        this.contextSize = contextSize;
-        for (let a = 0; a < outputSizes.length; a++) {
-            this.cacheSF[a] = gpu.createKernel(function (logits) {
-                let maxLogits = logits[this.thread.y][0];
-                let sum = 0;
-                for (let a = 0; a < this.constants.outputSize; a++) {
-                    if (logits[this.thread.y][a] > maxLogits) {
-                        maxLogits = logits[this.thread.y][a]
-                    }
-                }
-                for (let a = 0; a < this.constants.outputSize; a++) {
-                    sum += Math.exp(logits[this.thread.y][a] - maxLogits)
-                }
-
-                return Math.exp(logits[this.thread.y][this.thread.x] - maxLogits) / sum
-
-            }, {
-                pipeline: true,
-                immutable: true,
-                output: [outputSizes[a], contextSize],
-                constants: { outputSize: outputSizes[a] }
-            })
-            this.cacheSB[a] = gpu.createKernel(function (probs, target) {
-                const p = probs[this.thread.y][this.thread.x]
-                if (this.thread.x == target[this.thread.y]) {
-                    return p - 1.0
-                }
-                return p
-            }, {
-                output: [outputSizes[a], contextSize]
-            })
-        }
-        this.LossCalculate = gpu.createKernel(function (probs, target) {
-            const tar = target[this.thread.x]
-            return -Math.log((probs[this.thread.x][tar] + 1) + 1e-10)
-
-
-        }, {
-            output: [contextSize]
-        })
-    }
-    forwardSoftmax(logits) {
-        const splitedLogits = splitMatrix(logits)
-        for (let a = 0; a < splitedLogits.data.length; a++) {
-            const oldSoftmax = this.Probs[a]
-            this.Probs[a] = this.cacheSF[a](splitedLogits.data[a])
-            if (oldSoftmax && typeof oldSoftmax.delete == "function") {
-                oldSoftmax.delete()
-            }
-        }
-        return "Completed"
-    }
-    getSoftmax() {
-        const result = Array.from({ length: this.contextSize }, () => new Float32Array(out))
-        let offset = 0;
-        for (let a = 0; a < this.Probs.length; a++) {
-            const outputofSoftmax = this.Probs[a].toArray()
-            for (let a = 0; a < this.contextSize; a++) {
-                result[a].set(outputofSoftmax[a], offset)
-            }
-            offset += outputofSoftmax[0].length
-        }
-        return result
-    }
-    softmaxBackward(target) {
-        const result = Array.from(
-            { length: this.contextSize },
-            () => new Float32Array(out)
-        );
-        let offset = 0;
-        for (let chunk = 0; chunk < this.Probs.length; chunk++) {
-            const chunkSize = outputSizes[chunk];
-            const localTarget = new Int32Array(this.contextSize);
-            for (let row = 0; row < this.contextSize; row++) {
-                const globalTarget = target[row];
-                if (
-                    globalTarget >= offset &&
-                    globalTarget < offset + chunkSize
-                ) {
-                    localTarget[row] = globalTarget - offset;
-                } else {
-                    localTarget[row] = -1;
-                }
-            }
-            const gradient = this.cacheSB[chunk](
-                this.Probs[chunk],
-                localTarget
-            );
-            const data = gradient;
-            for (let row = 0; row < this.contextSize; row++) {
-                result[row].set(data[row], offset);
-            }
-            offset += chunkSize;
-        }
-
-        return result;
-    }
-    calculateLOSS(dLogits, target) {
-        const losses = this.LossCalculate(dLogits, target)
-        let loss = 0;
-
-        for (let i = 0; i < losses.length; i++) {
-            loss += losses[i];
-        }
-
-        loss /= losses.length;
-        return loss
-    }
-
-}