catniff 0.2.10 → 0.2.12

package/README.md

@@ -87,6 +87,9 @@ I'm mostly just learning and playing with this currently, so there are no concre
 * Proper documentation.
 * GPU acceleration.
 * Some general neural net APIs.
+* Refactor code.
+* Proper tests.
+* Option to load more backends.
 
 ## Copyrights and License
 

package/dist/core.d.ts

@@ -41,6 +41,8 @@ export declare class Tensor {
     sum(dims?: number[] | number, keepDims?: boolean): Tensor;
     prod(dims?: number[] | number, keepDims?: boolean): Tensor;
     mean(dims?: number[] | number, keepDims?: boolean): Tensor;
+    max(dims?: number[] | number, keepDims?: boolean): Tensor;
+    min(dims?: number[] | number, keepDims?: boolean): Tensor;
     add(other: TensorValue | Tensor): Tensor;
     sub(other: TensorValue | Tensor): Tensor;
     subtract: (other: TensorValue | Tensor) => Tensor;
@@ -112,6 +114,12 @@ export declare class Tensor {
     relu(): Tensor;
     sigmoid(): Tensor;
     tanh(): Tensor;
+    softplus(): Tensor;
+    softsign(): Tensor;
+    silu(): Tensor;
+    mish(): Tensor;
+    maximum(other: TensorValue | Tensor): Tensor;
+    minimum(other: TensorValue | Tensor): Tensor;
     round(): Tensor;
     floor(): Tensor;
     ceil(): Tensor;
@@ -120,6 +128,9 @@ export declare class Tensor {
     frac(): Tensor;
     clip(min: number, max: number): Tensor;
     clamp: (min: number, max: number) => Tensor;
+    erf(): Tensor;
+    erfc(): Tensor;
+    erfinv(): Tensor;
     transpose(dim1: number, dim2: number): Tensor;
     swapaxes: (dim1: number, dim2: number) => Tensor;
     swapdims: (dim1: number, dim2: number) => Tensor;

package/dist/core.js

@@ -1,6 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.Tensor = void 0;
+const utils_1 = require("./utils");
 class Tensor {
     value;
     shape;
@@ -251,7 +252,7 @@ class Tensor {
     // Tensor squeeze
     squeeze(dims) {
         if (typeof this.value === "number")
-            return new Tensor(this.value);
+            return this;
         if (typeof dims === "number") {
             dims = [dims];
         }
@@ -299,10 +300,9 @@ class Tensor {
     }
     // Tensor unsqueeze - adds dimension of size 1 at specified position
     unsqueeze(dim) {
-        if (typeof this.value === "number")
-            return new Tensor([this.value]);
-        if (dim < 0 || dim > this.shape.length) {
-            throw new Error(`Invalid dimension ${dim} for unsqueeze`);
+        let thisValue = this.value;
+        if (typeof thisValue === "number") {
+            thisValue = [thisValue];
         }
         // Insert size-1 dimension at specified position
         const newShape = [...this.shape];
@@ -319,7 +319,7 @@ class Tensor {
             newDimStride = this.strides[dim] * this.shape[dim];
         }
         newStrides.splice(dim, 0, newDimStride);
-        const out = new Tensor(this.value, { shape: newShape, strides: newStrides });
+        const out = new Tensor(thisValue, { shape: newShape, strides: newStrides });
         // Set up gradient if needed
         if (this.requiresGrad) {
             out.requiresGrad = true;
@@ -333,7 +333,7 @@ class Tensor {
     // Tensor sum reduction
     sum(dims, keepDims = false) {
         if (typeof this.value === "number")
-            return new Tensor(this.value);
+            return this;
         if (typeof dims === "number") {
             dims = [dims];
         }
@@ -363,6 +363,7 @@ class Tensor {
             const outFlatIndex = Tensor.coordsToIndex(outCoords, outputStrides);
             // Accumulate, outFlatIndex should match multiple realFlatIndexes
             const realFlatIndex = Tensor.coordsToIndex(coords, this.strides);
+            // Add into sum
             outputValue[outFlatIndex] += this.value[realFlatIndex];
             // Mark for gradient if needed
             if (this.requiresGrad) {
@@ -387,7 +388,7 @@ class Tensor {
     // Tensor product reduction
     prod(dims, keepDims = false) {
         if (typeof this.value === "number")
-            return new Tensor(this.value);
+            return this;
         if (typeof dims === "number") {
             dims = [dims];
         }
@@ -400,14 +401,6 @@ class Tensor {
         const outputSize = Tensor.shapeToSize(outputShape);
         const outputValue = new Array(outputSize).fill(1);
         const originalSize = Tensor.shapeToSize(this.shape);
-        // Gradient data
-        let gradShape, gradStrides, gradValue = [];
-        // Allocate gradient data only when needed
-        if (this.requiresGrad) {
-            gradShape = this.shape;
-            gradStrides = this.strides;
-            gradValue = new Array(originalSize).fill(0);
-        }
         // Calculate new value after multiplying
         for (let index = 0; index < originalSize; index++) {
             const coords = Tensor.indexToCoords(index, this.strides);
@@ -417,6 +410,7 @@ class Tensor {
             const outFlatIndex = Tensor.coordsToIndex(outCoords, outputStrides);
             // Accumulate, outFlatIndex should match multiple realFlatIndexes
             const realFlatIndex = Tensor.coordsToIndex(coords, this.strides);
+            // Multiply into product
             outputValue[outFlatIndex] *= this.value[realFlatIndex];
         }
         const out = new Tensor(outputValue, {
@@ -425,7 +419,7 @@ class Tensor {
         });
         // Set up gradient if needed
         if (this.requiresGrad) {
-            // Grad is the product of other elements of the same axis, which is product of all els divided by the current value
+            const gradShape = this.shape, gradStrides = this.strides, gradValue = new Array(originalSize).fill(0);
             for (let index = 0; index < originalSize; index++) {
                 const coords = Tensor.indexToCoords(index, this.strides);
                 // Force 0 on reduced axes to collapse into size-1 dims
@@ -434,7 +428,7 @@ class Tensor {
                 const outFlatIndex = Tensor.coordsToIndex(outCoords, outputStrides);
                 // Accumulate, outFlatIndex should match multiple realFlatIndexes
                 const realFlatIndex = Tensor.coordsToIndex(coords, this.strides);
-                // Calculate gradient at position
+                // Grad is the product of other elements of the same axis, which is product of all els divided by the current value
                 gradValue[realFlatIndex] = outputValue[outFlatIndex] / this.value[realFlatIndex];
             }
             out.requiresGrad = true;
@@ -449,7 +443,7 @@ class Tensor {
     // Tensor mean reduction
     mean(dims, keepDims = false) {
         if (typeof this.value === "number")
-            return new Tensor(this.value);
+            return this;
         if (typeof dims === "number") {
             dims = [dims];
         }
@@ -463,14 +457,6 @@ class Tensor {
         const outputValue = new Array(outputSize).fill(0);
         const outputFeeders = new Array(outputSize).fill(0);
         const originalSize = Tensor.shapeToSize(this.shape);
-        // Gradient data
-        let gradShape, gradStrides, gradValue = [];
-        // Allocate gradient data only when needed
-        if (this.requiresGrad) {
-            gradShape = this.shape;
-            gradStrides = this.strides;
-            gradValue = new Array(originalSize).fill(0);
-        }
         // Calculate sums and how many elements contribute to specific positions
         for (let index = 0; index < originalSize; index++) {
             const coords = Tensor.indexToCoords(index, this.strides);
@@ -480,6 +466,7 @@ class Tensor {
             const outFlatIndex = Tensor.coordsToIndex(outCoords, outputStrides);
             // Accumulate, outFlatIndex should match multiple realFlatIndexes
             const realFlatIndex = Tensor.coordsToIndex(coords, this.strides);
+            // Calculate sum and contributors to the sum
             outputValue[outFlatIndex] += this.value[realFlatIndex];
             outputFeeders[outFlatIndex]++;
         }
@@ -493,7 +480,8 @@ class Tensor {
         });
         // Set up gradient if needed
         if (this.requiresGrad) {
-            // Calculate grad by assiging 1 divide by the number of contributors to the position
+            const gradShape = this.shape, gradStrides = this.strides, gradValue = new Array(originalSize).fill(0);
+            // Calculate grad by assigning 1 divided by the number of contributors to the position
             for (let index = 0; index < originalSize; index++) {
                 const coords = Tensor.indexToCoords(index, this.strides);
                 // Force 0 on reduced axes to collapse into size-1 dims
@@ -514,6 +502,120 @@ class Tensor {
         }
         return keepDims ? out : out.squeeze(dims);
     }
+    // Tensor maximum reduction
+    max(dims, keepDims = false) {
+        if (typeof this.value === "number")
+            return this;
+        if (typeof dims === "number") {
+            dims = [dims];
+        }
+        if (typeof dims === "undefined") {
+            dims = Array.from({ length: this.shape.length }, (_, index) => index);
+        }
+        // Dims that are reduced now have size-1
+        const outputShape = this.shape.map((dim, i) => dims.includes(i) ? 1 : dim);
+        const outputStrides = Tensor.getStrides(outputShape);
+        const outputSize = Tensor.shapeToSize(outputShape);
+        const outputValue = new Array(outputSize).fill(-Infinity);
+        const originalSize = Tensor.shapeToSize(this.shape);
+        // Calculate maximum values of axes
+        for (let index = 0; index < originalSize; index++) {
+            const coords = Tensor.indexToCoords(index, this.strides);
+            // Force 0 on reduced axes to collapse into size-1 dims
+            const outCoords = coords.map((val, i) => dims.includes(i) ? 0 : val);
+            // Convert output coordinates to flat index
+            const outFlatIndex = Tensor.coordsToIndex(outCoords, outputStrides);
+            // Accumulate, outFlatIndex should match multiple realFlatIndexes
+            const realFlatIndex = Tensor.coordsToIndex(coords, this.strides);
+            // Get max over time
+            if (this.value[realFlatIndex] > outputValue[outFlatIndex]) {
+                outputValue[outFlatIndex] = this.value[realFlatIndex];
+            }
+        }
+        const out = new Tensor(outputValue, {
+            shape: outputShape,
+            strides: outputStrides
+        });
+        // Set up gradient if needed
+        if (this.requiresGrad) {
+            const gradShape = this.shape, gradStrides = this.strides, gradValue = new Array(originalSize).fill(0);
+            for (let index = 0; index < originalSize; index++) {
+                const coords = Tensor.indexToCoords(index, this.strides);
+                // Force 0 on reduced axes to collapse into size-1 dims
+                const outCoords = coords.map((val, i) => dims.includes(i) ? 0 : val);
+                // Convert output coordinates to flat index
+                const outFlatIndex = Tensor.coordsToIndex(outCoords, outputStrides);
+                // Accumulate, outFlatIndex should match multiple realFlatIndexes
+                const realFlatIndex = Tensor.coordsToIndex(coords, this.strides);
+                // Calculate grad by checking if a positon holds a value equal to the max value
+                gradValue[realFlatIndex] = outputValue[outFlatIndex] === this.value[realFlatIndex] ? 1 : 0;
+            }
+            out.requiresGrad = true;
+            out.children.push(this);
+            out.gradFn = () => {
+                const localGrad = new Tensor(gradValue, { shape: gradShape, strides: gradStrides });
+                Tensor.addGrad(this, out.grad.withGrad(false).mul(localGrad));
+            };
+        }
+        return keepDims ? out : out.squeeze(dims);
+    }
+    // Tensor minimum reduction
+    min(dims, keepDims = false) {
+        if (typeof this.value === "number")
+            return this;
+        if (typeof dims === "number") {
+            dims = [dims];
+        }
+        if (typeof dims === "undefined") {
+            dims = Array.from({ length: this.shape.length }, (_, index) => index);
+        }
+        // Dims that are reduced now have size-1
+        const outputShape = this.shape.map((dim, i) => dims.includes(i) ? 1 : dim);
+        const outputStrides = Tensor.getStrides(outputShape);
+        const outputSize = Tensor.shapeToSize(outputShape);
+        const outputValue = new Array(outputSize).fill(Infinity);
+        const originalSize = Tensor.shapeToSize(this.shape);
+        // Calculate minimum values of axes
+        for (let index = 0; index < originalSize; index++) {
+            const coords = Tensor.indexToCoords(index, this.strides);
+            // Force 0 on reduced axes to collapse into size-1 dims
+            const outCoords = coords.map((val, i) => dims.includes(i) ? 0 : val);
+            // Convert output coordinates to flat index
+            const outFlatIndex = Tensor.coordsToIndex(outCoords, outputStrides);
+            // Accumulate, outFlatIndex should match multiple realFlatIndexes
+            const realFlatIndex = Tensor.coordsToIndex(coords, this.strides);
+            // Get min over time
+            if (this.value[realFlatIndex] < outputValue[outFlatIndex]) {
+                outputValue[outFlatIndex] = this.value[realFlatIndex];
+            }
+        }
+        const out = new Tensor(outputValue, {
+            shape: outputShape,
+            strides: outputStrides
+        });
+        // Set up gradient if needed
+        if (this.requiresGrad) {
+            const gradShape = this.shape, gradStrides = this.strides, gradValue = new Array(originalSize).fill(0);
+            for (let index = 0; index < originalSize; index++) {
+                const coords = Tensor.indexToCoords(index, this.strides);
+                // Force 0 on reduced axes to collapse into size-1 dims
+                const outCoords = coords.map((val, i) => dims.includes(i) ? 0 : val);
+                // Convert output coordinates to flat index
+                const outFlatIndex = Tensor.coordsToIndex(outCoords, outputStrides);
+                // Accumulate, outFlatIndex should match multiple realFlatIndexes
+                const realFlatIndex = Tensor.coordsToIndex(coords, this.strides);
+                // Calculate grad by checking if a positon holds a value equal to the min value
+                gradValue[realFlatIndex] = outputValue[outFlatIndex] === this.value[realFlatIndex] ? 1 : 0;
+            }
+            out.requiresGrad = true;
+            out.children.push(this);
+            out.gradFn = () => {
+                const localGrad = new Tensor(gradValue, { shape: gradShape, strides: gradStrides });
+                Tensor.addGrad(this, out.grad.withGrad(false).mul(localGrad));
+            };
+        }
+        return keepDims ? out : out.squeeze(dims);
+    }
     // Tensor element-wise addition
     add(other) {
         return this.elementWiseABDAG(other, (a, b) => a + b, (self, other, outGrad) => outGrad, (self, other, outGrad) => outGrad);
@@ -534,7 +636,7 @@ class Tensor {
     }
     // Tensor element-wise division
     div(other) {
-        return this.elementWiseABDAG(other, (a, b) => a / b, (self, other, outGrad) => outGrad.div(other), (self, other, outGrad) => outGrad.mul(self.neg().div(other.pow(2))));
+        return this.elementWiseABDAG(other, (a, b) => a / b, (self, other, outGrad) => outGrad.div(other), (self, other, outGrad) => outGrad.mul(self.neg().div(other.square())));
     }
     divide = this.div;
     // Tensor element-wise modulo
@@ -618,7 +720,7 @@ class Tensor {
     negative = this.neg;
     // Tensor element-wise reciprocal
     reciprocal() {
-        return this.elementWiseSelfDAG((a) => 1 / a, (self, outGrad) => outGrad.mul(self.neg().pow(-2)));
+        return this.elementWiseSelfDAG((a) => 1 / a, (self, outGrad) => outGrad.mul(self.pow(-2).neg()));
     }
     // Tensor element-wise square
     square() {
@@ -643,21 +745,21 @@ class Tensor {
     }
     // Tensor element-wise tan
     tan() {
-        return this.elementWiseSelfDAG((a) => Math.tan(a), (self, outGrad) => outGrad.mul(self.tan().pow(2).add(1)));
+        return this.elementWiseSelfDAG((a) => Math.tan(a), (self, outGrad) => outGrad.mul(self.tan().square().add(1)));
     }
     // Tensor element-wise asin
     asin() {
-        return this.elementWiseSelfDAG((a) => Math.asin(a), (self, outGrad) => outGrad.div(self.pow(2).neg().add(1).sqrt()));
+        return this.elementWiseSelfDAG((a) => Math.asin(a), (self, outGrad) => outGrad.div(self.square().neg().add(1).sqrt()));
     }
     arcsin = this.asin;
     // Tensor element-wise acos
     acos() {
-        return this.elementWiseSelfDAG((a) => Math.acos(a), (self, outGrad) => outGrad.div(self.pow(2).neg().add(1).sqrt()).neg());
+        return this.elementWiseSelfDAG((a) => Math.acos(a), (self, outGrad) => outGrad.div(self.square().neg().add(1).sqrt()).neg());
     }
     arccos = this.acos;
     // Tensor element-wise atan
     atan() {
-        return this.elementWiseSelfDAG((a) => Math.atan(a), (self, outGrad) => outGrad.div(self.pow(2).add(1)));
+        return this.elementWiseSelfDAG((a) => Math.atan(a), (self, outGrad) => outGrad.div(self.square().add(1)));
     }
     arctan = this.atan;
     // Tensor element-wise atan2
@@ -675,7 +777,7 @@ class Tensor {
     }
     // Tensor element-wise asinh
     asinh() {
-        return this.elementWiseSelfDAG((a) => Math.asinh(a), (self, outGrad) => outGrad.div(self.pow(2).add(1).sqrt()));
+        return this.elementWiseSelfDAG((a) => Math.asinh(a), (self, outGrad) => outGrad.div(self.square().add(1).sqrt()));
     }
     arcsinh = this.asinh;
     // Tensor element-wise acosh
@@ -685,7 +787,7 @@ class Tensor {
     arccosh = this.acosh;
     // Tensor element-wise atanh
     atanh() {
-        return this.elementWiseSelfDAG((a) => Math.atanh(a), (self, outGrad) => outGrad.div(self.pow(2).neg().add(1)));
+        return this.elementWiseSelfDAG((a) => Math.atanh(a), (self, outGrad) => outGrad.div(self.square().neg().add(1)));
     }
     arctanh = this.atanh;
     // Tensor element-wise degree to radian
@@ -734,7 +836,7 @@ class Tensor {
     }
     // Tensor element-wise relu
     relu() {
-        return this.elementWiseSelfDAG((a) => Math.max(a, 0), (self, outGrad) => outGrad.mul(self.ge(0)));
+        return this.elementWiseSelfDAG((a) => Math.max(a, 0), (self, outGrad) => outGrad.mul(self.gt(0)));
     }
     // Tensor element-wise sigmoid
     sigmoid() {
@@ -745,7 +847,39 @@ class Tensor {
     }
     // Tensor element-wise tanh
     tanh() {
-        return this.elementWiseSelfDAG((a) => Math.tanh(a), (self, outGrad) => outGrad.mul(self.tanh().pow(2).neg().add(1)));
+        return this.elementWiseSelfDAG((a) => Math.tanh(a), (self, outGrad) => outGrad.mul(self.tanh().square().neg().add(1)));
+    }
+    // Tensor element-wise softplus
+    softplus() {
+        return this.elementWiseSelfDAG((a) => Math.log1p(Math.exp(a)), (self, outGrad) => outGrad.mul(self.sigmoid()));
+    }
+    // Tensor element-wise softsign
+    softsign() {
+        return this.elementWiseSelfDAG((a) => a / (1 + Math.abs(a)), (self, outGrad) => outGrad.div(self.abs().add(1).square()));
+    }
+    // Tensor element-wise silu (swish)
+    silu() {
+        return this.elementWiseSelfDAG((a) => a / (1 + Math.exp(-a)), (self, outGrad) => {
+            const sig = self.sigmoid();
+            return outGrad.mul(sig.add(self.mul(sig).mul(sig.neg().add(1))));
+        });
+    }
+    // Tensor element-wise mish
+    mish() {
+        return this.elementWiseSelfDAG((a) => a * Math.tanh(Math.log1p(Math.exp(a))), (self, outGrad) => {
+            const tanhSoftPlus = self.exp().add(1).log().tanh();
+            // tanh(softplus(x)) + x * (1 - tanh²(softplus(x))) * sigmoid(x)
+            const derivative = tanhSoftPlus.add(self.mul(tanhSoftPlus.square().neg().add(1)).mul(self.sigmoid()));
+            return outGrad.mul(derivative);
+        });
+    }
+    // Tensor element-wise maximum
+    maximum(other) {
+        return this.elementWiseABDAG(other, (a, b) => Math.max(a, b), (self, other, outGrad) => outGrad.mul(self.gt(other).add(self.eq(other).mul(0.5))), (self, other, outGrad) => outGrad.mul(other.gt(self).add(other.eq(self).mul(0.5))));
+    }
+    // Tensor element-wise minimum
+    minimum(other) {
+        return this.elementWiseABDAG(other, (a, b) => Math.min(a, b), (self, other, outGrad) => outGrad.mul(self.lt(other).add(self.eq(other).mul(0.5))), (self, other, outGrad) => outGrad.mul(other.lt(self).add(other.eq(self).mul(0.5))));
     }
     // Tensor element-wise round
     round() {
@@ -773,6 +907,18 @@ class Tensor {
         return this.elementWiseSelfDAG((a) => Math.max(min, Math.min(max, a)), (self, outGrad) => outGrad.mul(self.ge(min).mul(self.le(max))));
     }
     clamp = this.clip;
+    // Tensor element-wise error function
+    erf() {
+        return this.elementWiseSelfDAG((a) => (0, utils_1.erf)(a), (self, outGrad) => outGrad.mul(self.square().neg().exp().mul(2 / Math.sqrt(Math.PI))));
+    }
+    // Tensor element-wise complementary error function
+    erfc() {
+        return this.elementWiseSelfDAG((a) => (0, utils_1.erfc)(a), (self, outGrad) => outGrad.mul(self.square().neg().exp().mul(2 / Math.sqrt(Math.PI)).neg()));
+    }
+    // Tensor element-wise inverse error function
+    erfinv() {
+        return this.elementWiseSelfDAG((a) => (0, utils_1.erfinv)(a), (self, outGrad) => outGrad.mul(self.erfinv().square().exp().mul(Math.sqrt(Math.PI) / 2)));
+    }
     // Transpose
     transpose(dim1, dim2) {
         // If dimension out of bound, throw error

package/dist/utils.d.ts

@@ -0,0 +1,3 @@
+export declare function erf(x: number): number;
+export declare function erfc(x: number): number;
+export declare function erfinv(x: number): number;

package/dist/utils.js

@@ -0,0 +1,35 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.erf = erf;
+exports.erfc = erfc;
+exports.erfinv = erfinv;
+// Error function using Abramowitz and Stegun approximation
+function erf(x) {
+    const a1 = 0.254829592;
+    const a2 = -0.284496736;
+    const a3 = 1.421413741;
+    const a4 = -1.453152027;
+    const a5 = 1.061405429;
+    const p = 0.3275911;
+    const sign = x >= 0 ? 1 : -1;
+    x = Math.abs(x);
+    // Formula 7.1.26
+    const t = 1.0 / (1.0 + p * x);
+    const y = 1.0 - (((((a5 * t + a4) * t) + a3) * t + a2) * t + a1) * t * Math.exp(-x * x);
+    return sign * y;
+}
+// Complementary error function
+function erfc(x) {
+    return 1 - erf(x);
+}
+// Inverse error function using Winitzki approximation
+function erfinv(x) {
+    if (Math.abs(x) >= 1)
+        throw new Error("Input must be in range (-1, 1)");
+    const a = 0.147;
+    const ln = Math.log(1 - x * x);
+    const part1 = 2 / (Math.PI * a) + ln / 2;
+    const part2 = ln / a;
+    const sign = x >= 0 ? 1 : -1;
+    return sign * Math.sqrt(-part1 + Math.sqrt(part1 * part1 - part2));
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "catniff",
-  "version": "0.2.10",
+  "version": "0.2.12",
   "description": "A cute autograd engine for Javascript",
   "main": "index.js",
   "scripts": {