catniff 0.2.11 → 0.2.13

package/README.md

@@ -87,6 +87,9 @@ I'm mostly just learning and playing with this currently, so there are no concre
 * Proper documentation.
 * GPU acceleration.
 * Some general neural net APIs.
+* Refactor code.
+* Proper tests.
+* Option to load more backends.
 
 ## Copyrights and License
 

package/dist/core.d.ts

@@ -41,6 +41,8 @@ export declare class Tensor {
     sum(dims?: number[] | number, keepDims?: boolean): Tensor;
     prod(dims?: number[] | number, keepDims?: boolean): Tensor;
     mean(dims?: number[] | number, keepDims?: boolean): Tensor;
+    max(dims?: number[] | number, keepDims?: boolean): Tensor;
+    min(dims?: number[] | number, keepDims?: boolean): Tensor;
     add(other: TensorValue | Tensor): Tensor;
     sub(other: TensorValue | Tensor): Tensor;
     subtract: (other: TensorValue | Tensor) => Tensor;
@@ -137,9 +139,20 @@ export declare class Tensor {
     mm(other: TensorValue | Tensor): Tensor;
     mv(other: TensorValue | Tensor): Tensor;
     matmul(other: TensorValue | Tensor): Tensor;
+    static full(shape: number[], num: number, options?: TensorOptions): Tensor;
     static fullLike(tensor: Tensor, num: number, options?: TensorOptions): Tensor;
+    static ones(shape?: number[], options?: TensorOptions): Tensor;
     static onesLike(tensor: Tensor, options?: TensorOptions): Tensor;
+    static zeros(shape?: number[], options?: TensorOptions): Tensor;
     static zerosLike(tensor: Tensor, options?: TensorOptions): Tensor;
+    static rand(shape?: number[], options?: TensorOptions): Tensor;
+    static randLike(tensor: Tensor, options?: TensorOptions): Tensor;
+    static randn(shape?: number[], options?: TensorOptions): Tensor;
+    static randnLike(tensor: Tensor, options?: TensorOptions): Tensor;
+    static randint(shape: number[], low: number, high: number, options?: TensorOptions): Tensor;
+    static randintLike(tensor: Tensor, low: number, high: number, options?: TensorOptions): Tensor;
+    static normal(shape: number[], mean: number, stdDev: number, options?: TensorOptions): Tensor;
+    static uniform(shape: number[], low: number, high: number, options?: TensorOptions): Tensor;
     backward(): void;
     val(): TensorValue;
     withGrad(requiresGrad: boolean): Tensor;

package/dist/core.js

@@ -252,7 +252,7 @@ class Tensor {
     // Tensor squeeze
     squeeze(dims) {
         if (typeof this.value === "number")
-            return new Tensor(this.value);
+            return this;
         if (typeof dims === "number") {
             dims = [dims];
         }
@@ -300,10 +300,9 @@ class Tensor {
     }
     // Tensor unsqueeze - adds dimension of size 1 at specified position
     unsqueeze(dim) {
-        if (typeof this.value === "number")
-            return new Tensor([this.value]);
-        if (dim < 0 || dim > this.shape.length) {
-            throw new Error(`Invalid dimension ${dim} for unsqueeze`);
+        let thisValue = this.value;
+        if (typeof thisValue === "number") {
+            thisValue = [thisValue];
         }
         // Insert size-1 dimension at specified position
         const newShape = [...this.shape];
@@ -320,7 +319,7 @@ class Tensor {
             newDimStride = this.strides[dim] * this.shape[dim];
         }
         newStrides.splice(dim, 0, newDimStride);
-        const out = new Tensor(this.value, { shape: newShape, strides: newStrides });
+        const out = new Tensor(thisValue, { shape: newShape, strides: newStrides });
         // Set up gradient if needed
         if (this.requiresGrad) {
             out.requiresGrad = true;
@@ -334,7 +333,7 @@ class Tensor {
     // Tensor sum reduction
     sum(dims, keepDims = false) {
         if (typeof this.value === "number")
-            return new Tensor(this.value);
+            return this;
         if (typeof dims === "number") {
             dims = [dims];
         }
@@ -364,6 +363,7 @@ class Tensor {
             const outFlatIndex = Tensor.coordsToIndex(outCoords, outputStrides);
             // Accumulate, outFlatIndex should match multiple realFlatIndexes
             const realFlatIndex = Tensor.coordsToIndex(coords, this.strides);
+            // Add into sum
             outputValue[outFlatIndex] += this.value[realFlatIndex];
             // Mark for gradient if needed
             if (this.requiresGrad) {
@@ -388,7 +388,7 @@ class Tensor {
     // Tensor product reduction
     prod(dims, keepDims = false) {
         if (typeof this.value === "number")
-            return new Tensor(this.value);
+            return this;
         if (typeof dims === "number") {
             dims = [dims];
         }
@@ -401,14 +401,6 @@ class Tensor {
         const outputSize = Tensor.shapeToSize(outputShape);
         const outputValue = new Array(outputSize).fill(1);
         const originalSize = Tensor.shapeToSize(this.shape);
-        // Gradient data
-        let gradShape, gradStrides, gradValue = [];
-        // Allocate gradient data only when needed
-        if (this.requiresGrad) {
-            gradShape = this.shape;
-            gradStrides = this.strides;
-            gradValue = new Array(originalSize).fill(0);
-        }
         // Calculate new value after multiplying
         for (let index = 0; index < originalSize; index++) {
             const coords = Tensor.indexToCoords(index, this.strides);
@@ -418,6 +410,7 @@ class Tensor {
             const outFlatIndex = Tensor.coordsToIndex(outCoords, outputStrides);
             // Accumulate, outFlatIndex should match multiple realFlatIndexes
             const realFlatIndex = Tensor.coordsToIndex(coords, this.strides);
+            // Multiply into product
             outputValue[outFlatIndex] *= this.value[realFlatIndex];
         }
         const out = new Tensor(outputValue, {
@@ -426,7 +419,7 @@ class Tensor {
         });
         // Set up gradient if needed
         if (this.requiresGrad) {
-            // Grad is the product of other elements of the same axis, which is product of all els divided by the current value
+            const gradShape = this.shape, gradStrides = this.strides, gradValue = new Array(originalSize).fill(0);
             for (let index = 0; index < originalSize; index++) {
                 const coords = Tensor.indexToCoords(index, this.strides);
                 // Force 0 on reduced axes to collapse into size-1 dims
@@ -435,7 +428,7 @@ class Tensor {
                 const outFlatIndex = Tensor.coordsToIndex(outCoords, outputStrides);
                 // Accumulate, outFlatIndex should match multiple realFlatIndexes
                 const realFlatIndex = Tensor.coordsToIndex(coords, this.strides);
-                // Calculate gradient at position
+                // Grad is the product of other elements of the same axis, which is product of all els divided by the current value
                 gradValue[realFlatIndex] = outputValue[outFlatIndex] / this.value[realFlatIndex];
             }
             out.requiresGrad = true;
@@ -450,7 +443,7 @@ class Tensor {
     // Tensor mean reduction
     mean(dims, keepDims = false) {
         if (typeof this.value === "number")
-            return new Tensor(this.value);
+            return this;
         if (typeof dims === "number") {
             dims = [dims];
         }
@@ -464,14 +457,6 @@ class Tensor {
         const outputValue = new Array(outputSize).fill(0);
         const outputFeeders = new Array(outputSize).fill(0);
         const originalSize = Tensor.shapeToSize(this.shape);
-        // Gradient data
-        let gradShape, gradStrides, gradValue = [];
-        // Allocate gradient data only when needed
-        if (this.requiresGrad) {
-            gradShape = this.shape;
-            gradStrides = this.strides;
-            gradValue = new Array(originalSize).fill(0);
-        }
         // Calculate sums and how many elements contribute to specific positions
         for (let index = 0; index < originalSize; index++) {
             const coords = Tensor.indexToCoords(index, this.strides);
@@ -481,6 +466,7 @@ class Tensor {
             const outFlatIndex = Tensor.coordsToIndex(outCoords, outputStrides);
             // Accumulate, outFlatIndex should match multiple realFlatIndexes
             const realFlatIndex = Tensor.coordsToIndex(coords, this.strides);
+            // Calculate sum and contributors to the sum
             outputValue[outFlatIndex] += this.value[realFlatIndex];
             outputFeeders[outFlatIndex]++;
         }
@@ -494,7 +480,8 @@ class Tensor {
         });
         // Set up gradient if needed
         if (this.requiresGrad) {
-            // Calculate grad by assiging 1 divide by the number of contributors to the position
+            const gradShape = this.shape, gradStrides = this.strides, gradValue = new Array(originalSize).fill(0);
+            // Calculate grad by assigning 1 divided by the number of contributors to the position
             for (let index = 0; index < originalSize; index++) {
                 const coords = Tensor.indexToCoords(index, this.strides);
                 // Force 0 on reduced axes to collapse into size-1 dims
@@ -515,6 +502,120 @@ class Tensor {
         }
         return keepDims ? out : out.squeeze(dims);
     }
+    // Tensor maximum reduction
+    max(dims, keepDims = false) {
+        if (typeof this.value === "number")
+            return this;
+        if (typeof dims === "number") {
+            dims = [dims];
+        }
+        if (typeof dims === "undefined") {
+            dims = Array.from({ length: this.shape.length }, (_, index) => index);
+        }
+        // Dims that are reduced now have size-1
+        const outputShape = this.shape.map((dim, i) => dims.includes(i) ? 1 : dim);
+        const outputStrides = Tensor.getStrides(outputShape);
+        const outputSize = Tensor.shapeToSize(outputShape);
+        const outputValue = new Array(outputSize).fill(-Infinity);
+        const originalSize = Tensor.shapeToSize(this.shape);
+        // Calculate maximum values of axes
+        for (let index = 0; index < originalSize; index++) {
+            const coords = Tensor.indexToCoords(index, this.strides);
+            // Force 0 on reduced axes to collapse into size-1 dims
+            const outCoords = coords.map((val, i) => dims.includes(i) ? 0 : val);
+            // Convert output coordinates to flat index
+            const outFlatIndex = Tensor.coordsToIndex(outCoords, outputStrides);
+            // Accumulate, outFlatIndex should match multiple realFlatIndexes
+            const realFlatIndex = Tensor.coordsToIndex(coords, this.strides);
+            // Get max over time
+            if (this.value[realFlatIndex] > outputValue[outFlatIndex]) {
+                outputValue[outFlatIndex] = this.value[realFlatIndex];
+            }
+        }
+        const out = new Tensor(outputValue, {
+            shape: outputShape,
+            strides: outputStrides
+        });
+        // Set up gradient if needed
+        if (this.requiresGrad) {
+            const gradShape = this.shape, gradStrides = this.strides, gradValue = new Array(originalSize).fill(0);
+            for (let index = 0; index < originalSize; index++) {
+                const coords = Tensor.indexToCoords(index, this.strides);
+                // Force 0 on reduced axes to collapse into size-1 dims
+                const outCoords = coords.map((val, i) => dims.includes(i) ? 0 : val);
+                // Convert output coordinates to flat index
+                const outFlatIndex = Tensor.coordsToIndex(outCoords, outputStrides);
+                // Accumulate, outFlatIndex should match multiple realFlatIndexes
+                const realFlatIndex = Tensor.coordsToIndex(coords, this.strides);
+                // Calculate grad by checking if a positon holds a value equal to the max value
+                gradValue[realFlatIndex] = outputValue[outFlatIndex] === this.value[realFlatIndex] ? 1 : 0;
+            }
+            out.requiresGrad = true;
+            out.children.push(this);
+            out.gradFn = () => {
+                const localGrad = new Tensor(gradValue, { shape: gradShape, strides: gradStrides });
+                Tensor.addGrad(this, out.grad.withGrad(false).mul(localGrad));
+            };
+        }
+        return keepDims ? out : out.squeeze(dims);
+    }
+    // Tensor minimum reduction
+    min(dims, keepDims = false) {
+        if (typeof this.value === "number")
+            return this;
+        if (typeof dims === "number") {
+            dims = [dims];
+        }
+        if (typeof dims === "undefined") {
+            dims = Array.from({ length: this.shape.length }, (_, index) => index);
+        }
+        // Dims that are reduced now have size-1
+        const outputShape = this.shape.map((dim, i) => dims.includes(i) ? 1 : dim);
+        const outputStrides = Tensor.getStrides(outputShape);
+        const outputSize = Tensor.shapeToSize(outputShape);
+        const outputValue = new Array(outputSize).fill(Infinity);
+        const originalSize = Tensor.shapeToSize(this.shape);
+        // Calculate minimum values of axes
+        for (let index = 0; index < originalSize; index++) {
+            const coords = Tensor.indexToCoords(index, this.strides);
+            // Force 0 on reduced axes to collapse into size-1 dims
+            const outCoords = coords.map((val, i) => dims.includes(i) ? 0 : val);
+            // Convert output coordinates to flat index
+            const outFlatIndex = Tensor.coordsToIndex(outCoords, outputStrides);
+            // Accumulate, outFlatIndex should match multiple realFlatIndexes
+            const realFlatIndex = Tensor.coordsToIndex(coords, this.strides);
+            // Get min over time
+            if (this.value[realFlatIndex] < outputValue[outFlatIndex]) {
+                outputValue[outFlatIndex] = this.value[realFlatIndex];
+            }
+        }
+        const out = new Tensor(outputValue, {
+            shape: outputShape,
+            strides: outputStrides
+        });
+        // Set up gradient if needed
+        if (this.requiresGrad) {
+            const gradShape = this.shape, gradStrides = this.strides, gradValue = new Array(originalSize).fill(0);
+            for (let index = 0; index < originalSize; index++) {
+                const coords = Tensor.indexToCoords(index, this.strides);
+                // Force 0 on reduced axes to collapse into size-1 dims
+                const outCoords = coords.map((val, i) => dims.includes(i) ? 0 : val);
+                // Convert output coordinates to flat index
+                const outFlatIndex = Tensor.coordsToIndex(outCoords, outputStrides);
+                // Accumulate, outFlatIndex should match multiple realFlatIndexes
+                const realFlatIndex = Tensor.coordsToIndex(coords, this.strides);
+                // Calculate grad by checking if a positon holds a value equal to the min value
+                gradValue[realFlatIndex] = outputValue[outFlatIndex] === this.value[realFlatIndex] ? 1 : 0;
+            }
+            out.requiresGrad = true;
+            out.children.push(this);
+            out.gradFn = () => {
+                const localGrad = new Tensor(gradValue, { shape: gradShape, strides: gradStrides });
+                Tensor.addGrad(this, out.grad.withGrad(false).mul(localGrad));
+            };
+        }
+        return keepDims ? out : out.squeeze(dims);
+    }
     // Tensor element-wise addition
     add(other) {
         return this.elementWiseABDAG(other, (a, b) => a + b, (self, other, outGrad) => outGrad, (self, other, outGrad) => outGrad);
@@ -1000,24 +1101,139 @@ class Tensor {
         // Too lazy for batched matmul
         throw new Error(`Shapes [${this.shape}] and [${other.shape}] are not supported`);
     }
+    // Utility to create a new tensor filled with a number
+    static full(shape, num, options = {}) {
+        if (shape.length === 0)
+            return new Tensor(num, options);
+        const outputSize = Tensor.shapeToSize(shape);
+        const outputValue = new Array(outputSize).fill(num);
+        return new Tensor(outputValue, { shape, ...options });
+    }
     // Utility to create a new tensor with shape of another tensor, filled with a number
     static fullLike(tensor, num, options = {}) {
         if (typeof tensor.value === "number")
             return new Tensor(num, options);
         return new Tensor(new Array(tensor.value.length).fill(num), { shape: tensor.shape, strides: tensor.strides, ...options });
     }
+    // Utility to create a new tensor filled with 1
+    static ones(shape, options = {}) {
+        if (typeof shape === "undefined" || shape.length === 0)
+            return new Tensor(1, options);
+        const outputSize = Tensor.shapeToSize(shape);
+        const outputValue = new Array(outputSize).fill(1);
+        return new Tensor(outputValue, { shape, ...options });
+    }
     // Utility to create a new tensor with shape of another tensor, filled with 1
     static onesLike(tensor, options = {}) {
         if (typeof tensor.value === "number")
             return new Tensor(1, options);
         return new Tensor(new Array(tensor.value.length).fill(1), { shape: tensor.shape, strides: tensor.strides, ...options });
     }
+    // Utility to create a new tensor filled with 0
+    static zeros(shape, options = {}) {
+        if (typeof shape === "undefined" || shape.length === 0)
+            return new Tensor(0, options);
+        const outputSize = Tensor.shapeToSize(shape);
+        const outputValue = new Array(outputSize).fill(0);
+        return new Tensor(outputValue, { shape, ...options });
+    }
     // Utility to create a new tensor with shape of another tensor, filled with 0
     static zerosLike(tensor, options = {}) {
         if (typeof tensor.value === "number")
             return new Tensor(0, options);
         return new Tensor(new Array(tensor.value.length).fill(0), { shape: tensor.shape, strides: tensor.strides, ...options });
     }
+    // Utility to create a new tensor filled with a random number with uniform distribution from 0 to 1
+    static rand(shape, options = {}) {
+        if (typeof shape === "undefined" || shape.length === 0)
+            return new Tensor((0, utils_1.randUniform)(), options);
+        const outputSize = Tensor.shapeToSize(shape);
+        const outputValue = new Array(outputSize);
+        for (let index = 0; index < outputValue.length; index++) {
+            outputValue[index] = (0, utils_1.randUniform)();
+        }
+        return new Tensor(outputValue, { shape, ...options });
+    }
+    // Utility to create a new tensor with shape of another tensor, filled with a random number with uniform distribution from 0 to 1
+    static randLike(tensor, options = {}) {
+        if (typeof tensor.value === "number")
+            return new Tensor((0, utils_1.randUniform)(), options);
+        const outputValue = new Array(tensor.value.length);
+        for (let index = 0; index < outputValue.length; index++) {
+            outputValue[index] = (0, utils_1.randUniform)();
+        }
+        return new Tensor(outputValue, {
+            shape: tensor.shape, strides: tensor.strides, ...options
+        });
+    }
+    // Utility to create a new tensor filled with a random number with normal distribution of mean=0 and stddev=1
+    static randn(shape, options = {}) {
+        if (typeof shape === "undefined" || shape.length === 0)
+            return new Tensor((0, utils_1.randNormal)(), options);
+        const outputSize = Tensor.shapeToSize(shape);
+        const outputValue = new Array(outputSize);
+        for (let index = 0; index < outputValue.length; index++) {
+            outputValue[index] = (0, utils_1.randNormal)();
+        }
+        return new Tensor(outputValue, { shape, ...options });
+    }
+    // Utility to create a new tensor with shape of another tensor, filled with a random number with normal distribution of mean=0 and stddev=1
+    static randnLike(tensor, options = {}) {
+        if (typeof tensor.value === "number")
+            return new Tensor((0, utils_1.randNormal)(), options);
+        const outputValue = new Array(tensor.value.length);
+        for (let index = 0; index < outputValue.length; index++) {
+            outputValue[index] = (0, utils_1.randNormal)();
+        }
+        return new Tensor(outputValue, {
+            shape: tensor.shape, strides: tensor.strides, ...options
+        });
+    }
+    // Utility to create a new tensor filled with a random integer between low and high
+    static randint(shape, low, high, options = {}) {
+        if (shape.length === 0)
+            return new Tensor((0, utils_1.randInt)(low, high), options);
+        const outputSize = Tensor.shapeToSize(shape);
+        const outputValue = new Array(outputSize);
+        for (let index = 0; index < outputValue.length; index++) {
+            outputValue[index] = (0, utils_1.randInt)(low, high);
+        }
+        return new Tensor(outputValue, { shape, ...options });
+    }
+    // Utility to create a new tensor with shape of another tensor, filled with a random integer between low and high
+    static randintLike(tensor, low, high, options = {}) {
+        if (typeof tensor.value === "number")
+            return new Tensor((0, utils_1.randInt)(low, high), options);
+        const outputValue = new Array(tensor.value.length);
+        for (let index = 0; index < outputValue.length; index++) {
+            outputValue[index] = (0, utils_1.randInt)(low, high);
+        }
+        return new Tensor(outputValue, {
+            shape: tensor.shape, strides: tensor.strides, ...options
+        });
+    }
+    // Utility to create a new tensor filled with a random number with normal distribution of custom mean and stddev
+    static normal(shape, mean, stdDev, options = {}) {
+        if (shape.length === 0)
+            return new Tensor((0, utils_1.randNormal)(mean, stdDev), options);
+        const outputSize = Tensor.shapeToSize(shape);
+        const outputValue = new Array(outputSize);
+        for (let index = 0; index < outputValue.length; index++) {
+            outputValue[index] = (0, utils_1.randNormal)(mean, stdDev);
+        }
+        return new Tensor(outputValue, { shape, ...options });
+    }
+    // Utility to create a new tensor filled with a random number with uniform distribution from low to high
+    static uniform(shape, low, high, options = {}) {
+        if (shape.length === 0)
+            return new Tensor((0, utils_1.randUniform)(low, high), options);
+        const outputSize = Tensor.shapeToSize(shape);
+        const outputValue = new Array(outputSize);
+        for (let index = 0; index < outputValue.length; index++) {
+            outputValue[index] = (0, utils_1.randUniform)(low, high);
+        }
+        return new Tensor(outputValue, { shape, ...options });
+    }
     // Reverse-mode autodiff call
     backward() {
         // Build topological order

package/dist/utils.d.ts

@@ -1,3 +1,6 @@
 export declare function erf(x: number): number;
 export declare function erfc(x: number): number;
 export declare function erfinv(x: number): number;
+export declare function randUniform(low?: number, high?: number): number;
+export declare function randNormal(mean?: number, stdDev?: number): number;
+export declare function randInt(low: number, high: number): number;

package/dist/utils.js

@@ -3,6 +3,9 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.erf = erf;
 exports.erfc = erfc;
 exports.erfinv = erfinv;
+exports.randUniform = randUniform;
+exports.randNormal = randNormal;
+exports.randInt = randInt;
 // Error function using Abramowitz and Stegun approximation
 function erf(x) {
     const a1 = 0.254829592;
@@ -33,3 +36,15 @@ function erfinv(x) {
     const sign = x >= 0 ? 1 : -1;
     return sign * Math.sqrt(-part1 + Math.sqrt(part1 * part1 - part2));
 }
+function randUniform(low = 0, high = 1) {
+    return Math.random() * (high - low) + low;
+}
+function randNormal(mean = 0, stdDev = 1) {
+    const u = 1 - Math.random();
+    const v = 1 - Math.random();
+    const z = Math.sqrt(-2.0 * Math.log(u)) * Math.cos(2.0 * Math.PI * v);
+    return z * stdDev + mean;
+}
+function randInt(low, high) {
+    return Math.floor(Math.random() * (high - low) + low);
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "catniff",
-  "version": "0.2.11",
+  "version": "0.2.13",
   "description": "A cute autograd engine for Javascript",
   "main": "index.js",
   "scripts": {