catniff 0.2.4 → 0.2.6

package/dist/core.d.ts

@@ -1,7 +1,7 @@
 export type TensorValue = number | TensorValue[];
 export interface TensorOptions {
-    shape?: number[];
-    strides?: number[];
+    shape?: readonly number[];
+    strides?: readonly number[];
     grad?: Tensor;
     requiresGrad?: boolean;
     gradFn?: Function;
@@ -9,20 +9,27 @@ export interface TensorOptions {
 }
 export declare class Tensor {
     value: number[] | number;
-    readonly shape: number[];
-    readonly strides: number[];
+    readonly shape: readonly number[];
+    readonly strides: readonly number[];
     grad?: Tensor;
     requiresGrad: boolean;
     gradFn: Function;
     children: Tensor[];
     constructor(value: TensorValue, options?: TensorOptions);
     static flatten(tensor: TensorValue): number[] | number;
-    static getShape(tensor: TensorValue): number[];
-    static getStrides(shape: number[]): number[];
-    static padShape(stridesA: number[], stridesB: number[], shapeA: number[], shapeB: number[]): number[][];
-    static broadcastShapes(shapeA: number[], shapeB: number[]): number[];
-    static indexToCoords(index: number, shape: number[], strides: number[]): number[];
-    static coordsToIndex(coords: number[], shape: number[], strides: number[]): number;
+    static getShape(tensor: TensorValue): readonly number[];
+    static getStrides(shape: readonly number[]): readonly number[];
+    static padShape(stridesA: readonly number[], stridesB: readonly number[], shapeA: readonly number[], shapeB: readonly number[]): [
+        readonly number[],
+        readonly number[],
+        readonly number[],
+        readonly number[]
+    ];
+    static broadcastShapes(shapeA: readonly number[], shapeB: readonly number[]): readonly number[];
+    static indexToCoords(index: number, shape: readonly number[], strides: readonly number[]): number[];
+    static coordsToUnbroadcastedIndex(coords: number[], shape: readonly number[], strides: readonly number[]): number;
+    static coordsToIndex(coords: number[], strides: readonly number[]): number;
+    static shapeToSize(shape: readonly number[]): number;
     static elementWiseAB(tA: Tensor, tB: Tensor, op: (tA: number, tB: number) => number): Tensor;
     static elementWiseSelf(tA: Tensor, op: (tA: number) => number): Tensor;
     elementWiseABDAG(other: TensorValue | Tensor, op: (a: number, b: number) => number, thisGrad?: (self: Tensor, other: Tensor, outGrad: Tensor) => Tensor, otherGrad?: (self: Tensor, other: Tensor, outGrad: Tensor) => Tensor): Tensor;
@@ -32,6 +39,8 @@ export declare class Tensor {
     squeeze(dims?: number[] | number): Tensor;
     unsqueeze(dim: number): Tensor;
     sum(dims?: number[] | number, keepDims?: boolean): Tensor;
+    prod(dims?: number[] | number, keepDims?: boolean): Tensor;
+    mean(dims?: number[] | number, keepDims?: boolean): Tensor;
     add(other: TensorValue | Tensor): Tensor;
     sub(other: TensorValue | Tensor): Tensor;
     mul(other: TensorValue | Tensor): Tensor;

package/dist/core.js

@@ -93,7 +93,7 @@ class Tensor {
         }
         return newShape;
     }
-    // Convert flat index to array of coordinates
+    // Utility to convert flat index to array of coordinates
     static indexToCoords(index, shape, strides) {
         const coords = new Array(shape.length);
         let remaining = index;
@@ -105,17 +105,33 @@ class Tensor {
         }
         return coords;
     }
-    // Convert array of coordinates to *unbroadcasted* flat index 
-    static coordsToIndex(coords, shape, strides) {
+    // Utility to convert array of coordinates to *unbroadcasted* flat index 
+    static coordsToUnbroadcastedIndex(coords, shape, strides) {
         let index = 0;
         for (let i = 0; i < coords.length; i++) {
-            const coord = coords[i];
             // Handle broadcasting
-            const actualCoord = shape[i] === 1 ? 0 : coord;
+            const actualCoord = shape[i] === 1 ? 0 : coords[i];
             index += actualCoord * strides[i];
         }
         return index;
     }
+    // Utility to convert array of coordinates to flat index 
+    static coordsToIndex(coords, strides) {
+        let index = 0;
+        for (let i = 0; i < coords.length; i++) {
+            index += coords[i] * strides[i];
+        }
+        return index;
+    }
+    // Utility to convert shape into 1D value array size
+    static shapeToSize(shape) {
+        let prod = 1;
+        for (let i = 0; i < shape.length; i++) {
+            prod *= shape[i];
+        }
+        return prod;
+    }
+    ;
     // Utility for binary (two operators involved) element-wise ops
     static elementWiseAB(tA, tB, op) {
         if (typeof tA.value === "number" && typeof tB.value === "number") {
@@ -132,15 +148,15 @@ class Tensor {
         const outputShape = Tensor.broadcastShapes(paddedAShape, paddedBShape);
         // Get other output info
         const outputStrides = Tensor.getStrides(outputShape);
-        const outputSize = outputShape.reduce((a, b) => a * b, 1);
+        const outputSize = Tensor.shapeToSize(outputShape);
         const outputValue = new Array(outputSize);
         for (let i = 0; i < outputSize; i++) {
             // Get coordinates from 1D index
             const coordsOutput = Tensor.indexToCoords(i, outputShape, outputStrides);
             // Convert the coordinates to 1D index of flattened A with respect to A's shape
-            const indexA = Tensor.coordsToIndex(coordsOutput, paddedAShape, paddedAStrides);
+            const indexA = Tensor.coordsToUnbroadcastedIndex(coordsOutput, paddedAShape, paddedAStrides);
             // Convert the coordinates to 1D index of flattened B with respect to B's shape
-            const indexB = Tensor.coordsToIndex(coordsOutput, paddedBShape, paddedBStrides);
+            const indexB = Tensor.coordsToUnbroadcastedIndex(coordsOutput, paddedBShape, paddedBStrides);
             // Calculate with op
             outputValue[i] = op(tA.value[indexA], tB.value[indexB]);
         }
@@ -321,29 +337,33 @@ class Tensor {
         if (typeof dims === "undefined") {
             dims = Array.from({ length: this.shape.length }, (_, index) => index);
         }
+        // Dims that are reduced now have size-1
         const outputShape = this.shape.map((dim, i) => dims.includes(i) ? 1 : dim);
         const outputStrides = Tensor.getStrides(outputShape);
-        const outputSize = outputShape.reduce((a, b) => a * b, 1);
+        const outputSize = Tensor.shapeToSize(outputShape);
         const outputValue = new Array(outputSize).fill(0);
-        const originalSize = this.shape.reduce((a, b) => a * b, 1);
+        const originalSize = Tensor.shapeToSize(this.shape);
+        // Gradient data
         let gradShape, gradStrides, gradValue = [];
+        // Allocate gradient data only when needed
         if (this.requiresGrad) {
             gradShape = this.shape;
             gradStrides = this.strides;
             gradValue = new Array(originalSize).fill(0);
         }
+        // Calculate new value after sum
         for (let index = 0; index < originalSize; index++) {
             const coords = Tensor.indexToCoords(index, this.shape, this.strides);
             // Force 0 on reduced axes to collapse into size-1 dims
             const outCoords = coords.map((val, i) => dims.includes(i) ? 0 : val);
             // Convert output coordinates to flat index
-            const outFlatIndex = outCoords.reduce((acc, val, i) => acc + val * outputStrides[i], 0);
-            // Accumulate
-            const realFlatIndex = coords.reduce((acc, val, i) => acc + val * this.strides[i], 0);
+            const outFlatIndex = Tensor.coordsToIndex(outCoords, outputStrides);
+            // Accumulate, outFlatIndex should match multiple realFlatIndexes
+            const realFlatIndex = Tensor.coordsToIndex(coords, this.strides);
             outputValue[outFlatIndex] += this.value[realFlatIndex];
-            // Mark for gradient
+            // Mark for gradient if needed
             if (this.requiresGrad) {
-                (gradValue)[realFlatIndex] = 1;
+                gradValue[realFlatIndex] = 1;
             }
         }
         const out = new Tensor(outputValue, {
@@ -361,6 +381,136 @@ class Tensor {
         }
         return keepDims ? out : out.squeeze(dims);
     }
+    // Tensor product reduction
+    prod(dims, keepDims = false) {
+        if (typeof this.value === "number")
+            return new Tensor(this.value);
+        if (typeof dims === "number") {
+            dims = [dims];
+        }
+        if (typeof dims === "undefined") {
+            dims = Array.from({ length: this.shape.length }, (_, index) => index);
+        }
+        // Dims that are reduced now have size-1
+        const outputShape = this.shape.map((dim, i) => dims.includes(i) ? 1 : dim);
+        const outputStrides = Tensor.getStrides(outputShape);
+        const outputSize = Tensor.shapeToSize(outputShape);
+        const outputValue = new Array(outputSize).fill(1);
+        const originalSize = Tensor.shapeToSize(this.shape);
+        // Gradient data
+        let gradShape, gradStrides, gradValue = [];
+        // Allocate gradient data only when needed
+        if (this.requiresGrad) {
+            gradShape = this.shape;
+            gradStrides = this.strides;
+            gradValue = new Array(originalSize).fill(0);
+        }
+        // Calculate new value after multiplying
+        for (let index = 0; index < originalSize; index++) {
+            const coords = Tensor.indexToCoords(index, this.shape, this.strides);
+            // Force 0 on reduced axes to collapse into size-1 dims
+            const outCoords = coords.map((val, i) => dims.includes(i) ? 0 : val);
+            // Convert output coordinates to flat index
+            const outFlatIndex = Tensor.coordsToIndex(outCoords, outputStrides);
+            // Accumulate, outFlatIndex should match multiple realFlatIndexes
+            const realFlatIndex = Tensor.coordsToIndex(coords, this.strides);
+            outputValue[outFlatIndex] *= this.value[realFlatIndex];
+        }
+        const out = new Tensor(outputValue, {
+            shape: outputShape,
+            strides: outputStrides
+        });
+        // Set up gradient if needed
+        if (this.requiresGrad) {
+            // Grad is the product of other elements of the same axis, which is product of all els divided by the current value
+            for (let index = 0; index < originalSize; index++) {
+                const coords = Tensor.indexToCoords(index, this.shape, this.strides);
+                // Force 0 on reduced axes to collapse into size-1 dims
+                const outCoords = coords.map((val, i) => dims.includes(i) ? 0 : val);
+                // Convert output coordinates to flat index
+                const outFlatIndex = Tensor.coordsToIndex(outCoords, outputStrides);
+                // Accumulate, outFlatIndex should match multiple realFlatIndexes
+                const realFlatIndex = Tensor.coordsToIndex(coords, this.strides);
+                // Calculate gradient at position
+                gradValue[realFlatIndex] = outputValue[outFlatIndex] / this.value[realFlatIndex];
+            }
+            out.requiresGrad = true;
+            out.children.push(this);
+            out.gradFn = () => {
+                const localGrad = new Tensor(gradValue, { shape: gradShape, strides: gradStrides });
+                Tensor.addGrad(this, out.grad.withGrad(false).mul(localGrad));
+            };
+        }
+        return keepDims ? out : out.squeeze(dims);
+    }
+    // Tensor mean reduction
+    mean(dims, keepDims = false) {
+        if (typeof this.value === "number")
+            return new Tensor(this.value);
+        if (typeof dims === "number") {
+            dims = [dims];
+        }
+        if (typeof dims === "undefined") {
+            dims = Array.from({ length: this.shape.length }, (_, index) => index);
+        }
+        // Dims that are reduced now have size-1
+        const outputShape = this.shape.map((dim, i) => dims.includes(i) ? 1 : dim);
+        const outputStrides = Tensor.getStrides(outputShape);
+        const outputSize = Tensor.shapeToSize(outputShape);
+        const outputValue = new Array(outputSize).fill(0);
+        const outputFeeders = new Array(outputSize).fill(0);
+        const originalSize = Tensor.shapeToSize(this.shape);
+        // Gradient data
+        let gradShape, gradStrides, gradValue = [];
+        // Allocate gradient data only when needed
+        if (this.requiresGrad) {
+            gradShape = this.shape;
+            gradStrides = this.strides;
+            gradValue = new Array(originalSize).fill(0);
+        }
+        // Calculate sums and how many elements contribute to specific positions
+        for (let index = 0; index < originalSize; index++) {
+            const coords = Tensor.indexToCoords(index, this.shape, this.strides);
+            // Force 0 on reduced axes to collapse into size-1 dims
+            const outCoords = coords.map((val, i) => dims.includes(i) ? 0 : val);
+            // Convert output coordinates to flat index
+            const outFlatIndex = Tensor.coordsToIndex(outCoords, outputStrides);
+            // Accumulate, outFlatIndex should match multiple realFlatIndexes
+            const realFlatIndex = Tensor.coordsToIndex(coords, this.strides);
+            outputValue[outFlatIndex] += this.value[realFlatIndex];
+            outputFeeders[outFlatIndex]++;
+        }
+        // Calculate mean by dividing sum by the number of contributors to the position
+        for (let index = 0; index < outputSize; index++) {
+            outputValue[index] /= outputFeeders[index];
+        }
+        const out = new Tensor(outputValue, {
+            shape: outputShape,
+            strides: outputStrides
+        });
+        // Set up gradient if needed
+        if (this.requiresGrad) {
+            // Calculate grad by assiging 1 divide by the number of contributors to the position
+            for (let index = 0; index < originalSize; index++) {
+                const coords = Tensor.indexToCoords(index, this.shape, this.strides);
+                // Force 0 on reduced axes to collapse into size-1 dims
+                const outCoords = coords.map((val, i) => dims.includes(i) ? 0 : val);
+                // Convert output coordinates to flat index
+                const outFlatIndex = Tensor.coordsToIndex(outCoords, outputStrides);
+                // Accumulate, outFlatIndex should match multiple realFlatIndexes
+                const realFlatIndex = Tensor.coordsToIndex(coords, this.strides);
+                // Mean = 1/n * (el1 + el2 + ... + eln) so grad = 1/n
+                gradValue[realFlatIndex] = 1 / outputFeeders[outFlatIndex];
+            }
+            out.requiresGrad = true;
+            out.children.push(this);
+            out.gradFn = () => {
+                const localGrad = new Tensor(gradValue, { shape: gradShape, strides: gradStrides });
+                Tensor.addGrad(this, out.grad.withGrad(false).mul(localGrad));
+            };
+        }
+        return keepDims ? out : out.squeeze(dims);
+    }
     // Tensor element-wise addition
     add(other) {
         return this.elementWiseABDAG(other, (a, b) => a + b, (self, other, outGrad) => outGrad, (self, other, outGrad) => outGrad);
@@ -578,6 +728,7 @@ class Tensor {
         if (this.shape.length !== 1 || other.shape.length !== 1) {
             throw new Error("Inputs are not 1D tensors");
         }
+        // Simple vector dot product
         const vectLen = this.shape[0];
         const vectA = this.value;
         const vectB = other.value;
@@ -615,6 +766,7 @@ class Tensor {
         if (this.shape.length !== 2 || other.shape.length !== 2) {
             throw new Error("Inputs are not matrices");
         }
+        // Simple matrix multiplication
         const matA = this.value;
         const matB = other.value;
         const matAStrides = this.strides;
@@ -627,11 +779,12 @@ class Tensor {
             throw new Error("Invalid matrices shape for multiplication");
         const matCShape = [matARows, matBCols];
         const matCStrides = Tensor.getStrides(matCShape);
-        const matCSize = matCShape.reduce((a, b) => a * b, 1);
+        const matCSize = Tensor.shapeToSize(matCShape);
         const matC = new Array(matCSize).fill(0);
         for (let i = 0; i < matARows; i++) {
             for (let j = 0; j < matBCols; j++) {
                 for (let k = 0; k < matACols; k++) {
+                    // Tensor values are 1D arrays so we have to get real index using strides
                     matC[i * matCStrides[0] + j * matCStrides[1]] +=
                         matA[i * matAStrides[0] + k * matAStrides[1]] *
                             matB[k * matBStrides[0] + j * matBStrides[1]];

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "catniff",
-  "version": "0.2.4",
+  "version": "0.2.6",
   "description": "A cute autograd engine for Javascript",
   "main": "index.js",
   "scripts": {