catniff 0.1.3 → 0.1.5

package/README.md

@@ -26,7 +26,7 @@ console.log(x.grad); // 5
 
 Tensors in Catniff are either numbers (scalars/0-D tensors) or multidimensional number arrays (n-D tensors).
 
-There is a built-in `TensorMath` class to help with Tensor arithmetic, for example:
+There is a built-in `TensorMath` class to help with tensor arithmetic, for example:
 ```js
 const { TensorMath } = require("catniff");
 
@@ -69,6 +69,9 @@ All available APIs are in `./src/autograd.ts`.
 
 I'm mostly just learning and playing with this currently, so there are no concrete plans yet, but here are what I currently have in mind:
 
+* Fix whatever is the problem right now (there are a lot of problems right now lol).
+* Add more tensor ops.
+* Proper documentation.
 * GPU acceleration.
 * Some general neural net APIs.
 

package/dist/autograd.d.ts

@@ -11,7 +11,9 @@ export declare enum OP {
     LOG = 8,
     RELU = 9,
     SIGMOID = 10,
-    TANH = 11
+    TANH = 11,
+    T = 12,
+    MM = 13
 }
 export declare class Node {
     value: Tensor;
@@ -32,6 +34,9 @@ export declare class Node {
     relu(): Node;
     sigmoid(): Node;
     tanh(): Node;
+    t(): Node;
+    mm(other: Node | number): Node;
     backward(): void;
-    forceNode(value: Node | number): Node;
+    static forceNode(value: Node | number): Node;
+    static addGrad(node: Node, accumGrad: Tensor): void;
 }

package/dist/autograd.js

@@ -2,7 +2,7 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.Node = exports.OP = void 0;
 const tensor_1 = require("./tensor");
-const { add, sub, mul, pow, div, neg, exp, log, relu, sigmoid, tanh, ge } = tensor_1.TensorMath;
+const { add, sub, mul, pow, div, neg, exp, log, relu, sigmoid, tanh, ge, t, mm } = tensor_1.TensorMath;
 var OP;
 (function (OP) {
     OP[OP["NONE"] = 0] = "NONE";
@@ -17,6 +17,8 @@ var OP;
     OP[OP["RELU"] = 9] = "RELU";
     OP[OP["SIGMOID"] = 10] = "SIGMOID";
     OP[OP["TANH"] = 11] = "TANH";
+    OP[OP["T"] = 12] = "T";
+    OP[OP["MM"] = 13] = "MM";
 })(OP || (exports.OP = OP = {}));
 class Node {
     value;
@@ -34,35 +36,35 @@ class Node {
         this.feedBackward = () => { };
     }
     add(other) {
-        other = this.forceNode(other);
+        other = Node.forceNode(other);
         const out = new Node(add(this.value, other.value), [this, other], OP.ADD);
         out.feedBackward = () => {
             // x + y d/dx = 1, note that we apply the chain rule continuously so out.grad is multiplied into our derivative
-            this.grad = add(this.grad, out.grad);
+            Node.addGrad(this, out.grad);
             // x + y d/dy = 1
-            other.grad = add(other.grad, out.grad);
+            Node.addGrad(other, out.grad);
         };
         return out;
     }
     sub(other) {
-        other = this.forceNode(other);
+        other = Node.forceNode(other);
         const out = new Node(sub(this.value, other.value), [this, other], OP.SUB);
         out.feedBackward = () => {
             // x - y d/dx = 1
-            this.grad = add(this.grad, out.grad);
+            Node.addGrad(this, out.grad);
             // x - y d/dy = -1
-            other.grad = add(other.grad, neg(out.grad));
+            Node.addGrad(other, neg(out.grad));
         };
         return out;
     }
     mul(other) {
-        other = this.forceNode(other);
+        other = Node.forceNode(other);
         const out = new Node(mul(this.value, other.value), [this, other], OP.MUL);
         out.feedBackward = () => {
             // x * y d/dx = y
-            this.grad = add(this.grad, mul(out.grad, other.value));
-            // x + y d/dy = x
-            other.grad = add(other.grad, mul(out.grad, this.value));
+            Node.addGrad(this, mul(out.grad, other.value));
+            // x * y d/dy = x
+            Node.addGrad(other, mul(out.grad, this.value));
         };
         return out;
     }
@@ -71,26 +73,26 @@ class Node {
             const out = new Node(pow(this.value, other.value), [this, other], OP.POW);
             out.feedBackward = () => {
                 // x^a d/dx = a*x^(a-1)
-                this.grad = add(this.grad, mul(out.grad, mul(other.value, pow(this.value, sub(other.value, 1)))));
+                Node.addGrad(this, mul(out.grad, mul(other.value, pow(this.value, sub(other.value, 1)))));
                 // x^a d/da = x^a*lnx
-                other.grad = add(other.grad, mul(out.grad, mul(pow(this.value, other.value), log(this.value))));
+                Node.addGrad(other, mul(out.grad, mul(pow(this.value, other.value), log(this.value))));
             };
             return out;
         }
         const out = new Node(pow(this.value, other), [this], OP.POW);
         out.feedBackward = () => {
-            this.grad = add(this.grad, mul(out.grad, mul(other, pow(this.value, sub(other, 1)))));
+            Node.addGrad(this, mul(out.grad, mul(other, pow(this.value, sub(other, 1)))));
         };
         return out;
     }
     div(other) {
-        other = this.forceNode(other);
+        other = Node.forceNode(other);
         const out = new Node(div(this.value, other.value), [this, other], OP.DIV);
         out.feedBackward = () => {
             // x/y d/dx = 1/y
-            this.grad = add(this.grad, div(out.grad, other.value));
+            Node.addGrad(this, div(out.grad, other.value));
             // x/y d/dy = -x/y^2
-            other.grad = add(other.grad, mul(out.grad, div(neg(this.value), pow(other.value, 2))));
+            Node.addGrad(other, mul(out.grad, div(neg(this.value), pow(other.value, 2))));
         };
         return out;
     }
@@ -98,7 +100,7 @@ class Node {
         const out = new Node(neg(this.value), [this], OP.NEG);
         out.feedBackward = () => {
             // -x d/dx = -1
-            this.grad = add(this.grad, neg(out.grad));
+            Node.addGrad(this, neg(out.grad));
         };
         return out;
     }
@@ -107,7 +109,7 @@ class Node {
         const out = new Node(expResult, [this], OP.EXP);
         out.feedBackward = () => {
             // e^x d/dx = e^x
-            this.grad = add(this.grad, mul(out.grad, expResult));
+            Node.addGrad(this, mul(out.grad, expResult));
         };
         return out;
     }
@@ -115,14 +117,14 @@ class Node {
         const out = new Node(log(this.value), [this], OP.LOG);
         out.feedBackward = () => {
             // lnx d/dx = 1/x
-            this.grad = add(this.grad, div(out.grad, this.value));
+            Node.addGrad(this, div(out.grad, this.value));
         };
         return out;
     }
     relu() {
         const out = new Node(relu(this.value), [this], OP.RELU);
         out.feedBackward = () => {
-            this.grad = add(this.grad, mul(out.grad, ge(this.value, 0)));
+            Node.addGrad(this, mul(out.grad, ge(this.value, 0)));
         };
         return out;
     }
@@ -130,7 +132,7 @@ class Node {
         const sigmoidResult = sigmoid(this.value);
         const out = new Node(sigmoidResult, [this], OP.SIGMOID);
         out.feedBackward = () => {
-            this.grad = add(this.grad, mul(mul(out.grad, sigmoidResult), sub(1, sigmoidResult)));
+            Node.addGrad(this, mul(mul(out.grad, sigmoidResult), sub(1, sigmoidResult)));
         };
         return out;
     }
@@ -138,7 +140,23 @@ class Node {
         const tanhResult = tanh(this.value);
         const out = new Node(tanhResult, [this], OP.TANH);
         out.feedBackward = () => {
-            this.grad = add(this.grad, mul(out.grad, sub(1, mul(tanhResult, tanhResult))));
+            Node.addGrad(this, mul(out.grad, sub(1, mul(tanhResult, tanhResult))));
+        };
+        return out;
+    }
+    t() {
+        const out = new Node(t(this.value), [this], OP.T);
+        out.feedBackward = () => {
+            Node.addGrad(this, t(out.grad));
+        };
+        return out;
+    }
+    mm(other) {
+        other = Node.forceNode(other);
+        const out = new Node(mm(this.value, other.value), [this, other], OP.MM);
+        out.feedBackward = () => {
+            Node.addGrad(this, mm(out.grad, t(other.value)));
+            Node.addGrad(other, mm(t(this.value), out.grad));
         };
         return out;
     }
@@ -162,10 +180,29 @@ class Node {
             topo[index].feedBackward();
         }
     }
-    forceNode(value) {
+    static forceNode(value) {
         if (value instanceof Node)
             return value;
         return new Node(value);
     }
+    static addGrad(node, accumGrad) {
+        const axesToSqueeze = [];
+        const axesToReduce = [];
+        const shape = node.shape;
+        const gradShape = tensor_1.TensorMath.getShape(accumGrad);
+        const paddedDims = gradShape.length - shape.length;
+        for (let i = 0; i < paddedDims; i++) {
+            axesToReduce.push(i);
+            axesToSqueeze.push(i);
+        }
+        for (let i = 0; i < shape.length; i++) {
+            if (shape[i] === 1 && gradShape[i + paddedDims] > 1) {
+                axesToReduce.push(i + paddedDims);
+            }
+        }
+        const reducedGrad = tensor_1.TensorMath.sum(accumGrad, axesToReduce, true);
+        const squeezedGrad = tensor_1.TensorMath.squeeze(reducedGrad, axesToSqueeze);
+        node.grad = add(squeezedGrad, node.grad);
+    }
 }
 exports.Node = Node;

package/dist/tensor.d.ts

@@ -19,4 +19,10 @@ export declare class TensorMath {
     static relu(tA: Tensor): Tensor;
     static sigmoid(tA: Tensor): Tensor;
     static tanh(tA: Tensor): Tensor;
+    static squeezeAxis(tA: Tensor, axis: number): Tensor;
+    static squeeze(tA: Tensor, dims?: number[] | number): Tensor;
+    static sumAxis(tA: Tensor, axis: number): Tensor;
+    static sum(tA: Tensor, dims?: number[] | number, keepDims?: boolean): Tensor;
+    static t(tA: Tensor): Tensor;
+    static mm(tA: Tensor, tB: Tensor): Tensor;
 }

package/dist/tensor.js

@@ -253,5 +253,105 @@ class TensorMath {
             return tA.map(subA => TensorMath.tanh(subA));
         }
     }
+    static squeezeAxis(tA, axis) {
+        if (typeof tA === "number")
+            return tA;
+        if (axis === 0) {
+            return tA[0];
+        }
+        else {
+            return tA.map(slice => TensorMath.squeezeAxis(slice, axis - 1));
+        }
+    }
+    static squeeze(tA, dims) {
+        if (typeof tA === "number")
+            return tA;
+        if (typeof dims === "number") {
+            dims = [dims];
+        }
+        if (typeof dims === "undefined") {
+            const shape = TensorMath.getShape(tA);
+            dims = [];
+            for (let index = 0; index < shape.length; index++) {
+                if (shape[index] === 1) {
+                    dims.push(index);
+                }
+            }
+        }
+        dims = [...dims].sort((a, b) => b - a);
+        let out = tA;
+        for (const axis of dims) {
+            out = TensorMath.squeezeAxis(out, axis);
+        }
+        return out;
+    }
+    static sumAxis(tA, axis) {
+        if (typeof tA === "number")
+            return tA;
+        if (axis === 0) {
+            let result = tA[0];
+            for (let i = 1; i < tA.length; i++) {
+                result = TensorMath.add(result, tA[i]);
+            }
+            return [result];
+        }
+        else {
+            return tA.map(slice => TensorMath.sumAxis(slice, axis - 1));
+        }
+    }
+    static sum(tA, dims, keepDims = false) {
+        if (typeof tA === "number")
+            return tA;
+        if (typeof dims === "number") {
+            dims = [dims];
+        }
+        if (typeof dims === "undefined") {
+            dims = Array.from({ length: TensorMath.getShape(tA).length }, (_, index) => index);
+        }
+        dims = [...dims].sort((a, b) => b - a);
+        let out = tA;
+        for (const axis of dims) {
+            out = TensorMath.sumAxis(out, axis);
+        }
+        return keepDims ? out : TensorMath.squeeze(out, dims);
+    }
+    static t(tA) {
+        const shapeA = TensorMath.getShape(tA);
+        if (shapeA.length !== 2)
+            throw new Error("Input is not a matrix");
+        const matA = tA;
+        const matARows = matA.length;
+        const matACols = matA[0].length;
+        const matATranspose = Array.from({ length: matACols }, () => new Array(matARows).fill(0));
+        for (let i = 0; i < matARows; i++) {
+            for (let j = 0; j < matACols; j++) {
+                matATranspose[j][i] = matA[i][j];
+            }
+        }
+        return matATranspose;
+    }
+    static mm(tA, tB) {
+        const shapeA = TensorMath.getShape(tA);
+        const shapeB = TensorMath.getShape(tB);
+        if (shapeA.length !== 2 || shapeB.length !== 2)
+            throw new Error("Inputs are not matrices");
+        const matA = tA;
+        const matB = tB;
+        const matARows = matA.length;
+        const matACols = matA[0].length;
+        const matBRows = matB.length;
+        const matBCols = matB[0].length;
+        if (matACols !== matBRows)
+            throw new Error("Invalid matrices shape for multiplication");
+        const matC = Array.from({ length: matARows }, () => new Array(matBCols).fill(0));
+        for (let i = 0; i < matARows; i++) {
+            for (let j = 0; j < matBCols; j++) {
+                for (let k = 0; k < matACols; k++) {
+                    matC[i][j] += matA[i][k] * matB[k][j];
+                }
+            }
+        }
+        return matC;
+    }
 }
 exports.TensorMath = TensorMath;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "catniff",
-  "version": "0.1.3",
+  "version": "0.1.5",
   "description": "A cute autograd engine for Javascript",
   "main": "index.js",
   "scripts": {
@@ -14,12 +14,18 @@
     "cats",
     "catniff",
     "autograd",
+    "autodiff",
     "ml",
     "dl",
     "ai",
     "maths",
     "gradient",
-    "tensors"
+    "tensors",
+    "library",
+    "framework",
+    "neural-network",
+    "machine-learning",
+    "deep-learning"
   ],
   "author": "nguyenphuminh",
   "license": "GPL-3.0",