catniff 0.1.3 → 0.1.4

package/README.md

@@ -69,6 +69,7 @@ All available APIs are in `./src/autograd.ts`.
 
 I'm mostly just learning and playing with this currently, so there are no concrete plans yet, but here are what I currently have in mind:
 
+* Fix whatever is the problem right now (there are a lot of problems right now lol).
 * GPU acceleration.
 * Some general neural net APIs.
 

package/dist/autograd.d.ts

@@ -33,5 +33,6 @@ export declare class Node {
     sigmoid(): Node;
     tanh(): Node;
     backward(): void;
-    forceNode(value: Node | number): Node;
+    static forceNode(value: Node | number): Node;
+    static addGrad(node: Node, accumGrad: Tensor): void;
 }

package/dist/autograd.js

@@ -34,35 +34,35 @@ class Node {
         this.feedBackward = () => { };
     }
     add(other) {
-        other = this.forceNode(other);
+        other = Node.forceNode(other);
         const out = new Node(add(this.value, other.value), [this, other], OP.ADD);
         out.feedBackward = () => {
             // x + y d/dx = 1, note that we apply the chain rule continuously so out.grad is multiplied into our derivative
-            this.grad = add(this.grad, out.grad);
+            Node.addGrad(this, out.grad);
             // x + y d/dy = 1
-            other.grad = add(other.grad, out.grad);
+            Node.addGrad(other, out.grad);
         };
         return out;
     }
     sub(other) {
-        other = this.forceNode(other);
+        other = Node.forceNode(other);
         const out = new Node(sub(this.value, other.value), [this, other], OP.SUB);
         out.feedBackward = () => {
             // x - y d/dx = 1
-            this.grad = add(this.grad, out.grad);
+            Node.addGrad(this, out.grad);
             // x - y d/dy = -1
-            other.grad = add(other.grad, neg(out.grad));
+            Node.addGrad(other, neg(out.grad));
         };
         return out;
     }
     mul(other) {
-        other = this.forceNode(other);
+        other = Node.forceNode(other);
         const out = new Node(mul(this.value, other.value), [this, other], OP.MUL);
         out.feedBackward = () => {
             // x * y d/dx = y
-            this.grad = add(this.grad, mul(out.grad, other.value));
+            Node.addGrad(this, mul(out.grad, other.value));
             // x + y d/dy = x
-            other.grad = add(other.grad, mul(out.grad, this.value));
+            Node.addGrad(other, mul(out.grad, this.value));
         };
         return out;
     }
@@ -71,26 +71,26 @@ class Node {
             const out = new Node(pow(this.value, other.value), [this, other], OP.POW);
             out.feedBackward = () => {
                 // x^a d/dx = a*x^(a-1)
-                this.grad = add(this.grad, mul(out.grad, mul(other.value, pow(this.value, sub(other.value, 1)))));
+                Node.addGrad(this, mul(out.grad, mul(other.value, pow(this.value, sub(other.value, 1)))));
                 // x^a d/da = x^a*lnx
-                other.grad = add(other.grad, mul(out.grad, mul(pow(this.value, other.value), log(this.value))));
+                Node.addGrad(other, mul(out.grad, mul(pow(this.value, other.value), log(this.value))));
             };
             return out;
         }
         const out = new Node(pow(this.value, other), [this], OP.POW);
         out.feedBackward = () => {
-            this.grad = add(this.grad, mul(out.grad, mul(other, pow(this.value, sub(other, 1)))));
+            Node.addGrad(this, mul(out.grad, mul(other, pow(this.value, sub(other, 1)))));
         };
         return out;
     }
     div(other) {
-        other = this.forceNode(other);
+        other = Node.forceNode(other);
         const out = new Node(div(this.value, other.value), [this, other], OP.DIV);
         out.feedBackward = () => {
             // x/y d/dx = 1/y
-            this.grad = add(this.grad, div(out.grad, other.value));
+            Node.addGrad(this, div(out.grad, other.value));
             // x/y d/dy = -x/y^2
-            other.grad = add(other.grad, mul(out.grad, div(neg(this.value), pow(other.value, 2))));
+            Node.addGrad(other, mul(out.grad, div(neg(this.value), pow(other.value, 2))));
         };
         return out;
     }
@@ -98,7 +98,7 @@ class Node {
         const out = new Node(neg(this.value), [this], OP.NEG);
         out.feedBackward = () => {
             // -x d/dx = -1
-            this.grad = add(this.grad, neg(out.grad));
+            Node.addGrad(this, neg(out.grad));
         };
         return out;
     }
@@ -107,7 +107,7 @@ class Node {
         const out = new Node(expResult, [this], OP.EXP);
         out.feedBackward = () => {
             // e^x d/dx = e^x
-            this.grad = add(this.grad, mul(out.grad, expResult));
+            Node.addGrad(this, mul(out.grad, expResult));
         };
         return out;
     }
@@ -115,14 +115,14 @@ class Node {
         const out = new Node(log(this.value), [this], OP.LOG);
         out.feedBackward = () => {
             // lnx d/dx = 1/x
-            this.grad = add(this.grad, div(out.grad, this.value));
+            Node.addGrad(this, div(out.grad, this.value));
         };
         return out;
     }
     relu() {
         const out = new Node(relu(this.value), [this], OP.RELU);
         out.feedBackward = () => {
-            this.grad = add(this.grad, mul(out.grad, ge(this.value, 0)));
+            Node.addGrad(this, mul(out.grad, ge(this.value, 0)));
         };
         return out;
     }
@@ -130,7 +130,7 @@ class Node {
         const sigmoidResult = sigmoid(this.value);
         const out = new Node(sigmoidResult, [this], OP.SIGMOID);
         out.feedBackward = () => {
-            this.grad = add(this.grad, mul(mul(out.grad, sigmoidResult), sub(1, sigmoidResult)));
+            Node.addGrad(this, mul(mul(out.grad, sigmoidResult), sub(1, sigmoidResult)));
         };
         return out;
     }
@@ -138,7 +138,7 @@ class Node {
         const tanhResult = tanh(this.value);
         const out = new Node(tanhResult, [this], OP.TANH);
         out.feedBackward = () => {
-            this.grad = add(this.grad, mul(out.grad, sub(1, mul(tanhResult, tanhResult))));
+            Node.addGrad(this, mul(out.grad, sub(1, mul(tanhResult, tanhResult))));
         };
         return out;
     }
@@ -162,10 +162,29 @@ class Node {
             topo[index].feedBackward();
         }
     }
-    forceNode(value) {
+    static forceNode(value) {
         if (value instanceof Node)
             return value;
         return new Node(value);
     }
+    static addGrad(node, accumGrad) {
+        const axesToSqueeze = [];
+        const axesToReduce = [];
+        const shape = node.shape;
+        const gradShape = tensor_1.TensorMath.getShape(accumGrad);
+        const paddedDims = gradShape.length - shape.length;
+        for (let i = 0; i < paddedDims; i++) {
+            axesToReduce.push(i);
+            axesToSqueeze.push(i);
+        }
+        for (let i = 0; i < shape.length; i++) {
+            if (shape[i] === 1 && gradShape[i + paddedDims] > 1) {
+                axesToReduce.push(i + paddedDims);
+            }
+        }
+        const reducedGrad = tensor_1.TensorMath.sum(accumGrad, axesToReduce, true);
+        const squeezedGrad = tensor_1.TensorMath.squeeze(reducedGrad, axesToSqueeze);
+        node.grad = add(squeezedGrad, node.grad);
+    }
 }
 exports.Node = Node;

package/dist/tensor.d.ts

@@ -19,4 +19,8 @@ export declare class TensorMath {
     static relu(tA: Tensor): Tensor;
     static sigmoid(tA: Tensor): Tensor;
     static tanh(tA: Tensor): Tensor;
+    static squeezeAxis(tA: Tensor, axis: number): Tensor;
+    static squeeze(tA: Tensor, dims?: number[] | number): Tensor;
+    static sumAxis(tA: Tensor, axis: number): Tensor;
+    static sum(tA: Tensor, dims?: number[] | number, keepDims?: boolean): Tensor;
 }

package/dist/tensor.js

@@ -253,5 +253,67 @@ class TensorMath {
             return tA.map(subA => TensorMath.tanh(subA));
         }
     }
+    static squeezeAxis(tA, axis) {
+        if (typeof tA === "number")
+            return tA;
+        if (axis === 0) {
+            return tA[0];
+        }
+        else {
+            return tA.map(slice => TensorMath.squeezeAxis(slice, axis - 1));
+        }
+    }
+    static squeeze(tA, dims) {
+        if (typeof tA === "number")
+            return tA;
+        if (typeof dims === "number") {
+            dims = [dims];
+        }
+        if (typeof dims === "undefined") {
+            const shape = TensorMath.getShape(tA);
+            dims = [];
+            for (let index = 0; index < shape.length; index++) {
+                if (shape[index] === 1) {
+                    dims.push(index);
+                }
+            }
+        }
+        dims = [...dims].sort((a, b) => b - a);
+        let out = tA;
+        for (const axis of dims) {
+            out = TensorMath.squeezeAxis(out, axis);
+        }
+        return out;
+    }
+    static sumAxis(tA, axis) {
+        if (typeof tA === "number")
+            return tA;
+        if (axis === 0) {
+            let result = tA[0];
+            for (let i = 1; i < tA.length; i++) {
+                result = TensorMath.add(result, tA[i]);
+            }
+            return [result];
+        }
+        else {
+            return tA.map(slice => TensorMath.sumAxis(slice, axis - 1));
+        }
+    }
+    static sum(tA, dims, keepDims = false) {
+        if (typeof tA === "number")
+            return tA;
+        if (typeof dims === "number") {
+            dims = [dims];
+        }
+        if (typeof dims === "undefined") {
+            dims = Array.from({ length: TensorMath.getShape(tA).length }, (_, index) => index);
+        }
+        dims = [...dims].sort((a, b) => b - a);
+        let out = tA;
+        for (const axis of dims) {
+            out = TensorMath.sumAxis(out, axis);
+        }
+        return keepDims ? out : TensorMath.squeeze(out, dims);
+    }
 }
 exports.TensorMath = TensorMath;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "catniff",
-  "version": "0.1.3",
+  "version": "0.1.4",
   "description": "A cute autograd engine for Javascript",
   "main": "index.js",
   "scripts": {