catniff 0.6.11 → 0.6.13

package/README.md

@@ -132,6 +132,7 @@ All available APIs are in [`./src/`](./src/) if you want to dig deeper.
 
 * [Shakespeare-style text generator](https://github.com/nguyenphuminh/shakespeare-lm).
 * [Simple neural net for XOR calculation](./examples/xornet.js).
+* [N-th order derivative calculation](./examples/nthorder.js).
 * [Tensors](./examples/tensors.js).
 * [Optimizer](./examples/optim.js).
 * [Simple quadratic equation](./examples/quadratic.js).

package/dist/core.d.ts

@@ -23,6 +23,8 @@ export declare class Tensor {
     children: Tensor[];
     device: string;
     static training: boolean;
+    static noGrad: boolean;
+    static createGraph: boolean;
     constructor(value: TensorValue, options?: TensorOptions);
     static flattenValue(tensor: TensorValue): number[] | number;
     static getShape(tensor: TensorValue): number[];
@@ -214,7 +216,6 @@ export declare class Tensor {
         zeroGrad?: boolean;
     }): void;
     val(): TensorValue;
-    withGrad(requiresGrad: boolean): Tensor;
     detach(): Tensor;
     clone(): Tensor;
     replace(other: Tensor | TensorValue, allowShapeMismatch?: boolean): Tensor;

package/dist/core.js

@@ -14,6 +14,8 @@ class Tensor {
     children;
     device;
     static training = false;
+    static noGrad = false;
+    static createGraph = false;
     constructor(value, options = {}) {
         // Storage
         this.value = Tensor.flattenValue(value);
@@ -183,14 +185,22 @@ class Tensor {
     static elementWiseSelf(tA, op) {
         if (typeof tA.value === "number")
             return new Tensor(op(tA.value));
+        const contiguous = tA.isContiguous();
         const outputShape = tA.shape;
-        const outputStrides = Tensor.getStrides(outputShape);
+        const outputStrides = contiguous ? tA.strides : Tensor.getStrides(outputShape);
         const outputSize = tA.numel;
         const outputValue = new Array(outputSize);
-        for (let index = 0; index < outputSize; index++) {
-            const outputCoords = Tensor.indexToCoords(index, outputStrides);
-            const originalIndex = tA.offset + Tensor.coordsToIndex(outputCoords, tA.strides);
-            outputValue[index] = op(tA.value[originalIndex]);
+        if (contiguous) {
+            for (let index = 0; index < outputSize; index++) {
+                outputValue[index] = op(tA.value[index + tA.offset]);
+            }
+        }
+        else {
+            for (let index = 0; index < outputSize; index++) {
+                const outputCoords = Tensor.indexToCoords(index, outputStrides);
+                const originalIndex = tA.offset + Tensor.coordsToIndex(outputCoords, tA.strides);
+                outputValue[index] = op(tA.value[originalIndex]);
+            }
         }
         return new Tensor(outputValue, { shape: outputShape, strides: outputStrides, numel: tA.numel });
     }
@@ -210,12 +220,12 @@ class Tensor {
             out.gradFn = () => {
                 // Disable gradient collecting of gradients themselves
                 const outGrad = out.grad;
-                const selfNoGrad = this.detach();
-                const otherNoGrad = other.detach();
+                const selfWithGrad = Tensor.createGraph ? this : this.detach();
+                const otherWithGrad = Tensor.createGraph ? other : other.detach();
                 if (this.requiresGrad)
-                    Tensor.addGrad(this, thisGrad(selfNoGrad, otherNoGrad, outGrad));
+                    Tensor.addGrad(this, thisGrad(selfWithGrad, otherWithGrad, outGrad));
                 if (other.requiresGrad)
-                    Tensor.addGrad(other, otherGrad(selfNoGrad, otherNoGrad, outGrad));
+                    Tensor.addGrad(other, otherGrad(selfWithGrad, otherWithGrad, outGrad));
             };
         }
         return out;
@@ -231,9 +241,9 @@ class Tensor {
             out.gradFn = () => {
                 // Disable gradient collecting of gradients themselves
                 const outGrad = out.grad;
-                const selfNoGrad = this.detach();
+                const selfWithGrad = Tensor.createGraph ? this : this.detach();
                 if (this.requiresGrad)
-                    Tensor.addGrad(this, thisGrad(selfNoGrad, outGrad));
+                    Tensor.addGrad(this, thisGrad(selfWithGrad, outGrad));
             };
         }
         return out;
@@ -289,9 +299,6 @@ class Tensor {
     // Contiguity-related ops
     isContiguous() {
         const expectedStrides = Tensor.getStrides(this.shape);
-        if (expectedStrides.length !== this.strides.length) {
-            return false;
-        }
         for (let i = 0; i < this.strides.length; i++) {
             if (this.strides[i] !== expectedStrides[i]) {
                 return false;
@@ -340,6 +347,7 @@ class Tensor {
         const out = new Tensor(this.value, {
             shape: newShape,
             strides: outputStrides,
+            offset: this.offset,
             numel: outputSize,
             device: this.device
         });
@@ -354,28 +362,7 @@ class Tensor {
         return out;
     }
     reshape(newShape) {
-        // Verify shape size
-        const originalSize = this.numel;
-        const outputSize = Tensor.shapeToSize(newShape);
-        if (originalSize !== outputSize || typeof this.value === "number") {
-            throw new Error("Can not reshape: incompatible sizes");
-        }
-        // Create new tensor with forced compatibility (only contiguity for now)
-        const outputStrides = Tensor.getStrides(newShape);
-        const out = new Tensor(this.contiguous().value, {
-            shape: newShape,
-            strides: outputStrides,
-            numel: outputSize
-        });
-        // Gradient reshaped and flow back to the original tensor
-        if (this.requiresGrad) {
-            out.requiresGrad = true;
-            out.children.push(this);
-            out.gradFn = () => {
-                Tensor.addGrad(this, out.grad.reshape(this.shape));
-            };
-        }
-        return out;
+        return this.contiguous().view(newShape);
     }
     flatten(startDim = 0, endDim = -1) {
         // Handle negative indices
@@ -550,7 +537,7 @@ class Tensor {
     }
     // Tensor indexing
     index(indices) {
-        const tensorIndices = this.handleOther(indices).contiguous();
+        const tensorIndices = this.handleOther(indices).clone();
         if (typeof tensorIndices.value === "number") {
             return this.indexWithArray([tensorIndices.value]).squeeze(0);
         }
@@ -1353,12 +1340,12 @@ class Tensor {
             out.gradFn = () => {
                 // Disable gradient collecting of gradients themselves
                 const outGrad = out.grad;
-                const selfNoGrad = this.detach();
-                const otherNoGrad = other.detach();
+                const selfWithGrad = Tensor.createGraph ? this : this.detach();
+                const otherWithGrad = Tensor.createGraph ? other : other.detach();
                 if (this.requiresGrad)
-                    Tensor.addGrad(this, outGrad.mm(otherNoGrad.t()));
+                    Tensor.addGrad(this, outGrad.mm(otherWithGrad.t()));
                 if (other.requiresGrad)
-                    Tensor.addGrad(other, selfNoGrad.t().mm(outGrad));
+                    Tensor.addGrad(other, selfWithGrad.t().mm(outGrad));
             };
         }
         return out;
@@ -1411,12 +1398,12 @@ class Tensor {
             out.gradFn = () => {
                 // Disable gradient collecting of gradients themselves
                 const outGrad = out.grad;
-                const selfNoGrad = this.detach();
-                const otherNoGrad = other.detach();
+                const selfWithGrad = Tensor.createGraph ? this : this.detach();
+                const otherWithGrad = Tensor.createGraph ? other : other.detach();
                 if (this.requiresGrad)
-                    Tensor.addGrad(this, outGrad.bmm(otherNoGrad.transpose(1, 2)));
+                    Tensor.addGrad(this, outGrad.bmm(otherWithGrad.transpose(1, 2)));
                 if (other.requiresGrad)
-                    Tensor.addGrad(other, selfNoGrad.transpose(1, 2).bmm(outGrad));
+                    Tensor.addGrad(other, selfWithGrad.transpose(1, 2).bmm(outGrad));
             };
         }
         return out;
@@ -1511,12 +1498,12 @@ class Tensor {
                 out.gradFn = () => {
                     other = other;
                     const outGrad = out.grad;
-                    const selfNoGrad = self.detach();
-                    const otherNoGrad = other.detach();
+                    const selfWithGrad = Tensor.createGraph ? self : self.detach();
+                    const otherWithGrad = Tensor.createGraph ? other : other.detach();
                     if (this.requiresGrad)
-                        Tensor.addGrad(this, outGrad.matmul(otherNoGrad.transpose(-2, -1)));
+                        Tensor.addGrad(this, outGrad.matmul(otherWithGrad.transpose(-2, -1)));
                     if (other.requiresGrad)
-                        Tensor.addGrad(other, selfNoGrad.transpose(-2, -1).matmul(outGrad));
+                        Tensor.addGrad(other, selfWithGrad.transpose(-2, -1).matmul(outGrad));
                 };
             }
             return out;
@@ -1801,7 +1788,7 @@ class Tensor {
         const visited = new Set();
         function build(node) {
             // Only collects unvisited node and node that requires gradient
-            if (!visited.has(node) && node.requiresGrad) {
+            if (!visited.has(node) && node.requiresGrad && !Tensor.noGrad) {
                 visited.add(node);
                 // Reset grad to zeros if specified
                 if (zeroGrad) {
@@ -1841,17 +1828,6 @@ class Tensor {
         }
         return buildNested(this.value, this.shape, this.strides, this.offset);
     }
-    // Returns a view of the tensor with gradient turned on/off and detaches from autograd
-    withGrad(requiresGrad) {
-        return new Tensor(this.value, {
-            shape: this.shape,
-            strides: this.strides,
-            offset: this.offset,
-            numel: this.numel,
-            device: this.device,
-            requiresGrad
-        });
-    }
     // Returns a view of the tensor with gradient turned off and detaches from autograd
     detach() {
         return new Tensor(this.value, {
@@ -1863,15 +1839,39 @@ class Tensor {
             requiresGrad: false
         });
     }
-    // Returns a copy of the tensor (with new data allocation) and detaches from autograd
+    // Returns a copy of the tensor (with new data allocation) and keeps grad connection
     clone() {
-        return new Tensor(typeof this.value === "number" ? this.value : [...this.value], {
-            shape: this.shape,
-            strides: this.strides,
-            offset: this.offset,
-            numel: this.numel,
-            requiresGrad: this.requiresGrad
-        });
+        let out;
+        if (typeof this.value === "number") {
+            out = new Tensor(this.value);
+        }
+        else {
+            const contiguous = this.isContiguous();
+            const outputStrides = contiguous ? this.strides : Tensor.getStrides(this.shape);
+            const outputSize = this.numel;
+            const outputValue = new Array(outputSize);
+            if (contiguous) {
+                for (let index = 0; index < outputSize; index++) {
+                    outputValue[index] = this.value[this.offset + index];
+                }
+            }
+            else {
+                for (let index = 0; index < outputSize; index++) {
+                    const outputCoords = Tensor.indexToCoords(index, outputStrides);
+                    const originalIndex = Tensor.coordsToIndex(outputCoords, this.strides);
+                    outputValue[index] = this.value[this.offset + originalIndex];
+                }
+            }
+            out = new Tensor(outputValue, { shape: this.shape, strides: outputStrides, numel: outputSize });
+        }
+        if (this.requiresGrad) {
+            out.requiresGrad = true;
+            out.children.push(this);
+            out.gradFn = () => {
+                Tensor.addGrad(this, out.grad);
+            };
+        }
+        return out;
     }
     // Returns this tensor with value replaced with the value of another tensor
     replace(other, allowShapeMismatch = false) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "catniff",
-  "version": "0.6.11",
+  "version": "0.6.13",
   "description": "A small Torch-like deep learning framework for Javascript",
   "main": "index.js",
   "scripts": {