catniff 0.6.10 → 0.6.12

package/README.md

@@ -132,6 +132,7 @@ All available APIs are in [`./src/`](./src/) if you want to dig deeper.
 
 * [Shakespeare-style text generator](https://github.com/nguyenphuminh/shakespeare-lm).
 * [Simple neural net for XOR calculation](./examples/xornet.js).
+* [N-th order derivative calculation](./examples/nthorder.js).
 * [Tensors](./examples/tensors.js).
 * [Optimizer](./examples/optim.js).
 * [Simple quadratic equation](./examples/quadratic.js).

package/dist/core.d.ts

@@ -23,6 +23,8 @@ export declare class Tensor {
     children: Tensor[];
     device: string;
     static training: boolean;
+    static noGrad: boolean;
+    static createGraph: boolean;
     constructor(value: TensorValue, options?: TensorOptions);
     static flattenValue(tensor: TensorValue): number[] | number;
     static getShape(tensor: TensorValue): number[];
@@ -160,6 +162,9 @@ export declare class Tensor {
     log1p(): Tensor;
     relu(): Tensor;
     leakyRelu(negativeSlope?: number): Tensor;
+    elu(alpha?: number): Tensor;
+    selu(): Tensor;
+    celu(alpha?: number): Tensor;
     sigmoid(): Tensor;
     tanh(): Tensor;
     softplus(): Tensor;
@@ -211,7 +216,6 @@ export declare class Tensor {
         zeroGrad?: boolean;
     }): void;
     val(): TensorValue;
-    withGrad(requiresGrad: boolean): Tensor;
     detach(): Tensor;
     clone(): Tensor;
     replace(other: Tensor | TensorValue, allowShapeMismatch?: boolean): Tensor;

package/dist/core.js

@@ -14,6 +14,8 @@ class Tensor {
     children;
     device;
     static training = false;
+    static noGrad = false;
+    static createGraph = false;
     constructor(value, options = {}) {
         // Storage
         this.value = Tensor.flattenValue(value);
@@ -210,12 +212,12 @@ class Tensor {
             out.gradFn = () => {
                 // Disable gradient collecting of gradients themselves
                 const outGrad = out.grad;
-                const selfNoGrad = this.detach();
-                const otherNoGrad = other.detach();
+                const selfWithGrad = Tensor.createGraph ? this : this.detach();
+                const otherWithGrad = Tensor.createGraph ? other : other.detach();
                 if (this.requiresGrad)
-                    Tensor.addGrad(this, thisGrad(selfNoGrad, otherNoGrad, outGrad));
+                    Tensor.addGrad(this, thisGrad(selfWithGrad, otherWithGrad, outGrad));
                 if (other.requiresGrad)
-                    Tensor.addGrad(other, otherGrad(selfNoGrad, otherNoGrad, outGrad));
+                    Tensor.addGrad(other, otherGrad(selfWithGrad, otherWithGrad, outGrad));
             };
         }
         return out;
@@ -231,9 +233,9 @@ class Tensor {
             out.gradFn = () => {
                 // Disable gradient collecting of gradients themselves
                 const outGrad = out.grad;
-                const selfNoGrad = this.detach();
+                const selfWithGrad = Tensor.createGraph ? this : this.detach();
                 if (this.requiresGrad)
-                    Tensor.addGrad(this, thisGrad(selfNoGrad, outGrad));
+                    Tensor.addGrad(this, thisGrad(selfWithGrad, outGrad));
             };
         }
         return out;
@@ -1170,6 +1172,26 @@ class Tensor {
             return outGrad.mul(self.gt(0).add(self.le(0).mul(negativeSlope)));
         });
     }
+    // Tensor element-wise elu
+    elu(alpha = 1) {
+        return this.elementWiseSelfDAG((a) => a > 0 ? a : alpha * (Math.expm1(a)), (self, outGrad) => {
+            return outGrad.mul(self.gt(0).add(self.le(0).mul(self.exp().mul(alpha))));
+        });
+    }
+    // Tensor element-wise selu
+    selu() {
+        const alpha = 1.6732632423543772848170429916717;
+        const scale = 1.0507009873554804934193349852946;
+        return this.elementWiseSelfDAG((a) => scale * (a >= 0 ? a : alpha * Math.expm1(a)), (self, outGrad) => {
+            return outGrad.mul(self.gt(0).mul(scale).add(self.le(0).mul(self.exp().mul(alpha * scale))));
+        });
+    }
+    // Tensor element-wise celu
+    celu(alpha = 1) {
+        return this.elementWiseSelfDAG((a) => a >= 0 ? a : alpha * (Math.expm1(a / alpha)), (self, outGrad) => {
+            return outGrad.mul(self.gt(0).add(self.le(0).mul(self.div(alpha).exp())));
+        });
+    }
     // Tensor element-wise sigmoid
     sigmoid() {
         return this.elementWiseSelfDAG((a) => 1 / (1 + Math.exp(-a)), (self, outGrad) => {
@@ -1333,12 +1355,12 @@ class Tensor {
             out.gradFn = () => {
                 // Disable gradient collecting of gradients themselves
                 const outGrad = out.grad;
-                const selfNoGrad = this.detach();
-                const otherNoGrad = other.detach();
+                const selfWithGrad = Tensor.createGraph ? this : this.detach();
+                const otherWithGrad = Tensor.createGraph ? other : other.detach();
                 if (this.requiresGrad)
-                    Tensor.addGrad(this, outGrad.mm(otherNoGrad.t()));
+                    Tensor.addGrad(this, outGrad.mm(otherWithGrad.t()));
                 if (other.requiresGrad)
-                    Tensor.addGrad(other, selfNoGrad.t().mm(outGrad));
+                    Tensor.addGrad(other, selfWithGrad.t().mm(outGrad));
             };
         }
         return out;
@@ -1391,12 +1413,12 @@ class Tensor {
             out.gradFn = () => {
                 // Disable gradient collecting of gradients themselves
                 const outGrad = out.grad;
-                const selfNoGrad = this.detach();
-                const otherNoGrad = other.detach();
+                const selfWithGrad = Tensor.createGraph ? this : this.detach();
+                const otherWithGrad = Tensor.createGraph ? other : other.detach();
                 if (this.requiresGrad)
-                    Tensor.addGrad(this, outGrad.bmm(otherNoGrad.transpose(1, 2)));
+                    Tensor.addGrad(this, outGrad.bmm(otherWithGrad.transpose(1, 2)));
                 if (other.requiresGrad)
-                    Tensor.addGrad(other, selfNoGrad.transpose(1, 2).bmm(outGrad));
+                    Tensor.addGrad(other, selfWithGrad.transpose(1, 2).bmm(outGrad));
             };
         }
         return out;
@@ -1491,12 +1513,12 @@ class Tensor {
                 out.gradFn = () => {
                     other = other;
                     const outGrad = out.grad;
-                    const selfNoGrad = self.detach();
-                    const otherNoGrad = other.detach();
+                    const selfWithGrad = Tensor.createGraph ? self : self.detach();
+                    const otherWithGrad = Tensor.createGraph ? other : other.detach();
                     if (this.requiresGrad)
-                        Tensor.addGrad(this, outGrad.matmul(otherNoGrad.transpose(-2, -1)));
+                        Tensor.addGrad(this, outGrad.matmul(otherWithGrad.transpose(-2, -1)));
                     if (other.requiresGrad)
-                        Tensor.addGrad(other, selfNoGrad.transpose(-2, -1).matmul(outGrad));
+                        Tensor.addGrad(other, selfWithGrad.transpose(-2, -1).matmul(outGrad));
                 };
             }
             return out;
@@ -1781,7 +1803,7 @@ class Tensor {
         const visited = new Set();
         function build(node) {
             // Only collects unvisited node and node that requires gradient
-            if (!visited.has(node) && node.requiresGrad) {
+            if (!visited.has(node) && node.requiresGrad && !Tensor.noGrad) {
                 visited.add(node);
                 // Reset grad to zeros if specified
                 if (zeroGrad) {
@@ -1821,17 +1843,6 @@ class Tensor {
         }
         return buildNested(this.value, this.shape, this.strides, this.offset);
     }
-    // Returns a view of the tensor with gradient turned on/off and detaches from autograd
-    withGrad(requiresGrad) {
-        return new Tensor(this.value, {
-            shape: this.shape,
-            strides: this.strides,
-            offset: this.offset,
-            numel: this.numel,
-            device: this.device,
-            requiresGrad
-        });
-    }
     // Returns a view of the tensor with gradient turned off and detaches from autograd
     detach() {
         return new Tensor(this.value, {
@@ -1843,15 +1854,18 @@ class Tensor {
             requiresGrad: false
         });
     }
-    // Returns a copy of the tensor (with new data allocation) and detaches from autograd
+    // Returns a copy of the tensor (with new data allocation) and keeps grad connection
     clone() {
-        return new Tensor(typeof this.value === "number" ? this.value : [...this.value], {
-            shape: this.shape,
-            strides: this.strides,
-            offset: this.offset,
-            numel: this.numel,
-            requiresGrad: this.requiresGrad
-        });
+        const newValue = typeof this.value === "number" ? this.value : [...this.value];
+        const out = new Tensor(newValue);
+        if (this.requiresGrad) {
+            out.requiresGrad = true;
+            out.children.push(this);
+            out.gradFn = () => {
+                Tensor.addGrad(this, out.grad);
+            };
+        }
+        return out;
     }
     // Returns this tensor with value replaced with the value of another tensor
     replace(other, allowShapeMismatch = false) {

package/dist/optim.d.ts

@@ -55,10 +55,10 @@ declare class AdamW extends BaseOptimizer {
     constructor(params: Tensor[], options?: AdamWOptions);
     step(): void;
 }
-export declare class Optim {
-    static BaseOptimizer: typeof BaseOptimizer;
-    static SGD: typeof SGD;
-    static Adam: typeof Adam;
-    static AdamW: typeof AdamW;
-}
+export declare const Optim: {
+    BaseOptimizer: typeof BaseOptimizer;
+    SGD: typeof SGD;
+    Adam: typeof Adam;
+    AdamW: typeof AdamW;
+};
 export {};

package/dist/optim.js

@@ -184,10 +184,9 @@ class AdamW extends BaseOptimizer {
         }
     }
 }
-class Optim {
-    static BaseOptimizer = BaseOptimizer;
-    static SGD = SGD;
-    static Adam = Adam;
-    static AdamW = AdamW;
-}
-exports.Optim = Optim;
+exports.Optim = {
+    BaseOptimizer,
+    SGD,
+    Adam,
+    AdamW
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "catniff",
-  "version": "0.6.10",
+  "version": "0.6.12",
   "description": "A small Torch-like deep learning framework for Javascript",
   "main": "index.js",
   "scripts": {