catniff 0.5.3 → 0.5.5

package/dist/core.d.ts

@@ -18,6 +18,7 @@ export declare class Tensor {
     gradFn: Function;
     children: Tensor[];
     device: string;
+    static training: boolean;
     constructor(value: TensorValue, options?: TensorOptions);
     static flatten(tensor: TensorValue): number[] | number;
     static getShape(tensor: TensorValue): readonly number[];
@@ -124,6 +125,7 @@ export declare class Tensor {
     softsign(): Tensor;
     silu(): Tensor;
     mish(): Tensor;
+    gelu(approximate?: string): Tensor;
     maximum(other: TensorValue | Tensor): Tensor;
     minimum(other: TensorValue | Tensor): Tensor;
     round(): Tensor;
@@ -146,6 +148,7 @@ export declare class Tensor {
     bmm(other: TensorValue | Tensor): Tensor;
     mv(other: TensorValue | Tensor): Tensor;
     matmul(other: TensorValue | Tensor): Tensor;
+    dropout(rate: number): Tensor;
     static full(shape: number[], num: number, options?: TensorOptions): Tensor;
     static fullLike(tensor: Tensor, num: number, options?: TensorOptions): Tensor;
     static ones(shape?: number[], options?: TensorOptions): Tensor;
@@ -160,7 +163,9 @@ export declare class Tensor {
     static randintLike(tensor: Tensor, low: number, high: number, options?: TensorOptions): Tensor;
     static normal(shape: number[], mean: number, stdDev: number, options?: TensorOptions): Tensor;
     static uniform(shape: number[], low: number, high: number, options?: TensorOptions): Tensor;
-    backward(): void;
+    backward(options?: {
+        zeroGrad?: boolean;
+    }): void;
     val(): TensorValue;
     withGrad(requiresGrad: boolean): Tensor;
     detach(): Tensor;

package/dist/core.js

@@ -11,6 +11,7 @@ class Tensor {
     gradFn;
     children;
     device;
+    static training = false;
     constructor(value, options = {}) {
         this.value = Tensor.flatten(value);
         this.shape = options.shape || Tensor.getShape(value);
@@ -999,6 +1000,34 @@ class Tensor {
             return outGrad.mul(derivative);
         });
     }
+    // Tensor element-wise gelu
+    gelu(approximate = "none") {
+        if (approximate === "none") {
+            return this.elementWiseSelfDAG((a) => 0.5 * a * (1 + (0, utils_1.erf)(a / Math.sqrt(2))), (self, outGrad) => {
+                const sqrt2 = Math.sqrt(2);
+                const sqrt2OverPi = Math.sqrt(2 / Math.PI);
+                const xOverSqrt2 = self.div(sqrt2);
+                const erfVal = xOverSqrt2.erf();
+                const phi = xOverSqrt2.square().neg().exp().div(sqrt2OverPi);
+                const derivative = erfVal.add(1).mul(0.5).add(self.mul(phi));
+                return outGrad.mul(derivative);
+            });
+        }
+        else if (approximate === "tanh") {
+            return this.elementWiseSelfDAG((a) => 0.5 * a * (1 + Math.tanh(Math.sqrt(2 / Math.PI) * (a + 0.044715 * a * a * a))), (self, outGrad) => {
+                const sqrt2OverPi = Math.sqrt(2 / Math.PI);
+                const c = 0.044715;
+                const tanhArg = self.add(self.pow(3).mul(c)).mul(sqrt2OverPi);
+                const tanhVal = tanhArg.tanh();
+                const sechSquared = tanhVal.square().neg().add(1);
+                const term1 = tanhVal.add(1).mul(0.5);
+                const term2 = self.mul(sechSquared).mul(sqrt2OverPi).mul(self.square().mul(c * 3).add(1)).mul(0.5);
+                const derivative = term1.add(term2);
+                return outGrad.mul(derivative);
+            });
+        }
+        throw new Error("Specified approximation does not exist");
+    }
     // Tensor element-wise maximum
     maximum(other) {
         return this.elementWiseABDAG(other, (a, b) => Math.max(a, b), (self, other, outGrad) => outGrad.mul(self.gt(other).add(self.eq(other).mul(0.5))), (self, other, outGrad) => outGrad.mul(other.gt(self).add(other.eq(self).mul(0.5))));
@@ -1259,9 +1288,8 @@ class Tensor {
         else if (this.shape.length === 2 && other.shape.length === 2) {
             return this.mm(other);
         }
-        else if ((isThis1D && other.shape.length > 2) ||
-            (isOther1D && this.shape.length > 2) ||
-            (other.shape.length > 2 && this.shape.length > 2)) {
+        else if ((this.shape.length > 0 && other.shape.length >= 2) ||
+            (this.shape.length >= 2 && other.shape.length > 0)) {
             // Append/prepend dims if needed
             const self = isThis1D ? this.unsqueeze(0) : this;
             other = isOther1D ? other.unsqueeze(1) : other;
@@ -1335,6 +1363,15 @@ class Tensor {
         }
         throw new Error(`Shapes [${this.shape}] and [${other.shape}] are not supported`);
     }
+    // Dropout
+    dropout(rate) {
+        if (!Tensor.training || rate === 0)
+            return this;
+        const keepRate = 1 - rate;
+        const uniform = Tensor.randLike(this);
+        const mask = uniform.lt(keepRate);
+        return this.mul(mask).div(keepRate);
+    }
     // Utility to create a new tensor filled with a number
     static full(shape, num, options = {}) {
         if (shape.length === 0)
@@ -1347,7 +1384,12 @@ class Tensor {
     static fullLike(tensor, num, options = {}) {
         if (typeof tensor.value === "number")
             return new Tensor(num, options);
-        return new Tensor(new Array(tensor.value.length).fill(num), { shape: tensor.shape, strides: tensor.strides, ...options });
+        return new Tensor(new Array(tensor.value.length).fill(num), {
+            shape: tensor.shape,
+            strides: tensor.strides,
+            device: tensor.device,
+            ...options
+        });
     }
     // Utility to create a new tensor filled with 1
     static ones(shape, options = {}) {
@@ -1361,7 +1403,12 @@ class Tensor {
     static onesLike(tensor, options = {}) {
         if (typeof tensor.value === "number")
             return new Tensor(1, options);
-        return new Tensor(new Array(tensor.value.length).fill(1), { shape: tensor.shape, strides: tensor.strides, ...options });
+        return new Tensor(new Array(tensor.value.length).fill(1), {
+            shape: tensor.shape,
+            strides: tensor.strides,
+            device: tensor.device,
+            ...options
+        });
     }
     // Utility to create a new tensor filled with 0
     static zeros(shape, options = {}) {
@@ -1375,7 +1422,12 @@ class Tensor {
     static zerosLike(tensor, options = {}) {
         if (typeof tensor.value === "number")
             return new Tensor(0, options);
-        return new Tensor(new Array(tensor.value.length).fill(0), { shape: tensor.shape, strides: tensor.strides, ...options });
+        return new Tensor(new Array(tensor.value.length).fill(0), {
+            shape: tensor.shape,
+            strides: tensor.strides,
+            device: tensor.device,
+            ...options
+        });
     }
     // Utility to create a new tensor filled with a random number with uniform distribution from 0 to 1
     static rand(shape, options = {}) {
@@ -1397,7 +1449,10 @@ class Tensor {
             outputValue[index] = (0, utils_1.randUniform)();
         }
         return new Tensor(outputValue, {
-            shape: tensor.shape, strides: tensor.strides, ...options
+            shape: tensor.shape,
+            strides: tensor.strides,
+            device: tensor.device,
+            ...options
         });
     }
     // Utility to create a new tensor filled with a random number with normal distribution of mean=0 and stddev=1
@@ -1420,7 +1475,10 @@ class Tensor {
             outputValue[index] = (0, utils_1.randNormal)();
         }
         return new Tensor(outputValue, {
-            shape: tensor.shape, strides: tensor.strides, ...options
+            shape: tensor.shape,
+            strides: tensor.strides,
+            device: tensor.device,
+            ...options
         });
     }
     // Utility to create a new tensor filled with a random integer between low and high
@@ -1443,7 +1501,10 @@ class Tensor {
             outputValue[index] = (0, utils_1.randInt)(low, high);
         }
         return new Tensor(outputValue, {
-            shape: tensor.shape, strides: tensor.strides, ...options
+            shape: tensor.shape,
+            strides: tensor.strides,
+            device: tensor.device,
+            ...options
         });
     }
     // Utility to create a new tensor filled with a random number with normal distribution of custom mean and stddev
@@ -1469,14 +1530,20 @@ class Tensor {
         return new Tensor(outputValue, { shape, ...options });
     }
     // Reverse-mode autodiff call
-    backward() {
+    backward(options = {}) {
+        // Init
+        const zeroGrad = options.zeroGrad ?? true;
         // Build topological order
         const topo = [];
         const visited = new Set();
         function build(node) {
+            // Only collects unvisited node and node that requires gradient
             if (!visited.has(node) && node.requiresGrad) {
                 visited.add(node);
-                node.grad = Tensor.zerosLike(node); // Reset grad with 0
+                // Reset grad to zeros if specified
+                if (zeroGrad) {
+                    node.grad = Tensor.zerosLike(node);
+                }
                 for (let child of node.children)
                     build(child);
                 topo.push(node);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "catniff",
-  "version": "0.5.3",
+  "version": "0.5.5",
   "description": "A small Torch-like deep learning framework for Javascript",
   "main": "index.js",
   "scripts": {