catniff 0.5.4 → 0.5.5

package/dist/core.d.ts

@@ -125,6 +125,7 @@ export declare class Tensor {
     softsign(): Tensor;
     silu(): Tensor;
     mish(): Tensor;
+    gelu(approximate?: string): Tensor;
     maximum(other: TensorValue | Tensor): Tensor;
     minimum(other: TensorValue | Tensor): Tensor;
     round(): Tensor;

package/dist/core.js

@@ -1000,6 +1000,34 @@ class Tensor {
             return outGrad.mul(derivative);
         });
     }
+    // Tensor element-wise gelu
+    gelu(approximate = "none") {
+        if (approximate === "none") {
+            return this.elementWiseSelfDAG((a) => 0.5 * a * (1 + (0, utils_1.erf)(a / Math.sqrt(2))), (self, outGrad) => {
+                const sqrt2 = Math.sqrt(2);
+                const sqrt2OverPi = Math.sqrt(2 / Math.PI);
+                const xOverSqrt2 = self.div(sqrt2);
+                const erfVal = xOverSqrt2.erf();
+                const phi = xOverSqrt2.square().neg().exp().div(sqrt2OverPi);
+                const derivative = erfVal.add(1).mul(0.5).add(self.mul(phi));
+                return outGrad.mul(derivative);
+            });
+        }
+        else if (approximate === "tanh") {
+            return this.elementWiseSelfDAG((a) => 0.5 * a * (1 + Math.tanh(Math.sqrt(2 / Math.PI) * (a + 0.044715 * a * a * a))), (self, outGrad) => {
+                const sqrt2OverPi = Math.sqrt(2 / Math.PI);
+                const c = 0.044715;
+                const tanhArg = self.add(self.pow(3).mul(c)).mul(sqrt2OverPi);
+                const tanhVal = tanhArg.tanh();
+                const sechSquared = tanhVal.square().neg().add(1);
+                const term1 = tanhVal.add(1).mul(0.5);
+                const term2 = self.mul(sechSquared).mul(sqrt2OverPi).mul(self.square().mul(c * 3).add(1)).mul(0.5);
+                const derivative = term1.add(term2);
+                return outGrad.mul(derivative);
+            });
+        }
+        throw new Error("Specified approximation does not exist");
+    }
     // Tensor element-wise maximum
     maximum(other) {
         return this.elementWiseABDAG(other, (a, b) => Math.max(a, b), (self, other, outGrad) => outGrad.mul(self.gt(other).add(self.eq(other).mul(0.5))), (self, other, outGrad) => outGrad.mul(other.gt(self).add(other.eq(self).mul(0.5))));
@@ -1260,9 +1288,8 @@ class Tensor {
         else if (this.shape.length === 2 && other.shape.length === 2) {
             return this.mm(other);
         }
-        else if ((isThis1D && other.shape.length > 2) ||
-            (isOther1D && this.shape.length > 2) ||
-            (other.shape.length > 2 && this.shape.length > 2)) {
+        else if ((this.shape.length > 0 && other.shape.length >= 2) ||
+            (this.shape.length >= 2 && other.shape.length > 0)) {
             // Append/prepend dims if needed
             const self = isThis1D ? this.unsqueeze(0) : this;
             other = isOther1D ? other.unsqueeze(1) : other;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "catniff",
-  "version": "0.5.4",
+  "version": "0.5.5",
   "description": "A small Torch-like deep learning framework for Javascript",
   "main": "index.js",
   "scripts": {