catniff 0.2.12 → 0.2.14

package/README.md

@@ -1,6 +1,6 @@
 # Catniff
 
-Catniff is an experimental tensor ops library and autograd engine made to be Torch-like (its name is a play on "catnip" and "differentiation"). This project is heavily under development currently, so keep in mind that APIs can be completely unstable and backwards-incompatible.
+Catniff is a small deep learning framework for Javacript, built to be Torch-like, but more direct on tensors and autograd usage like Tinygrad. This project is under development currently, so keep in mind that APIs can be unstable and backwards-incompatible. On a side-note, the name is a play on "catnip" and "differentiation".
 
 ## Setup
 
@@ -80,14 +80,12 @@ All available APIs are in [`./src/core.ts`](./src/core.ts) if you want to dig de
 
 ## Todos
 
-I'm mostly just learning and playing with this currently, so there are no concrete plans yet, but here is what I currently have in mind:
-
-* Fix whatever is the problem right now (there are a lot of problems right now lol).
-* Add more tensor ops.
-* Proper documentation.
+* Bug fixes.
+* More tensor ops.
+* More detailed documentation.
 * GPU acceleration.
 * Some general neural net APIs.
-* Refactor code.
+* Code refactoring.
 * Proper tests.
 * Option to load more backends.
 

package/dist/core.d.ts

@@ -137,11 +137,23 @@ export declare class Tensor {
     t(): Tensor;
     dot(other: TensorValue | Tensor): Tensor;
     mm(other: TensorValue | Tensor): Tensor;
+    bmm(other: TensorValue | Tensor): Tensor;
     mv(other: TensorValue | Tensor): Tensor;
     matmul(other: TensorValue | Tensor): Tensor;
+    static full(shape: number[], num: number, options?: TensorOptions): Tensor;
     static fullLike(tensor: Tensor, num: number, options?: TensorOptions): Tensor;
+    static ones(shape?: number[], options?: TensorOptions): Tensor;
     static onesLike(tensor: Tensor, options?: TensorOptions): Tensor;
+    static zeros(shape?: number[], options?: TensorOptions): Tensor;
     static zerosLike(tensor: Tensor, options?: TensorOptions): Tensor;
+    static rand(shape?: number[], options?: TensorOptions): Tensor;
+    static randLike(tensor: Tensor, options?: TensorOptions): Tensor;
+    static randn(shape?: number[], options?: TensorOptions): Tensor;
+    static randnLike(tensor: Tensor, options?: TensorOptions): Tensor;
+    static randint(shape: number[], low: number, high: number, options?: TensorOptions): Tensor;
+    static randintLike(tensor: Tensor, low: number, high: number, options?: TensorOptions): Tensor;
+    static normal(shape: number[], mean: number, stdDev: number, options?: TensorOptions): Tensor;
+    static uniform(shape: number[], low: number, high: number, options?: TensorOptions): Tensor;
     backward(): void;
     val(): TensorValue;
     withGrad(requiresGrad: boolean): Tensor;

package/dist/core.js

@@ -1049,6 +1049,64 @@ class Tensor {
         }
         return out;
     }
+    // Batched 3D tensor matmul
+    bmm(other) {
+        other = Tensor.forceTensor(other);
+        // Verify 3D shape
+        if (this.shape.length !== 3 || other.shape.length !== 3 || this.shape[0] !== other.shape[0]) {
+            throw new Error("Inputs are not 3D tensors with the same first dim size");
+        }
+        // Simple matrix multiplication
+        const batchA = this.value;
+        const batchB = other.value;
+        const batchAStrides = this.strides;
+        const batchBStrides = other.strides;
+        const batchSize = this.shape[0];
+        const batchARows = this.shape[1];
+        const batchACols = this.shape[2];
+        const batchBRows = other.shape[1];
+        const batchBCols = other.shape[2];
+        if (batchACols !== batchBRows)
+            throw new Error("Invalid matrices shape for multiplication");
+        const batchCShape = [batchSize, batchARows, batchBCols];
+        const batchCStrides = Tensor.getStrides(batchCShape);
+        const batchCSize = Tensor.shapeToSize(batchCShape);
+        const batchC = new Array(batchCSize).fill(0);
+        for (let q = 0; q < batchSize; q++) {
+            for (let i = 0; i < batchARows; i++) {
+                for (let j = 0; j < batchBCols; j++) {
+                    for (let k = 0; k < batchACols; k++) {
+                        // Tensor values are 1D arrays so we have to get real index using strides
+                        batchC[q * batchCStrides[0] + i * batchCStrides[1] + j * batchCStrides[2]] +=
+                            batchA[q * batchAStrides[0] + i * batchAStrides[1] + k * batchAStrides[2]] *
+                                batchB[q * batchBStrides[0] + k * batchBStrides[1] + j * batchBStrides[2]];
+                    }
+                }
+            }
+        }
+        const out = new Tensor(batchC, { shape: batchCShape, strides: batchCStrides });
+        if (this.requiresGrad) {
+            out.requiresGrad = true;
+            out.children.push(this);
+        }
+        if (other.requiresGrad) {
+            out.requiresGrad = true;
+            out.children.push(other);
+        }
+        if (out.requiresGrad) {
+            out.gradFn = () => {
+                // Disable gradient collecting of gradients themselves
+                const outGrad = out.grad.withGrad(false);
+                const selfNoGrad = this.withGrad(false);
+                const otherNoGrad = other.withGrad(false);
+                if (this.requiresGrad)
+                    Tensor.addGrad(this, outGrad.bmm(otherNoGrad.transpose(1, 2)));
+                if (other.requiresGrad)
+                    Tensor.addGrad(other, selfNoGrad.transpose(1, 2).bmm(outGrad));
+            };
+        }
+        return out;
+    }
     // Convert right-side 1D tensor to a vector (nx1 tensor) to do matmul
     mv(other) {
         other = Tensor.forceTensor(other);
@@ -1101,24 +1159,139 @@ class Tensor {
         // Too lazy for batched matmul
         throw new Error(`Shapes [${this.shape}] and [${other.shape}] are not supported`);
     }
+    // Utility to create a new tensor filled with a number
+    static full(shape, num, options = {}) {
+        if (shape.length === 0)
+            return new Tensor(num, options);
+        const outputSize = Tensor.shapeToSize(shape);
+        const outputValue = new Array(outputSize).fill(num);
+        return new Tensor(outputValue, { shape, ...options });
+    }
     // Utility to create a new tensor with shape of another tensor, filled with a number
     static fullLike(tensor, num, options = {}) {
         if (typeof tensor.value === "number")
             return new Tensor(num, options);
         return new Tensor(new Array(tensor.value.length).fill(num), { shape: tensor.shape, strides: tensor.strides, ...options });
     }
+    // Utility to create a new tensor filled with 1
+    static ones(shape, options = {}) {
+        if (typeof shape === "undefined" || shape.length === 0)
+            return new Tensor(1, options);
+        const outputSize = Tensor.shapeToSize(shape);
+        const outputValue = new Array(outputSize).fill(1);
+        return new Tensor(outputValue, { shape, ...options });
+    }
     // Utility to create a new tensor with shape of another tensor, filled with 1
     static onesLike(tensor, options = {}) {
         if (typeof tensor.value === "number")
             return new Tensor(1, options);
         return new Tensor(new Array(tensor.value.length).fill(1), { shape: tensor.shape, strides: tensor.strides, ...options });
     }
+    // Utility to create a new tensor filled with 0
+    static zeros(shape, options = {}) {
+        if (typeof shape === "undefined" || shape.length === 0)
+            return new Tensor(0, options);
+        const outputSize = Tensor.shapeToSize(shape);
+        const outputValue = new Array(outputSize).fill(0);
+        return new Tensor(outputValue, { shape, ...options });
+    }
     // Utility to create a new tensor with shape of another tensor, filled with 0
     static zerosLike(tensor, options = {}) {
         if (typeof tensor.value === "number")
             return new Tensor(0, options);
         return new Tensor(new Array(tensor.value.length).fill(0), { shape: tensor.shape, strides: tensor.strides, ...options });
     }
+    // Utility to create a new tensor filled with a random number with uniform distribution from 0 to 1
+    static rand(shape, options = {}) {
+        if (typeof shape === "undefined" || shape.length === 0)
+            return new Tensor((0, utils_1.randUniform)(), options);
+        const outputSize = Tensor.shapeToSize(shape);
+        const outputValue = new Array(outputSize);
+        for (let index = 0; index < outputValue.length; index++) {
+            outputValue[index] = (0, utils_1.randUniform)();
+        }
+        return new Tensor(outputValue, { shape, ...options });
+    }
+    // Utility to create a new tensor with shape of another tensor, filled with a random number with uniform distribution from 0 to 1
+    static randLike(tensor, options = {}) {
+        if (typeof tensor.value === "number")
+            return new Tensor((0, utils_1.randUniform)(), options);
+        const outputValue = new Array(tensor.value.length);
+        for (let index = 0; index < outputValue.length; index++) {
+            outputValue[index] = (0, utils_1.randUniform)();
+        }
+        return new Tensor(outputValue, {
+            shape: tensor.shape, strides: tensor.strides, ...options
+        });
+    }
+    // Utility to create a new tensor filled with a random number with normal distribution of mean=0 and stddev=1
+    static randn(shape, options = {}) {
+        if (typeof shape === "undefined" || shape.length === 0)
+            return new Tensor((0, utils_1.randNormal)(), options);
+        const outputSize = Tensor.shapeToSize(shape);
+        const outputValue = new Array(outputSize);
+        for (let index = 0; index < outputValue.length; index++) {
+            outputValue[index] = (0, utils_1.randNormal)();
+        }
+        return new Tensor(outputValue, { shape, ...options });
+    }
+    // Utility to create a new tensor with shape of another tensor, filled with a random number with normal distribution of mean=0 and stddev=1
+    static randnLike(tensor, options = {}) {
+        if (typeof tensor.value === "number")
+            return new Tensor((0, utils_1.randNormal)(), options);
+        const outputValue = new Array(tensor.value.length);
+        for (let index = 0; index < outputValue.length; index++) {
+            outputValue[index] = (0, utils_1.randNormal)();
+        }
+        return new Tensor(outputValue, {
+            shape: tensor.shape, strides: tensor.strides, ...options
+        });
+    }
+    // Utility to create a new tensor filled with a random integer between low and high
+    static randint(shape, low, high, options = {}) {
+        if (shape.length === 0)
+            return new Tensor((0, utils_1.randInt)(low, high), options);
+        const outputSize = Tensor.shapeToSize(shape);
+        const outputValue = new Array(outputSize);
+        for (let index = 0; index < outputValue.length; index++) {
+            outputValue[index] = (0, utils_1.randInt)(low, high);
+        }
+        return new Tensor(outputValue, { shape, ...options });
+    }
+    // Utility to create a new tensor with shape of another tensor, filled with a random integer between low and high
+    static randintLike(tensor, low, high, options = {}) {
+        if (typeof tensor.value === "number")
+            return new Tensor((0, utils_1.randInt)(low, high), options);
+        const outputValue = new Array(tensor.value.length);
+        for (let index = 0; index < outputValue.length; index++) {
+            outputValue[index] = (0, utils_1.randInt)(low, high);
+        }
+        return new Tensor(outputValue, {
+            shape: tensor.shape, strides: tensor.strides, ...options
+        });
+    }
+    // Utility to create a new tensor filled with a random number with normal distribution of custom mean and stddev
+    static normal(shape, mean, stdDev, options = {}) {
+        if (shape.length === 0)
+            return new Tensor((0, utils_1.randNormal)(mean, stdDev), options);
+        const outputSize = Tensor.shapeToSize(shape);
+        const outputValue = new Array(outputSize);
+        for (let index = 0; index < outputValue.length; index++) {
+            outputValue[index] = (0, utils_1.randNormal)(mean, stdDev);
+        }
+        return new Tensor(outputValue, { shape, ...options });
+    }
+    // Utility to create a new tensor filled with a random number with uniform distribution from low to high
+    static uniform(shape, low, high, options = {}) {
+        if (shape.length === 0)
+            return new Tensor((0, utils_1.randUniform)(low, high), options);
+        const outputSize = Tensor.shapeToSize(shape);
+        const outputValue = new Array(outputSize);
+        for (let index = 0; index < outputValue.length; index++) {
+            outputValue[index] = (0, utils_1.randUniform)(low, high);
+        }
+        return new Tensor(outputValue, { shape, ...options });
+    }
     // Reverse-mode autodiff call
     backward() {
         // Build topological order

package/dist/utils.d.ts

@@ -1,3 +1,6 @@
 export declare function erf(x: number): number;
 export declare function erfc(x: number): number;
 export declare function erfinv(x: number): number;
+export declare function randUniform(low?: number, high?: number): number;
+export declare function randNormal(mean?: number, stdDev?: number): number;
+export declare function randInt(low: number, high: number): number;

package/dist/utils.js

@@ -3,6 +3,9 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.erf = erf;
 exports.erfc = erfc;
 exports.erfinv = erfinv;
+exports.randUniform = randUniform;
+exports.randNormal = randNormal;
+exports.randInt = randInt;
 // Error function using Abramowitz and Stegun approximation
 function erf(x) {
     const a1 = 0.254829592;
@@ -33,3 +36,15 @@ function erfinv(x) {
     const sign = x >= 0 ? 1 : -1;
     return sign * Math.sqrt(-part1 + Math.sqrt(part1 * part1 - part2));
 }
+function randUniform(low = 0, high = 1) {
+    return Math.random() * (high - low) + low;
+}
+function randNormal(mean = 0, stdDev = 1) {
+    const u = 1 - Math.random();
+    const v = 1 - Math.random();
+    const z = Math.sqrt(-2.0 * Math.log(u)) * Math.cos(2.0 * Math.PI * v);
+    return z * stdDev + mean;
+}
+function randInt(low, high) {
+    return Math.floor(Math.random() * (high - low) + low);
+}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "catniff",
-  "version": "0.2.12",
-  "description": "A cute autograd engine for Javascript",
+  "version": "0.2.14",
+  "description": "A small Torch-like deep learning framework for Javascript with tensor and autograd support",
   "main": "index.js",
   "scripts": {
     "test": "echo \"Error: no test specified\" && exit 1"
@@ -11,7 +11,6 @@
     "url": "git+https://github.com/nguyenphuminh/catniff.git"
   },
   "keywords": [
-    "cats",
     "catniff",
     "autograd",
     "autodiff",
@@ -27,6 +26,7 @@
     "machine-learning",
     "deep-learning",
     "micrograd",
+    "tinygrad",
     "torch",
     "pytorch"
   ],