catniff 0.4.2 → 0.5.1

package/README.md

@@ -9,22 +9,6 @@ Install through npm:
 npm install catniff
 ```
 
-## Example
-
-Here is a little demo of a quadratic function:
-```js
-const { Tensor } = require("catniff");
-
-const x = new Tensor(2, { requiresGrad: true });
-const L = x.pow(2).add(x); // x^2 + x
-
-L.backward();
-
-console.log(x.grad.val()); // 5
-```
-
-View all examples in [`./examples`](./examples).
-
 ## Tensors
 
 Tensors in Catniff can be created by passing in a number or an nD array, and there are built-in methods that can be used to perform tensor arithmetic:
@@ -92,7 +76,7 @@ optim.step();
 console.log("Updated weight:", w.data);  // Should move toward 3.0
 ```
 
-And it can still do much more, check out the docs mentioned below for more information.
+And it can still do much more, check out the docs and examples below for more information.
 
 ## Documentation
 
@@ -100,12 +84,19 @@ Full documentation is available in [`./docs/documentation.md`](./docs/documentat
 
 All available APIs are in [`./src/`](./src/) if you want to dig deeper.
 
+## Examples
+
+* [Simple neural net for XOR calculation](./examples/xornet.js).
+* [Tensors](./examples/tensors.js).
+* [Optimizer](./examples/optim.js).
+* [Simple quadratic equation](./examples/quadratic.js).
+
 ## Todos
 
 * Bug fixes.
 * More tensor ops.
 * GPU acceleration.
-* Some general neural net APIs.
+* More general neural net APIs.
 * More detailed documentation.
 * Code refactoring.
 * Proper tests.

package/dist/core.js

@@ -20,6 +20,13 @@ class Tensor {
         this.gradFn = options.gradFn || (() => { });
         this.children = options.children || [];
         this.device = options.device || "cpu";
+        // Move tensor to device
+        if (this.device !== "cpu") {
+            const backend = Tensor.backends.get(this.device);
+            if (backend && backend.transfer) {
+                backend.transfer(this);
+            }
+        }
     }
     // Utility to flatten an nD array to be 1D
     static flatten(tensor) {
@@ -1527,6 +1534,7 @@ class Tensor {
         return new Tensor(typeof this.value === "number" ? this.value : [...this.value], {
             shape: this.shape,
             strides: this.strides,
+            device: this.device,
             requiresGrad: this.requiresGrad
         });
     }
@@ -1549,7 +1557,8 @@ class Tensor {
     to(device) {
         const backend = Tensor.backends.get(device);
         if (backend && backend.transfer) {
-            return backend.transfer(this);
+            backend.transfer(this);
+            return this;
         }
         throw new Error(`No device found to transfer tensor to or a handler is not implemented for device.`);
     }

package/dist/nn.d.ts

@@ -0,0 +1,23 @@
+import { Tensor, TensorValue } from "./core";
+declare class Linear {
+    weight: Tensor;
+    bias?: Tensor;
+    constructor(inFeatures: number, outFeatures: number, bias?: boolean, device?: string);
+    forward(input: Tensor | TensorValue): Tensor;
+}
+declare class RNNCell {
+    weightIH: Tensor;
+    weightHH: Tensor;
+    biasIH?: Tensor;
+    biasHH?: Tensor;
+    constructor(inputSize: number, hiddenSize: number, bias?: boolean, device?: string);
+    forward(input: Tensor | TensorValue, hidden: Tensor | TensorValue): Tensor;
+}
+export declare const nn: {
+    Linear: typeof Linear;
+    RNNCell: typeof RNNCell;
+    state: {
+        getParameters(model: any, visited?: WeakSet<object>): Tensor[];
+    };
+};
+export {};

package/dist/nn.js

@@ -0,0 +1,74 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.nn = void 0;
+const core_1 = require("./core");
+class Linear {
+    weight;
+    bias;
+    constructor(inFeatures, outFeatures, bias = true, device) {
+        const bound = 1 / Math.sqrt(inFeatures);
+        this.weight = core_1.Tensor.uniform([outFeatures, inFeatures], -bound, bound, { requiresGrad: true, device });
+        if (bias) {
+            this.bias = core_1.Tensor.uniform([outFeatures], -bound, bound, { requiresGrad: true, device });
+        }
+    }
+    forward(input) {
+        input = core_1.Tensor.forceTensor(input);
+        let output = input.matmul(this.weight.t());
+        if (this.bias) {
+            output = output.add(this.bias);
+        }
+        return output;
+    }
+}
+class RNNCell {
+    weightIH;
+    weightHH;
+    biasIH;
+    biasHH;
+    constructor(inputSize, hiddenSize, bias = true, device) {
+        const bound = 1 / Math.sqrt(hiddenSize);
+        this.weightIH = core_1.Tensor.uniform([hiddenSize, inputSize], -bound, bound, { requiresGrad: true, device });
+        this.weightHH = core_1.Tensor.uniform([hiddenSize, hiddenSize], -bound, bound, { requiresGrad: true, device });
+        if (bias) {
+            this.biasIH = core_1.Tensor.uniform([hiddenSize], -bound, bound, { requiresGrad: true, device });
+            this.biasHH = core_1.Tensor.uniform([hiddenSize], -bound, bound, { requiresGrad: true, device });
+        }
+    }
+    forward(input, hidden) {
+        input = core_1.Tensor.forceTensor(input);
+        hidden = core_1.Tensor.forceTensor(hidden);
+        let output = input.matmul(this.weightIH.t())
+            .add(hidden.matmul(this.weightHH.t()));
+        if (this.biasIH && this.biasHH) {
+            output = output.add(this.biasIH).add(this.biasHH);
+        }
+        return output.tanh();
+    }
+}
+const state = {
+    getParameters(model, visited = new WeakSet()) {
+        if (visited.has(model)) {
+            return [];
+        }
+        visited.add(model);
+        const parameters = [];
+        for (const key in model) {
+            if (!model.hasOwnProperty(key))
+                continue;
+            const value = model[key];
+            if (value instanceof core_1.Tensor) {
+                parameters.push(value);
+            }
+            else if (typeof value === "object" && value !== null) {
+                parameters.push(...state.getParameters(value, visited));
+            }
+        }
+        return parameters;
+    }
+};
+exports.nn = {
+    Linear,
+    RNNCell,
+    state
+};

package/dist/optim.js

@@ -20,9 +20,8 @@ class SGD {
     }
     step() {
         for (const param of this.params) {
-            if (!param.grad) {
-                throw new Error("Can not apply SGD on empty grad");
-            }
+            if (!param.grad || !param.requiresGrad)
+                continue;
             let grad = param.grad.detach(), detachedParam = param.detach();
             // Apply weight decay (L2 regularization)
             if (this.weightDecay !== 0) {
@@ -80,9 +79,8 @@ class Adam {
         const biasCorrection1 = 1 - Math.pow(beta1, this.stepCount);
         const biasCorrection2 = 1 - Math.pow(beta2, this.stepCount);
         for (const param of this.params) {
-            if (!param.grad) {
-                throw new Error("Can not apply Adam on empty grad");
-            }
+            if (!param.grad || !param.requiresGrad)
+                continue;
             let grad = param.grad.detach(), detachedParam = param.detach();
             // Apply weight decay (L2 regularization)
             if (this.weightDecay !== 0) {

package/index.d.ts

@@ -1,2 +1,3 @@
 export * from "./dist/core";
 export * from "./dist/optim";
+export * from "./dist/nn";

package/index.js

@@ -1,4 +1,5 @@
 module.exports = {
     ...require("./dist/core"),
-    ...require("./dist/optim")
+    ...require("./dist/optim"),
+    ...require("./dist/nn")
 };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "catniff",
-  "version": "0.4.2",
-  "description": "A small Torch-like deep learning framework for Javascript with tensor and autograd support",
+  "version": "0.5.1",
+  "description": "A small Torch-like deep learning framework for Javascript",
   "main": "index.js",
   "scripts": {
     "test": "echo \"Error: no test specified\" && exit 1"