catniff 0.2.16 → 0.3.0

package/README.md

@@ -76,18 +76,18 @@ console.log(X.grad.val(), Y.grad.val());
 
 Full documentation is available in [`./docs/documentation.md`](./docs/documentation.md).
 
-All available APIs are in [`./src/core.ts`](./src/core.ts) if you want to dig deeper.
+All available APIs are in [`./src/`](./src/) if you want to dig deeper.
 
 ## Todos
 
 * Bug fixes.
 * More tensor ops.
-* More detailed documentation.
 * GPU acceleration.
+* Option to load more backends.
 * Some general neural net APIs.
+* More detailed documentation.
 * Code refactoring.
 * Proper tests.
-* Option to load more backends.
 
 ## Copyrights and License
 

package/dist/core.d.ts

@@ -158,4 +158,7 @@ export declare class Tensor {
     backward(): void;
     val(): TensorValue;
     withGrad(requiresGrad: boolean): Tensor;
+    detach(): Tensor;
+    clone(): Tensor;
+    replace(other: Tensor, allowShapeMismatch?: boolean): Tensor;
 }

package/dist/core.js

@@ -1456,7 +1456,7 @@ class Tensor {
         }
         return buildNested(this.value, this.shape, this.strides);
     }
-    // Returns a copy of the tensor with gradient turned on/off and detaches from autograd
+    // Returns a view of the tensor with gradient turned on/off and detaches from autograd
     withGrad(requiresGrad) {
         return new Tensor(this.value, {
             shape: this.shape,
@@ -1464,5 +1464,34 @@ class Tensor {
             requiresGrad
         });
     }
+    // Returns a view of the tensor with gradient turned off and detaches from autograd
+    detach() {
+        return new Tensor(this.value, {
+            shape: this.shape,
+            strides: this.strides,
+            requiresGrad: false
+        });
+    }
+    // Returns a copy of the tensor (with new data allocation) and detaches from autograd
+    clone() {
+        return new Tensor(typeof this.value === "number" ? this.value : [...this.value], {
+            shape: this.shape,
+            strides: this.strides,
+            requiresGrad: this.requiresGrad
+        });
+    }
+    // Returns this tensor with value replaced with the value of another tensor
+    replace(other, allowShapeMismatch = false) {
+        // Verify shape
+        if (!allowShapeMismatch) {
+            for (let index = 0; index < this.shape.length; index++) {
+                if (this.shape[index] !== other.shape[index]) {
+                    throw new Error("Shape mismatch when trying to do tensor value replacement");
+                }
+            }
+        }
+        this.value = other.value;
+        return this;
+    }
 }
 exports.Tensor = Tensor;

package/dist/optim.d.ts

@@ -0,0 +1,23 @@
+import { Tensor } from "./core";
+export interface SGDOptions {
+    lr?: number;
+    momentum?: number;
+    dampening?: number;
+    weightDecay?: number;
+    nesterov?: boolean;
+}
+declare class SGD {
+    params: Tensor[];
+    momentumBuffers: Map<Tensor, Tensor>;
+    lr: number;
+    momentum: number;
+    dampening: number;
+    weightDecay: number;
+    nesterov: boolean;
+    constructor(params: Tensor[], options?: SGDOptions);
+    step(): void;
+}
+export declare class Optim {
+    static SGD: typeof SGD;
+}
+export {};

package/dist/optim.js

@@ -0,0 +1,61 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Optim = void 0;
+class SGD {
+    params;
+    momentumBuffers = new Map();
+    lr;
+    momentum;
+    dampening;
+    weightDecay;
+    nesterov;
+    constructor(params, options) {
+        this.params = params;
+        this.lr = options?.lr || 0.001;
+        this.momentum = options?.momentum || 0;
+        this.dampening = options?.dampening || 0;
+        this.weightDecay = options?.weightDecay || 0;
+        this.nesterov = options?.nesterov || false;
+    }
+    step() {
+        for (const param of this.params) {
+            if (typeof param.grad === "undefined") {
+                throw new Error("Can not apply SGD on empty grad");
+            }
+            let grad = param.grad.detach(), detachedParam = param.detach();
+            // Apply weight decay (L2 regularization)
+            if (this.weightDecay !== 0) {
+                grad = grad.add(detachedParam.mul(this.weightDecay));
+            }
+            // Apply momentum
+            if (this.momentum !== 0) {
+                let buf = this.momentumBuffers.get(param);
+                if (!buf) {
+                    // First time: initialize momentum buffer with current gradient
+                    buf = grad.clone();
+                    this.momentumBuffers.set(param, buf);
+                }
+                else {
+                    // Update momentum buffer: buf = momentum * buf + (1 - dampening) * grad
+                    buf = buf.mul(this.momentum).add(grad.mul(1 - this.dampening));
+                    this.momentumBuffers.set(param, buf);
+                }
+                if (this.nesterov) {
+                    // Nesterov momentum: grad = grad + momentum * buf
+                    grad = grad.add(buf.mul(this.momentum));
+                }
+                else {
+                    // Standard momentum: use momentum buffer as gradient
+                    grad = buf;
+                }
+            }
+            // Update parameter: param = param - lr * grad
+            const newParam = detachedParam.sub(grad.mul(this.lr));
+            param.replace(newParam);
+        }
+    }
+}
+class Optim {
+    static SGD = SGD;
+}
+exports.Optim = Optim;

package/index.d.ts

@@ -1 +1,2 @@
 export * from "./dist/core";
+export * from "./dist/optim";

package/index.js

@@ -1,3 +1,4 @@
 module.exports = {
-    ...require("./dist/core")
+    ...require("./dist/core"),
+    ...require("./dist/optim")
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "catniff",
-  "version": "0.2.16",
+  "version": "0.3.0",
   "description": "A small Torch-like deep learning framework for Javascript with tensor and autograd support",
   "main": "index.js",
   "scripts": {