catniff 0.2.16 → 0.3.1

package/README.md

@@ -76,18 +76,18 @@ console.log(X.grad.val(), Y.grad.val());
 
 Full documentation is available in [`./docs/documentation.md`](./docs/documentation.md).
 
-All available APIs are in [`./src/core.ts`](./src/core.ts) if you want to dig deeper.
+All available APIs are in [`./src/`](./src/) if you want to dig deeper.
 
 ## Todos
 
 * Bug fixes.
 * More tensor ops.
-* More detailed documentation.
 * GPU acceleration.
+* Option to load more backends.
 * Some general neural net APIs.
+* More detailed documentation.
 * Code refactoring.
 * Proper tests.
-* Option to load more backends.
 
 ## Copyrights and License
 

package/dist/core.d.ts

@@ -158,4 +158,7 @@ export declare class Tensor {
     backward(): void;
     val(): TensorValue;
     withGrad(requiresGrad: boolean): Tensor;
+    detach(): Tensor;
+    clone(): Tensor;
+    replace(other: Tensor, allowShapeMismatch?: boolean): Tensor;
 }

package/dist/core.js

@@ -1456,7 +1456,7 @@ class Tensor {
         }
         return buildNested(this.value, this.shape, this.strides);
     }
-    // Returns a copy of the tensor with gradient turned on/off and detaches from autograd
+    // Returns a view of the tensor with gradient turned on/off and detaches from autograd
     withGrad(requiresGrad) {
         return new Tensor(this.value, {
             shape: this.shape,
@@ -1464,5 +1464,34 @@ class Tensor {
             requiresGrad
         });
     }
+    // Returns a view of the tensor with gradient turned off and detaches from autograd
+    detach() {
+        return new Tensor(this.value, {
+            shape: this.shape,
+            strides: this.strides,
+            requiresGrad: false
+        });
+    }
+    // Returns a copy of the tensor (with new data allocation) and detaches from autograd
+    clone() {
+        return new Tensor(typeof this.value === "number" ? this.value : [...this.value], {
+            shape: this.shape,
+            strides: this.strides,
+            requiresGrad: this.requiresGrad
+        });
+    }
+    // Returns this tensor with value replaced with the value of another tensor
+    replace(other, allowShapeMismatch = false) {
+        // Verify shape
+        if (!allowShapeMismatch) {
+            for (let index = 0; index < this.shape.length; index++) {
+                if (this.shape[index] !== other.shape[index]) {
+                    throw new Error("Shape mismatch when trying to do tensor value replacement");
+                }
+            }
+        }
+        this.value = other.value;
+        return this;
+    }
 }
 exports.Tensor = Tensor;

package/dist/optim.d.ts

@@ -0,0 +1,42 @@
+import { Tensor } from "./core";
+export interface SGDOptions {
+    lr?: number;
+    momentum?: number;
+    dampening?: number;
+    weightDecay?: number;
+    nesterov?: boolean;
+}
+declare class SGD {
+    params: Tensor[];
+    momentumBuffers: Map<Tensor, Tensor>;
+    lr: number;
+    momentum: number;
+    dampening: number;
+    weightDecay: number;
+    nesterov: boolean;
+    constructor(params: Tensor[], options?: SGDOptions);
+    step(): void;
+}
+export interface AdamOptions {
+    lr?: number;
+    betas?: [number, number];
+    eps?: number;
+    weightDecay?: number;
+}
+declare class Adam {
+    params: Tensor[];
+    momentumBuffers: Map<Tensor, Tensor>;
+    velocityBuffers: Map<Tensor, Tensor>;
+    stepCount: number;
+    lr: number;
+    betas: [number, number];
+    eps: number;
+    weightDecay: number;
+    constructor(params: Tensor[], options?: AdamOptions);
+    step(): void;
+}
+export declare class Optim {
+    static SGD: typeof SGD;
+    static Adam: typeof Adam;
+}
+export {};

package/dist/optim.js

@@ -0,0 +1,125 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Optim = void 0;
+const core_1 = require("./core");
+class SGD {
+    params;
+    momentumBuffers = new Map();
+    lr;
+    momentum;
+    dampening;
+    weightDecay;
+    nesterov;
+    constructor(params, options) {
+        this.params = params;
+        this.lr = options?.lr || 0.001;
+        this.momentum = options?.momentum || 0;
+        this.dampening = options?.dampening || 0;
+        this.weightDecay = options?.weightDecay || 0;
+        this.nesterov = options?.nesterov || false;
+    }
+    step() {
+        for (const param of this.params) {
+            if (!param.grad) {
+                throw new Error("Can not apply SGD on empty grad");
+            }
+            let grad = param.grad.detach(), detachedParam = param.detach();
+            // Apply weight decay (L2 regularization)
+            if (this.weightDecay !== 0) {
+                grad = grad.add(detachedParam.mul(this.weightDecay));
+            }
+            // Apply momentum
+            if (this.momentum !== 0) {
+                let buf = this.momentumBuffers.get(param);
+                if (!buf) {
+                    // First time: initialize momentum buffer with current gradient
+                    buf = grad.clone();
+                    this.momentumBuffers.set(param, buf);
+                }
+                else {
+                    // Update momentum buffer: buf = momentum * buf + (1 - dampening) * grad
+                    buf = buf.mul(this.momentum).add(grad.mul(1 - this.dampening));
+                    this.momentumBuffers.set(param, buf);
+                }
+                if (this.nesterov) {
+                    // Nesterov momentum: grad = grad + momentum * buf
+                    grad = grad.add(buf.mul(this.momentum));
+                }
+                else {
+                    // Standard momentum: use momentum buffer as gradient
+                    grad = buf;
+                }
+            }
+            // Update parameter: param = param - lr * grad
+            const newParam = detachedParam.sub(grad.mul(this.lr));
+            param.replace(newParam);
+        }
+    }
+}
+class Adam {
+    params;
+    momentumBuffers = new Map(); // First moment (m_t)
+    velocityBuffers = new Map(); // Second moment (v_t)
+    stepCount = 0;
+    lr;
+    betas;
+    eps;
+    weightDecay;
+    constructor(params, options) {
+        this.params = params;
+        this.lr = options?.lr || 0.001;
+        this.betas = options?.betas || [0.9, 0.999];
+        this.eps = options?.eps || 1e-8;
+        this.weightDecay = options?.weightDecay || 0;
+    }
+    step() {
+        this.stepCount++;
+        const beta1 = this.betas[0];
+        const beta2 = this.betas[1];
+        // Bias correction factors
+        const biasCorrection1 = 1 - Math.pow(beta1, this.stepCount);
+        const biasCorrection2 = 1 - Math.pow(beta2, this.stepCount);
+        for (const param of this.params) {
+            if (!param.grad) {
+                throw new Error("Can not apply Adam on empty grad");
+            }
+            let grad = param.grad.detach(), detachedParam = param.detach();
+            // Apply weight decay (L2 regularization)
+            if (this.weightDecay !== 0) {
+                grad = grad.add(detachedParam.mul(this.weightDecay));
+            }
+            // Get or initialize first moment buffer (momentum)
+            let momentumBuffer = this.momentumBuffers.get(param);
+            if (!momentumBuffer) {
+                momentumBuffer = core_1.Tensor.zerosLike(grad); // Initialize with zeros (same shape as grad)
+                this.momentumBuffers.set(param, momentumBuffer);
+            }
+            // Get or initialize second moment buffer (velocity)
+            let velocityBuffer = this.velocityBuffers.get(param);
+            if (!velocityBuffer) {
+                velocityBuffer = core_1.Tensor.zerosLike(grad); // Initialize with zeros (same shape as grad)
+                this.velocityBuffers.set(param, velocityBuffer);
+            }
+            // Update biased first moment estimate: m_t = β1 * m_{t-1} + (1 - β1) * g_t
+            momentumBuffer = momentumBuffer.mul(beta1).add(grad.mul(1 - beta1));
+            this.momentumBuffers.set(param, momentumBuffer);
+            // Update biased second moment estimate: v_t = β2 * v_{t-1} + (1 - β2) * g_t^2
+            velocityBuffer = velocityBuffer.mul(beta2).add(grad.pow(2).mul(1 - beta2));
+            this.velocityBuffers.set(param, velocityBuffer);
+            // Compute bias-corrected first moment: m̂_t = m_t / (1 - β1^t)
+            const correctedMomentum = momentumBuffer.div(biasCorrection1);
+            // Compute bias-corrected second moment: v̂_t = v_t / (1 - β2^t)
+            const correctedVelocity = velocityBuffer.div(biasCorrection2);
+            // Update parameters: θ_t = θ_{t-1} - α * m̂_t / (√v̂_t + ε)
+            const denom = correctedVelocity.sqrt().add(this.eps);
+            const stepSize = correctedMomentum.div(denom).mul(this.lr);
+            const newParam = detachedParam.sub(stepSize);
+            param.replace(newParam);
+        }
+    }
+}
+class Optim {
+    static SGD = SGD;
+    static Adam = Adam;
+}
+exports.Optim = Optim;

package/index.d.ts

@@ -1 +1,2 @@
 export * from "./dist/core";
+export * from "./dist/optim";

package/index.js

@@ -1,3 +1,4 @@
 module.exports = {
-    ...require("./dist/core")
+    ...require("./dist/core"),
+    ...require("./dist/optim")
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "catniff",
-  "version": "0.2.16",
+  "version": "0.3.1",
   "description": "A small Torch-like deep learning framework for Javascript with tensor and autograd support",
   "main": "index.js",
   "scripts": {