catniff 0.6.8 → 0.6.10

package/README.md

@@ -1,4 +1,4 @@
-# Catniff
+# Catniff 😺🌿
 
 Catniff is a small deep learning framework for Javacript, built to be Torch-like, but more direct on tensors and autograd usage like Tinygrad. This project is under development currently, so keep in mind that APIs can be unstable and backwards-incompatible. On a side-note, the name is a play on "catnip" and "differentiation".
 

package/dist/core.d.ts

@@ -1,8 +1,8 @@
 import { Backend } from "./backend";
 export type TensorValue = number | TensorValue[];
 export interface TensorOptions {
-    shape?: readonly number[];
-    strides?: readonly number[];
+    shape?: number[];
+    strides?: number[];
     offset?: number;
     numel?: number;
     grad?: Tensor;
@@ -13,8 +13,8 @@ export interface TensorOptions {
 }
 export declare class Tensor {
     value: number[] | number;
-    readonly shape: readonly number[];
-    readonly strides: readonly number[];
+    shape: number[];
+    strides: number[];
     offset: number;
     numel: number;
     grad?: Tensor;
@@ -24,20 +24,20 @@ export declare class Tensor {
     device: string;
     static training: boolean;
     constructor(value: TensorValue, options?: TensorOptions);
-    static flatten(tensor: TensorValue): number[] | number;
-    static getShape(tensor: TensorValue): readonly number[];
-    static getStrides(shape: readonly number[]): readonly number[];
-    static padShape(stridesA: readonly number[], stridesB: readonly number[], shapeA: readonly number[], shapeB: readonly number[]): [
-        readonly number[],
-        readonly number[],
-        readonly number[],
-        readonly number[]
+    static flattenValue(tensor: TensorValue): number[] | number;
+    static getShape(tensor: TensorValue): number[];
+    static getStrides(shape: number[]): number[];
+    static padShape(stridesA: number[], stridesB: number[], shapeA: number[], shapeB: number[]): [
+        number[],
+        number[],
+        number[],
+        number[]
     ];
-    static broadcastShapes(shapeA: readonly number[], shapeB: readonly number[]): readonly number[];
-    static indexToCoords(index: number, strides: readonly number[]): number[];
-    static coordsToUnbroadcastedIndex(coords: number[], shape: readonly number[], strides: readonly number[]): number;
-    static coordsToIndex(coords: number[], strides: readonly number[]): number;
-    static shapeToSize(shape: readonly number[]): number;
+    static broadcastShapes(shapeA: number[], shapeB: number[]): number[];
+    static indexToCoords(index: number, strides: number[]): number[];
+    static coordsToUnbroadcastedIndex(coords: number[], shape: number[], strides: number[]): number;
+    static coordsToIndex(coords: number[], strides: number[]): number;
+    static shapeToSize(shape: number[]): number;
     static elementWiseAB(tA: Tensor, tB: Tensor, op: (tA: number, tB: number) => number): Tensor;
     static elementWiseSelf(tA: Tensor, op: (tA: number) => number): Tensor;
     elementWiseABDAG(other: TensorValue | Tensor, op: (a: number, b: number) => number, thisGrad?: (self: Tensor, other: Tensor, outGrad: Tensor) => Tensor, otherGrad?: (self: Tensor, other: Tensor, outGrad: Tensor) => Tensor): Tensor;
@@ -47,8 +47,9 @@ export declare class Tensor {
     static normalizeDims(dims: number[], numDims: number): number[];
     isContiguous(): boolean;
     contiguous(): Tensor;
-    view(newShape: readonly number[]): Tensor;
-    reshape(newShape: readonly number[]): Tensor;
+    view(newShape: number[]): Tensor;
+    reshape(newShape: number[]): Tensor;
+    flatten(startDim?: number, endDim?: number): Tensor;
     transpose(dim1: number, dim2: number): Tensor;
     swapaxes: (dim1: number, dim2: number) => Tensor;
     swapdims: (dim1: number, dim2: number) => Tensor;
@@ -88,6 +89,7 @@ export declare class Tensor {
     var(dims?: number[] | number, keepDims?: boolean): Tensor;
     std(dims?: number[] | number, keepDims?: boolean): Tensor;
     softmax(dim?: number): Tensor;
+    softmin(dim?: number): Tensor;
     add(other: TensorValue | Tensor): Tensor;
     sub(other: TensorValue | Tensor): Tensor;
     subtract: (other: TensorValue | Tensor) => Tensor;
@@ -157,6 +159,7 @@ export declare class Tensor {
     log10(): Tensor;
     log1p(): Tensor;
     relu(): Tensor;
+    leakyRelu(negativeSlope?: number): Tensor;
     sigmoid(): Tensor;
     tanh(): Tensor;
     softplus(): Tensor;
@@ -186,17 +189,17 @@ export declare class Tensor {
     triu(diagonal?: number): Tensor;
     tril(diagonal?: number): Tensor;
     maskedFill(mask: Tensor | TensorValue, value: number): Tensor;
-    static full(shape: readonly number[], num: number, options?: TensorOptions): Tensor;
+    static full(shape: number[], num: number, options?: TensorOptions): Tensor;
     static fullLike(tensor: Tensor, num: number, options?: TensorOptions): Tensor;
-    static ones(shape?: readonly number[], options?: TensorOptions): Tensor;
+    static ones(shape?: number[], options?: TensorOptions): Tensor;
     static onesLike(tensor: Tensor, options?: TensorOptions): Tensor;
-    static zeros(shape?: readonly number[], options?: TensorOptions): Tensor;
+    static zeros(shape?: number[], options?: TensorOptions): Tensor;
     static zerosLike(tensor: Tensor, options?: TensorOptions): Tensor;
-    static rand(shape?: readonly number[], options?: TensorOptions): Tensor;
+    static rand(shape?: number[], options?: TensorOptions): Tensor;
     static randLike(tensor: Tensor, options?: TensorOptions): Tensor;
-    static randn(shape?: readonly number[], options?: TensorOptions): Tensor;
+    static randn(shape?: number[], options?: TensorOptions): Tensor;
     static randnLike(tensor: Tensor, options?: TensorOptions): Tensor;
-    static randint(shape: readonly number[], low: number, high: number, options?: TensorOptions): Tensor;
+    static randint(shape: number[], low: number, high: number, options?: TensorOptions): Tensor;
     static randintLike(tensor: Tensor, low: number, high: number, options?: TensorOptions): Tensor;
     static randperm(n: number, options?: TensorOptions): Tensor;
     static normal(shape: number[], mean: number, stdDev: number, options?: TensorOptions): Tensor;

package/dist/core.js

@@ -16,7 +16,7 @@ class Tensor {
     static training = false;
     constructor(value, options = {}) {
         // Storage
-        this.value = Tensor.flatten(value);
+        this.value = Tensor.flattenValue(value);
         // Tensor metadata
         this.shape = options.shape || Tensor.getShape(value);
         this.strides = options.strides || Tensor.getStrides(this.shape);
@@ -32,7 +32,7 @@ class Tensor {
         this.to_(this.device);
     }
     // Utility to flatten an nD array to be 1D
-    static flatten(tensor) {
+    static flattenValue(tensor) {
         // Handle scalar tensors
         if (typeof tensor === "number")
             return tensor;
@@ -329,7 +329,7 @@ class Tensor {
         // Verify shape size
         const originalSize = this.numel;
         const outputSize = Tensor.shapeToSize(newShape);
-        if (originalSize !== outputSize) {
+        if (originalSize !== outputSize || typeof this.value === "number") {
             throw new Error("Can not create view: incompatible sizes");
         }
         // Verify compatibility (only contiguity for now)
@@ -357,7 +357,7 @@ class Tensor {
         // Verify shape size
         const originalSize = this.numel;
         const outputSize = Tensor.shapeToSize(newShape);
-        if (originalSize !== outputSize) {
+        if (originalSize !== outputSize || typeof this.value === "number") {
             throw new Error("Can not reshape: incompatible sizes");
         }
         // Create new tensor with forced compatibility (only contiguity for now)
@@ -377,6 +377,40 @@ class Tensor {
         }
         return out;
     }
+    flatten(startDim = 0, endDim = -1) {
+        // Handle negative indices
+        if (startDim < 0) {
+            startDim += this.shape.length;
+        }
+        if (endDim < 0) {
+            endDim += this.shape.length;
+        }
+        // If dimension out of bound, throw error
+        if (startDim >= this.shape.length || endDim >= this.shape.length || startDim < 0 || endDim < 0) {
+            throw new Error("Dimensions do not exist to flatten");
+        }
+        const newShape = [];
+        let middleSize = 1;
+        for (let index = 0; index < this.shape.length; index++) {
+            // Keep dims before startDim
+            if (index < startDim) {
+                newShape.push(this.shape[index]);
+            }
+            // Multiply dims from startDim to endDim
+            if (index >= startDim && index <= endDim) {
+                middleSize *= this.shape[index];
+            }
+            // Push new flatten middle
+            if (index === endDim) {
+                newShape.push(middleSize);
+            }
+            // Keep dims after endDim
+            if (index > endDim) {
+                newShape.push(this.shape[index]);
+            }
+        }
+        return this.reshape(newShape);
+    }
     // Transpose
     transpose(dim1, dim2) {
         // Handle negative indices
@@ -611,6 +645,10 @@ class Tensor {
         if (dim < 0) {
             dim += this.shape.length;
         }
+        // If dimension out of bound, throw error
+        if (dim >= this.shape.length || dim < 0) {
+            throw new Error("Dimension do not exist to chunk");
+        }
         const sliceOpt = new Array(this.shape.length);
         for (let index = 0; index < sliceOpt.length; index++) {
             sliceOpt[index] = [];
@@ -873,14 +911,37 @@ class Tensor {
         if (typeof this.value === "number")
             return this;
         // Handle negative indexing
-        if (dim < 0)
-            dim = this.shape.length + dim;
+        if (dim < 0) {
+            dim += this.shape.length;
+        }
+        // If dimension out of bound, throw error
+        if (dim >= this.shape.length || dim < 0) {
+            throw new Error("Dimension do not exist to apply softmax");
+        }
         const maxVals = this.max(dim, true);
         const shifted = this.sub(maxVals);
         const expVals = shifted.exp();
         const sumExp = expVals.sum(dim, true);
         return expVals.div(sumExp);
     }
+    // Tensor softmin
+    softmin(dim = -1) {
+        if (typeof this.value === "number")
+            return this;
+        // Handle negative indexing
+        if (dim < 0) {
+            dim += this.shape.length;
+        }
+        // If dimension out of bound, throw error
+        if (dim >= this.shape.length || dim < 0) {
+            throw new Error("Dimension do not exist to apply softmin");
+        }
+        const maxVals = this.max(dim, true);
+        const shifted = maxVals.sub(this);
+        const expVals = shifted.exp();
+        const sumExp = expVals.sum(dim, true);
+        return expVals.div(sumExp);
+    }
     // Tensor element-wise addition
     add(other) {
         return this.elementWiseABDAG(other, (a, b) => a + b, (self, other, outGrad) => outGrad, (self, other, outGrad) => outGrad);
@@ -1103,6 +1164,12 @@ class Tensor {
     relu() {
         return this.elementWiseSelfDAG((a) => Math.max(a, 0), (self, outGrad) => outGrad.mul(self.gt(0)));
     }
+    // Tensor element-wise leaky relu
+    leakyRelu(negativeSlope = 0.01) {
+        return this.elementWiseSelfDAG((a) => Math.max(a, 0) + negativeSlope * Math.min(a, 0), (self, outGrad) => {
+            return outGrad.mul(self.gt(0).add(self.le(0).mul(negativeSlope)));
+        });
+    }
     // Tensor element-wise sigmoid
     sigmoid() {
         return this.elementWiseSelfDAG((a) => 1 / (1 + Math.exp(-a)), (self, outGrad) => {
@@ -1133,7 +1200,7 @@ class Tensor {
     mish() {
         return this.elementWiseSelfDAG((a) => a * Math.tanh(Math.log1p(Math.exp(a))), (self, outGrad) => {
             const tanhSoftPlus = self.exp().add(1).log().tanh();
-            // tanh(softplus(x)) + x * (1 - tanh²(softplus(x))) * sigmoid(x)
+            // tanh(softplus(x)) + x * (1 - tanh^2(softplus(x))) * sigmoid(x)
             const derivative = tanhSoftPlus.add(self.mul(tanhSoftPlus.square().neg().add(1)).mul(self.sigmoid()));
             return outGrad.mul(derivative);
         });

package/dist/optim.js

@@ -108,17 +108,17 @@ class Adam extends BaseOptimizer {
                 velocityBuffer = core_1.Tensor.zerosLike(grad); // Initialize with zeros (same shape as grad)
                 this.velocityBuffers.set(param, velocityBuffer);
             }
-            // Update biased first moment estimate: m_t = β1 * m_{t-1} + (1 - β1) * g_t
+            // Update biased first moment estimate: m_t = beta1 * m_{t-1} + (1 - beta1) * g_t
             momentumBuffer = momentumBuffer.mul(beta1).add(grad.mul(1 - beta1));
             this.momentumBuffers.set(param, momentumBuffer);
-            // Update biased second moment estimate: v_t = β2 * v_{t-1} + (1 - β2) * g_t^2
+            // Update biased second moment estimate: v_t = beta2 * v_{t-1} + (1 - beta2) * g_t^2
             velocityBuffer = velocityBuffer.mul(beta2).add(grad.pow(2).mul(1 - beta2));
             this.velocityBuffers.set(param, velocityBuffer);
-            // Compute bias-corrected first moment: m̂_t = m_t / (1 - β1^t)
+            // Compute bias-corrected first moment: m_hat_t = m_t / (1 - beta1^t)
             const correctedMomentum = momentumBuffer.div(biasCorrection1);
-            // Compute bias-corrected second moment: v̂_t = v_t / (1 - β2^t)
+            // Compute bias-corrected second moment: v_hat_t = v_t / (1 - beta2^t)
             const correctedVelocity = velocityBuffer.div(biasCorrection2);
-            // Update parameters: θ_t = θ_{t-1} - α * m̂_t / (√v̂_t + ε)
+            // Update parameters: theta_t = theta_{t-1} - alpha * m_hat_t / (sqrt(v_hat_t) + epsilon)
             const denom = correctedVelocity.sqrt().add(this.eps);
             const stepSize = correctedMomentum.div(denom).mul(this.lr);
             const newParam = detachedParam.sub(stepSize);
@@ -166,17 +166,17 @@ class AdamW extends BaseOptimizer {
                 velocityBuffer = core_1.Tensor.zerosLike(grad); // Initialize with zeros (same shape as grad)
                 this.velocityBuffers.set(param, velocityBuffer);
             }
-            // Update biased first moment estimate: m_t = β1 * m_{t-1} + (1 - β1) * g_t
+            // Update biased first moment estimate: m_t = beta1 * m_{t-1} + (1 - beta1) * g_t
             momentumBuffer = momentumBuffer.mul(beta1).add(grad.mul(1 - beta1));
             this.momentumBuffers.set(param, momentumBuffer);
-            // Update biased second moment estimate: v_t = β2 * v_{t-1} + (1 - β2) * g_t^2
+            // Update biased second moment estimate: v_t = beta2 * v_{t-1} + (1 - beta2) * g_t^2
             velocityBuffer = velocityBuffer.mul(beta2).add(grad.pow(2).mul(1 - beta2));
             this.velocityBuffers.set(param, velocityBuffer);
-            // Compute bias-corrected first moment: m̂_t = m_t / (1 - β1^t)
+            // Compute bias-corrected first moment: m_hat_t = m_t / (1 - beta1^t)
             const correctedMomentum = momentumBuffer.div(biasCorrection1);
-            // Compute bias-corrected second moment: v̂_t = v_t / (1 - β2^t)
+            // Compute bias-corrected second moment: v_hat_t = v_t / (1 - beta2^t)
             const correctedVelocity = velocityBuffer.div(biasCorrection2);
-            // Update parameters: θ_t = θ_t - α * m̂_t / (√v̂_t + ε)
+            // Update parameters: theta_t = theta_{t-1} - alpha * m_hat_t / (sqrt(v_hat_t) + epsilon)
             const denom = correctedVelocity.sqrt().add(this.eps);
             const stepSize = correctedMomentum.div(denom).mul(this.lr);
             const newParam = detachedParam.sub(stepSize);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "catniff",
-  "version": "0.6.8",
+  "version": "0.6.10",
   "description": "A small Torch-like deep learning framework for Javascript",
   "main": "index.js",
   "scripts": {