catniff 0.6.15 → 0.7.1

package/dist/core.d.ts

@@ -61,6 +61,7 @@ export declare class Tensor {
     index(indices: Tensor | TensorValue): Tensor;
     slice(ranges: number[][]): Tensor;
     chunk(chunks: number, dim?: number): Tensor[];
+    cat(other: Tensor | TensorValue, dim?: number): Tensor;
     squeeze(dims?: number[] | number): Tensor;
     unsqueeze(dim: number): Tensor;
     static reduce(tensor: Tensor, dims: number[] | number | undefined, keepDims: boolean, config: {

package/dist/core.js

@@ -218,7 +218,6 @@ class Tensor {
         }
         if (out.requiresGrad) {
             out.gradFn = () => {
-                // Disable gradient collecting of gradients themselves
                 const outGrad = out.grad;
                 const selfWithGrad = Tensor.createGraph ? this : this.detach();
                 const otherWithGrad = Tensor.createGraph ? other : other.detach();
@@ -239,7 +238,6 @@ class Tensor {
         }
         if (out.requiresGrad) {
             out.gradFn = () => {
-                // Disable gradient collecting of gradients themselves
                 const outGrad = out.grad;
                 const selfWithGrad = Tensor.createGraph ? this : this.detach();
                 if (this.requiresGrad)
@@ -649,6 +647,90 @@ class Tensor {
         }
         return results;
     }
+    // Tensor concatentation
+    cat(other, dim = 0) {
+        other = this.handleOther(other);
+        // Handle scalars
+        if (typeof this.value === "number" || typeof other.value === "number") {
+            throw new Error("Can not concatenate scalars");
+        }
+        // Handle negative indices
+        if (dim < 0) {
+            dim += this.shape.length;
+        }
+        // If dimension out of bound, throw error
+        if (dim >= this.shape.length || dim < 0) {
+            throw new Error("Dimension does not exist to concatenate");
+        }
+        // If shape does not match, throw error
+        if (this.shape.length !== other.shape.length) {
+            throw new Error("Shape does not match to concatenate");
+        }
+        const outputShape = new Array(this.shape.length);
+        for (let currentDim = 0; currentDim < this.shape.length; currentDim++) {
+            if (currentDim === dim) {
+                outputShape[currentDim] = this.shape[currentDim] + other.shape[currentDim];
+            }
+            else if (this.shape[currentDim] !== other.shape[currentDim]) {
+                throw new Error("Shape does not match to concatenate");
+            }
+            else {
+                outputShape[currentDim] = this.shape[currentDim];
+            }
+        }
+        const outputSize = Tensor.shapeToSize(outputShape);
+        const outputStrides = Tensor.getStrides(outputShape);
+        const outputValue = new Array(outputSize);
+        for (let outIndex = 0; outIndex < outputSize; outIndex++) {
+            const coords = Tensor.indexToCoords(outIndex, outputStrides);
+            // Check which tensor this output position comes from
+            if (coords[dim] < this.shape[dim]) {
+                // Comes from this tensor
+                const srcIndex = Tensor.coordsToIndex(coords, this.strides);
+                outputValue[outIndex] = this.value[srcIndex + this.offset];
+            }
+            else {
+                // Comes from other tensor - adjust coordinate in concat dimension
+                const otherCoords = [...coords];
+                otherCoords[dim] -= this.shape[dim];
+                const srcIndex = Tensor.coordsToIndex(otherCoords, other.strides);
+                outputValue[outIndex] = other.value[srcIndex + other.offset];
+            }
+        }
+        const out = new Tensor(outputValue, {
+            shape: outputShape,
+            strides: outputStrides,
+            numel: outputSize
+        });
+        if (this.requiresGrad) {
+            out.requiresGrad = true;
+            out.children.push(this);
+        }
+        if (other.requiresGrad) {
+            out.requiresGrad = true;
+            out.children.push(other);
+        }
+        if (out.requiresGrad) {
+            out.gradFn = () => {
+                const outGrad = out.grad;
+                const thisRanges = new Array(this.shape.length);
+                const otherRanges = new Array(other.shape.length);
+                for (let currentDim = 0; currentDim < this.shape.length; currentDim++) {
+                    if (currentDim === dim) {
+                        thisRanges[currentDim] = [0, this.shape[currentDim], 1];
+                        otherRanges[currentDim] = [this.shape[currentDim], outputShape[currentDim], 1];
+                    }
+                    else {
+                        thisRanges[currentDim] = [];
+                        otherRanges[currentDim] = [];
+                    }
+                }
+                Tensor.addGrad(this, outGrad.slice(thisRanges));
+                Tensor.addGrad(other, outGrad.slice(otherRanges));
+            };
+        }
+        return out;
+    }
     // Tensor squeeze
     squeeze(dims) {
         if (typeof this.value === "number")
@@ -1338,7 +1420,6 @@ class Tensor {
         }
         if (out.requiresGrad) {
             out.gradFn = () => {
-                // Disable gradient collecting of gradients themselves
                 const outGrad = out.grad;
                 const selfWithGrad = Tensor.createGraph ? this : this.detach();
                 const otherWithGrad = Tensor.createGraph ? other : other.detach();
@@ -1396,7 +1477,6 @@ class Tensor {
         }
         if (out.requiresGrad) {
             out.gradFn = () => {
-                // Disable gradient collecting of gradients themselves
                 const outGrad = out.grad;
                 const selfWithGrad = Tensor.createGraph ? this : this.detach();
                 const otherWithGrad = Tensor.createGraph ? other : other.detach();

package/dist/lrscheduler.d.ts

@@ -0,0 +1,13 @@
+import { BaseOptimizer } from "./optim";
+export declare class StepLR {
+    optimizer: BaseOptimizer;
+    stepSize: number;
+    gamma: number;
+    lastEpoch: number;
+    baseLR: number;
+    constructor(optimizer: BaseOptimizer, stepSize: number, gamma?: number, lastEpoch?: number);
+    step(epoch?: number): void;
+}
+export declare const LRScheduler: {
+    StepLR: typeof StepLR;
+};

package/dist/lrscheduler.js

@@ -0,0 +1,31 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.LRScheduler = exports.StepLR = void 0;
+class StepLR {
+    optimizer;
+    stepSize;
+    gamma;
+    lastEpoch;
+    baseLR;
+    constructor(optimizer, stepSize, gamma = 0.1, lastEpoch = -1) {
+        this.optimizer = optimizer;
+        this.stepSize = stepSize;
+        this.gamma = gamma;
+        this.lastEpoch = lastEpoch;
+        this.baseLR = this.optimizer.lr;
+    }
+    step(epoch) {
+        if (typeof epoch === "undefined") {
+            this.lastEpoch++;
+            epoch = this.lastEpoch;
+        }
+        else {
+            this.lastEpoch = epoch;
+        }
+        this.optimizer.lr = this.baseLR * this.gamma ** Math.floor(epoch / this.stepSize);
+    }
+}
+exports.StepLR = StepLR;
+exports.LRScheduler = {
+    StepLR
+};

package/dist/nn.d.ts

@@ -1,11 +1,11 @@
 import { Tensor, TensorValue } from "./core";
-declare class Linear {
+export declare class Linear {
     weight: Tensor;
     bias?: Tensor;
     constructor(inFeatures: number, outFeatures: number, bias?: boolean, device?: string);
     forward(input: Tensor | TensorValue): Tensor;
 }
-declare class RNNCell {
+export declare class RNNCell {
     weightIH: Tensor;
     weightHH: Tensor;
     biasIH?: Tensor;
@@ -13,7 +13,7 @@ declare class RNNCell {
     constructor(inputSize: number, hiddenSize: number, bias?: boolean, device?: string);
     forward(input: Tensor | TensorValue, hidden: Tensor | TensorValue): Tensor;
 }
-declare class GRUCell {
+export declare class GRUCell {
     weightIR: Tensor;
     weightIZ: Tensor;
     weightIN: Tensor;
@@ -29,7 +29,7 @@ declare class GRUCell {
     constructor(inputSize: number, hiddenSize: number, bias?: boolean, device?: string);
     forward(input: Tensor | TensorValue, hidden: Tensor | TensorValue): Tensor;
 }
-declare class LSTMCell {
+export declare class LSTMCell {
     weightII: Tensor;
     weightIF: Tensor;
     weightIG: Tensor;
@@ -49,7 +49,7 @@ declare class LSTMCell {
     constructor(inputSize: number, hiddenSize: number, bias?: boolean, device?: string);
     forward(input: Tensor | TensorValue, hidden: Tensor | TensorValue, cell: Tensor | TensorValue): [Tensor, Tensor];
 }
-declare class LayerNorm {
+export declare class LayerNorm {
     weight?: Tensor;
     bias?: Tensor;
     eps: number;
@@ -57,19 +57,19 @@ declare class LayerNorm {
     constructor(normalizedShape: number | number[], eps?: number, elementwiseAffine?: boolean, bias?: boolean, device?: string);
     forward(input: Tensor): Tensor;
 }
-declare class RMSNorm {
+export declare class RMSNorm {
     weight?: Tensor;
     eps: number;
     normalizedShape: number[];
     constructor(normalizedShape: number | number[], eps?: number, elementwiseAffine?: boolean, device?: string);
     forward(input: Tensor): Tensor;
 }
-declare class Embedding {
+export declare class Embedding {
     weight: Tensor;
     constructor(numEmbeddings: number, embeddingDim: number, device: string);
     forward(input: Tensor | TensorValue): Tensor;
 }
-declare class MultiheadAttention {
+export declare class MultiheadAttention {
     qProjection: Linear;
     kProjection: Linear;
     vProjection: Linear;
@@ -100,4 +100,3 @@ export declare const nn: {
         loadStateDict(model: any, stateDict: StateDict, prefix?: string, visited?: WeakSet<object>): void;
     };
 };
-export {};

package/dist/nn.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.nn = void 0;
+exports.nn = exports.MultiheadAttention = exports.Embedding = exports.RMSNorm = exports.LayerNorm = exports.LSTMCell = exports.GRUCell = exports.RNNCell = exports.Linear = void 0;
 const core_1 = require("./core");
 function linearTransform(input, weight, bias) {
     let output = input.matmul(weight.t());
@@ -24,6 +24,7 @@ class Linear {
         return linearTransform(input, this.weight, this.bias);
     }
 }
+exports.Linear = Linear;
 function rnnTransform(input, hidden, inputWeight, hiddenWeight, inputBias, hiddenBias) {
     let output = input.matmul(inputWeight.t()).add(hidden.matmul(hiddenWeight.t()));
     if (inputBias) {
@@ -54,6 +55,7 @@ class RNNCell {
         return rnnTransform(input, hidden, this.weightIH, this.weightHH, this.biasIH, this.biasHH).tanh();
     }
 }
+exports.RNNCell = RNNCell;
 class GRUCell {
     weightIR;
     weightIZ;
@@ -93,6 +95,7 @@ class GRUCell {
         return (z.neg().add(1).mul(n).add(z.mul(hidden)));
     }
 }
+exports.GRUCell = GRUCell;
 class LSTMCell {
     weightII;
     weightIF;
@@ -144,6 +147,7 @@ class LSTMCell {
         return [h, c];
     }
 }
+exports.LSTMCell = LSTMCell;
 class LayerNorm {
     weight;
     bias;
@@ -188,6 +192,7 @@ class LayerNorm {
         return normalized;
     }
 }
+exports.LayerNorm = LayerNorm;
 class RMSNorm {
     weight;
     eps;
@@ -224,6 +229,7 @@ class RMSNorm {
         return normalized;
     }
 }
+exports.RMSNorm = RMSNorm;
 class Embedding {
     weight;
     constructor(numEmbeddings, embeddingDim, device) {
@@ -233,6 +239,7 @@ class Embedding {
         return this.weight.index(input);
     }
 }
+exports.Embedding = Embedding;
 class MultiheadAttention {
     qProjection;
     kProjection;
@@ -284,6 +291,7 @@ class MultiheadAttention {
         return [output, needWeights ? attnWeights : undefined];
     }
 }
+exports.MultiheadAttention = MultiheadAttention;
 const state = {
     getParameters(model, visited = new WeakSet()) {
         if (visited.has(model))

package/dist/optim.d.ts

@@ -1,7 +1,11 @@
 import { Tensor } from "./core";
-declare abstract class BaseOptimizer {
+export interface BaseOptimizerOptions {
+    lr?: number;
+}
+export declare abstract class BaseOptimizer {
     params: Tensor[];
-    constructor(params: Tensor[]);
+    lr: number;
+    constructor(params: Tensor[], options?: BaseOptimizerOptions);
     zeroGrad(): void;
 }
 export interface SGDOptions {
@@ -11,9 +15,8 @@ export interface SGDOptions {
     weightDecay?: number;
     nesterov?: boolean;
 }
-declare class SGD extends BaseOptimizer {
+export declare class SGD extends BaseOptimizer {
     momentumBuffers: Map<Tensor, Tensor>;
-    lr: number;
     momentum: number;
     dampening: number;
     weightDecay: number;
@@ -27,11 +30,10 @@ export interface AdamOptions {
     eps?: number;
     weightDecay?: number;
 }
-declare class Adam extends BaseOptimizer {
+export declare class Adam extends BaseOptimizer {
     momentumBuffers: Map<Tensor, Tensor>;
     velocityBuffers: Map<Tensor, Tensor>;
     stepCount: number;
-    lr: number;
     betas: [number, number];
     eps: number;
     weightDecay: number;
@@ -44,11 +46,10 @@ export interface AdamWOptions {
     eps?: number;
     weightDecay?: number;
 }
-declare class AdamW extends BaseOptimizer {
+export declare class AdamW extends BaseOptimizer {
     momentumBuffers: Map<Tensor, Tensor>;
     velocityBuffers: Map<Tensor, Tensor>;
     stepCount: number;
-    lr: number;
     betas: [number, number];
     eps: number;
     weightDecay: number;
@@ -61,4 +62,3 @@ export declare const Optim: {
     Adam: typeof Adam;
     AdamW: typeof AdamW;
 };
-export {};

package/dist/optim.js

@@ -1,11 +1,13 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.Optim = void 0;
+exports.Optim = exports.AdamW = exports.Adam = exports.SGD = exports.BaseOptimizer = void 0;
 const core_1 = require("./core");
 class BaseOptimizer {
     params;
-    constructor(params) {
+    lr;
+    constructor(params, options) {
         this.params = params;
+        this.lr = options?.lr || 0.001;
     }
     zeroGrad() {
         for (let index = 0; index < this.params.length; index++) {
@@ -14,16 +16,15 @@ class BaseOptimizer {
         }
     }
 }
+exports.BaseOptimizer = BaseOptimizer;
 class SGD extends BaseOptimizer {
     momentumBuffers = new Map();
-    lr;
     momentum;
     dampening;
     weightDecay;
     nesterov;
     constructor(params, options) {
-        super(params);
-        this.lr = options?.lr || 0.001;
+        super(params, options);
         this.momentum = options?.momentum || 0;
         this.dampening = options?.dampening || 0;
         this.weightDecay = options?.weightDecay || 0;
@@ -66,17 +67,16 @@ class SGD extends BaseOptimizer {
         }
     }
 }
+exports.SGD = SGD;
 class Adam extends BaseOptimizer {
     momentumBuffers = new Map(); // First moment (m_t)
     velocityBuffers = new Map(); // Second moment (v_t)
     stepCount = 0;
-    lr;
     betas;
     eps;
     weightDecay;
     constructor(params, options) {
-        super(params);
-        this.lr = options?.lr || 0.001;
+        super(params, options);
         this.betas = options?.betas || [0.9, 0.999];
         this.eps = options?.eps || 1e-8;
         this.weightDecay = options?.weightDecay || 0;
@@ -126,17 +126,16 @@ class Adam extends BaseOptimizer {
         }
     }
 }
+exports.Adam = Adam;
 class AdamW extends BaseOptimizer {
     momentumBuffers = new Map(); // First moment (m_t)
     velocityBuffers = new Map(); // Second moment (v_t)
     stepCount = 0;
-    lr;
     betas;
     eps;
     weightDecay;
     constructor(params, options) {
-        super(params);
-        this.lr = options?.lr || 0.001;
+        super(params, options);
         this.betas = options?.betas || [0.9, 0.999];
         this.eps = options?.eps || 1e-8;
         this.weightDecay = options?.weightDecay || 0.01;
@@ -184,6 +183,7 @@ class AdamW extends BaseOptimizer {
         }
     }
 }
+exports.AdamW = AdamW;
 exports.Optim = {
     BaseOptimizer,
     SGD,

package/index.d.ts

@@ -1,3 +1,4 @@
 export * from "./dist/core";
 export * from "./dist/optim";
 export * from "./dist/nn";
+export * from "./dist/lrscheduler";

package/index.js

@@ -1,5 +1,6 @@
 module.exports = {
     ...require("./dist/core"),
     ...require("./dist/optim"),
-    ...require("./dist/nn")
+    ...require("./dist/nn"),
+    ...require("./dist/lrscheduler")
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "catniff",
-  "version": "0.6.15",
+  "version": "0.7.1",
   "description": "A small Torch-like deep learning framework for Javascript",
   "main": "index.js",
   "scripts": {