catniff 0.8.5 → 0.8.7

package/dist/nn.d.ts

@@ -50,6 +50,37 @@ export declare class LSTMCell {
     constructor(inputSize: number, hiddenSize: number, bias?: boolean, device?: string, dtype?: dtype);
     forward(input: Tensor | TensorValue, hidden: Tensor | TensorValue, cell: Tensor | TensorValue): [Tensor, Tensor];
 }
+export declare class BatchNorm {
+    weight?: Tensor;
+    bias?: Tensor;
+    runningMean?: Tensor;
+    runningVar?: Tensor;
+    eps: number;
+    momentum: number;
+    numFeatures: number;
+    affine: boolean;
+    trackRunningStats: boolean;
+    numBatchesTracked: number;
+    constructor(numFeatures: number, eps?: number, momentum?: number, affine?: boolean, trackRunningStats?: boolean, device?: string, dtype?: dtype);
+    forward(input: Tensor): Tensor;
+}
+export declare class InstanceNorm {
+    weight?: Tensor;
+    bias?: Tensor;
+    eps: number;
+    numFeatures: number;
+    constructor(numFeatures: number, eps?: number, affine?: boolean, device?: string, dtype?: dtype);
+    forward(input: Tensor): Tensor;
+}
+export declare class GroupNorm {
+    weight?: Tensor;
+    bias?: Tensor;
+    eps: number;
+    numGroups: number;
+    numChannels: number;
+    constructor(numGroups: number, numChannels: number, eps?: number, affine?: boolean, device?: string, dtype?: dtype);
+    forward(input: Tensor): Tensor;
+}
 export declare class LayerNorm {
     weight?: Tensor;
     bias?: Tensor;
@@ -91,6 +122,9 @@ export declare const nn: {
     RNNCell: typeof RNNCell;
     GRUCell: typeof GRUCell;
     LSTMCell: typeof LSTMCell;
+    BatchNorm: typeof BatchNorm;
+    InstanceNorm: typeof InstanceNorm;
+    GroupNorm: typeof GroupNorm;
     LayerNorm: typeof LayerNorm;
     RMSNorm: typeof RMSNorm;
     Embedding: typeof Embedding;

package/dist/nn.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.nn = exports.MultiheadAttention = exports.Embedding = exports.RMSNorm = exports.LayerNorm = exports.LSTMCell = exports.GRUCell = exports.RNNCell = exports.Linear = void 0;
+exports.nn = exports.MultiheadAttention = exports.Embedding = exports.RMSNorm = exports.LayerNorm = exports.GroupNorm = exports.InstanceNorm = exports.BatchNorm = exports.LSTMCell = exports.GRUCell = exports.RNNCell = exports.Linear = void 0;
 exports.scaledDotProductAttention = scaledDotProductAttention;
 const core_1 = require("./core");
 function linearTransform(input, weight, bias) {
@@ -149,6 +149,187 @@ class LSTMCell {
     }
 }
 exports.LSTMCell = LSTMCell;
+class BatchNorm {
+    weight;
+    bias;
+    runningMean;
+    runningVar;
+    eps;
+    momentum;
+    numFeatures;
+    affine;
+    trackRunningStats;
+    numBatchesTracked;
+    constructor(numFeatures, eps = 1e-5, momentum = 0.1, affine = true, trackRunningStats = true, device, dtype) {
+        this.numFeatures = numFeatures;
+        this.eps = eps;
+        this.momentum = momentum;
+        this.affine = affine;
+        this.trackRunningStats = trackRunningStats;
+        this.numBatchesTracked = 0;
+        if (this.affine) {
+            this.weight = core_1.Tensor.ones([numFeatures], { requiresGrad: true, device, dtype });
+            this.bias = core_1.Tensor.zeros([numFeatures], { requiresGrad: true, device, dtype });
+        }
+        if (this.trackRunningStats) {
+            this.runningMean = core_1.Tensor.zeros([numFeatures], { requiresGrad: false, device, dtype });
+            this.runningVar = core_1.Tensor.ones([numFeatures], { requiresGrad: false, device, dtype });
+        }
+    }
+    forward(input) {
+        // Input shape: (N, C, ...) where C = numFeatures
+        // Normalize over batch dimension and spatial dimensions (if any)
+        if (input.shape.length < 2) {
+            throw new Error("Input must have at least 2 dimensions (batch, features)");
+        }
+        if (input.shape[1] !== this.numFeatures) {
+            throw new Error(`Expected ${this.numFeatures} features, got ${input.shape[1]}`);
+        }
+        let mean;
+        let variance;
+        if (core_1.Tensor.training || !this.trackRunningStats) {
+            // Training or trackRunningStats disabled - calculate mean and variance from scratch
+            // Calculate mean and variance over batch and spatial dimensions
+            // Keep only the channel dimension
+            const dims = [0, ...Array.from({ length: input.shape.length - 2 }, (_, i) => i + 2)];
+            mean = input.mean(dims, true);
+            variance = input.sub(mean).pow(2).mean(dims, true);
+            // Update running statistics if enabled and in training mode
+            if (this.trackRunningStats && core_1.Tensor.training) {
+                const exponentialAverageFactor = this.momentum;
+                this.runningMean = this.runningMean
+                    .mul(1 - exponentialAverageFactor)
+                    .add(mean.squeeze().mul(exponentialAverageFactor));
+                // Use unbiased variance for running estimate
+                const n = input.shape.reduce((acc, val, idx) => idx === 1 ? acc : acc * val, 1);
+                const unbiasingFactor = n / (n - 1);
+                this.runningVar = this.runningVar
+                    .mul(1 - exponentialAverageFactor)
+                    .add(variance.squeeze().mul(exponentialAverageFactor * unbiasingFactor));
+                this.numBatchesTracked++;
+            }
+        }
+        else {
+            // Inference with trackRunningStats enabled - use running statistics
+            mean = this.runningMean.reshape([1, this.numFeatures, ...Array(input.shape.length - 2).fill(1)]);
+            variance = this.runningVar.reshape([1, this.numFeatures, ...Array(input.shape.length - 2).fill(1)]);
+        }
+        // Normalize
+        let normalized = input.sub(mean).div(variance.add(this.eps).sqrt());
+        // Apply affine transformation
+        if (this.affine) {
+            const weightReshaped = this.weight.reshape([1, this.numFeatures, ...Array(input.shape.length - 2).fill(1)]);
+            const biasReshaped = this.bias.reshape([1, this.numFeatures, ...Array(input.shape.length - 2).fill(1)]);
+            normalized = normalized.mul(weightReshaped).add(biasReshaped);
+        }
+        return normalized;
+    }
+}
+exports.BatchNorm = BatchNorm;
+class InstanceNorm {
+    weight;
+    bias;
+    eps;
+    numFeatures;
+    constructor(numFeatures, eps = 1e-5, affine = true, device, dtype) {
+        this.numFeatures = numFeatures;
+        this.eps = eps;
+        if (affine) {
+            this.weight = core_1.Tensor.ones([numFeatures], { requiresGrad: true, device, dtype });
+            this.bias = core_1.Tensor.zeros([numFeatures], { requiresGrad: true, device, dtype });
+        }
+    }
+    forward(input) {
+        // Input should be at least 3D: [N, C, ...spatial dims]
+        if (input.shape.length < 3) {
+            throw new Error("InstanceNorm expects at least 3D input [N, C, ...spatial]");
+        }
+        if (input.shape[1] !== this.numFeatures) {
+            throw new Error(`Expected ${this.numFeatures} channels, got ${input.shape[1]}`);
+        }
+        // Normalize across spatial dimensions (all dims after channel dim)
+        const dims = [];
+        for (let i = 2; i < input.shape.length; i++) {
+            dims.push(i);
+        }
+        const mean = input.mean(dims, true);
+        const variance = input.sub(mean).pow(2).mean(dims, true);
+        let normalized = input.sub(mean).div(variance.add(this.eps).sqrt());
+        if (this.weight) {
+            // Reshape weight to [1, C, 1, 1, ...] for broadcasting
+            const weightShape = [1, this.numFeatures, ...Array(input.shape.length - 2).fill(1)];
+            const weightReshaped = this.weight.reshape(weightShape);
+            normalized = normalized.mul(weightReshaped);
+        }
+        if (this.bias) {
+            // Reshape bias to [1, C, 1, 1, ...] for broadcasting
+            const biasShape = [1, this.numFeatures, ...Array(input.shape.length - 2).fill(1)];
+            const biasReshaped = this.bias.reshape(biasShape);
+            normalized = normalized.add(biasReshaped);
+        }
+        return normalized;
+    }
+}
+exports.InstanceNorm = InstanceNorm;
+class GroupNorm {
+    weight;
+    bias;
+    eps;
+    numGroups;
+    numChannels;
+    constructor(numGroups, numChannels, eps = 1e-5, affine = true, device, dtype) {
+        if (numChannels % numGroups !== 0) {
+            throw new Error(`num_channels (${numChannels}) must be divisible by num_groups (${numGroups})`);
+        }
+        this.numGroups = numGroups;
+        this.numChannels = numChannels;
+        this.eps = eps;
+        if (affine) {
+            this.weight = core_1.Tensor.ones([numChannels], { requiresGrad: true, device, dtype });
+            this.bias = core_1.Tensor.zeros([numChannels], { requiresGrad: true, device, dtype });
+        }
+    }
+    forward(input) {
+        // Input should be at least 3D: [N, C, ...spatial dims]
+        if (input.shape.length < 3) {
+            throw new Error("GroupNorm expects at least 3D input [N, C, ...spatial]");
+        }
+        if (input.shape[1] !== this.numChannels) {
+            throw new Error(`Expected ${this.numChannels} channels, got ${input.shape[1]}`);
+        }
+        const N = input.shape[0];
+        const C = input.shape[1];
+        const spatialDims = input.shape.slice(2);
+        const channelsPerGroup = C / this.numGroups;
+        // Reshape: [N, C, ...spatial] -> [N, G, C//G, ...spatial]
+        const reshapedInput = input.reshape([N, this.numGroups, channelsPerGroup, ...spatialDims]);
+        // Normalize across (C//G, ...spatial) dimensions for each group
+        // That's dims [2, 3, 4, ...] in the reshaped tensor
+        const dims = [];
+        for (let i = 2; i < reshapedInput.shape.length; i++) {
+            dims.push(i);
+        }
+        const mean = reshapedInput.mean(dims, true);
+        const variance = reshapedInput.sub(mean).pow(2).mean(dims, true);
+        let normalized = reshapedInput.sub(mean).div(variance.add(this.eps).sqrt());
+        // Reshape back: [N, G, C//G, ...spatial] -> [N, C, ...spatial]
+        normalized = normalized.reshape(input.shape);
+        if (this.weight) {
+            // Reshape weight to [1, C, 1, 1, ...] for broadcasting
+            const weightShape = [1, this.numChannels, ...Array(spatialDims.length).fill(1)];
+            const weightReshaped = this.weight.reshape(weightShape);
+            normalized = normalized.mul(weightReshaped);
+        }
+        if (this.bias) {
+            // Reshape bias to [1, C, 1, 1, ...] for broadcasting
+            const biasShape = [1, this.numChannels, ...Array(spatialDims.length).fill(1)];
+            const biasReshaped = this.bias.reshape(biasShape);
+            normalized = normalized.add(biasReshaped);
+        }
+        return normalized;
+    }
+}
+exports.GroupNorm = GroupNorm;
 class LayerNorm {
     weight;
     bias;
@@ -383,6 +564,9 @@ exports.nn = {
     RNNCell,
     GRUCell,
     LSTMCell,
+    BatchNorm,
+    InstanceNorm,
+    GroupNorm,
     LayerNorm,
     RMSNorm,
     Embedding,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "catniff",
-  "version": "0.8.5",
+  "version": "0.8.7",
   "description": "Torch-like deep learning framework for Javascript",
   "main": "index.js",
   "scripts": {