@dniskav/neuron 0.2.3 → 0.2.6

package/README.md

@@ -21,9 +21,11 @@ A minimal, dependency-free neural network library built from scratch in TypeScri
 | `AttentionHead` | Single scaled dot-product self-attention head (Q / K / V projections + backprop). |
 | `LayerNorm` | Layer normalization with learnable γ / β per feature. |
 | `WeightMatrix` | 2D weight matrix with per-scalar Adam optimizers. Optional per-element gradient clipping via `update(dW, lr, clipValue)`. |
+| `BiasVector` | 1D bias vector with per-scalar Adam optimizers. Companion to `WeightMatrix` for bias terms. |
 | `EmbeddingMatrix` | Lookup-table embedding matrix with SGD updates. |
 | `sigmoid` `relu` `tanh` `linear` | Built-in activation functions. |
-| `SGD` `Momentum` `Adam` | Optimizers. Each instance tracks its own state per weight. |
+| `SGD` `Momentum` `Adam` `ClipOptimizer` | Optimizers. Each instance tracks its own state per weight. `ClipOptimizer` wraps any optimizer with gradient clipping. |
+| `defaultOptimizer` | Default `OptimizerFactory` (`() => new SGD()`). Shared across `NeuronN`, `Layer`, `NetworkN`, `NetworkLSTM`. |
 | `mse` `crossEntropy` | Loss functions for evaluation and logging. |
 | `mseDelta` `crossEntropyDelta` | Output-layer delta functions for use with `trainWithDeltas`. |
 
@@ -83,8 +85,8 @@ for (let epoch = 0; epoch < 5000; epoch++) {
   }
 }
 
-console.log(net.predict([0, 1])); // ~0.97
-console.log(net.predict([1, 1])); // ~0.03
+console.log(net.predict([0, 1])[0]); // ~0.97
+console.log(net.predict([1, 1])[0]); // ~0.03
 ```
 
 ### NetworkN — deep network with custom architecture
@@ -307,6 +309,29 @@ const attnWeights = net.getAttentionWeights();
 // attnWeights[blockIdx][headIdx] → seqLen × seqLen matrix
 ```
 
+## Changelog
+
+### v0.2.6
+- **Fix:** `Network.predict` now returns `number[]` (consistent with all other network classes)
+- **Fix:** `Network.train` now uses the configured optimizer and `activation.dfn()` instead of hardcoded SGD and sigmoid derivative
+- **Fix:** `LayerNorm.backwardOne` now correctly uses pre-update γ when computing the input gradient
+- **Fix:** LSTM and GRU gate initialization corrected from He (`√(2/n)`) to Xavier fan-in+out (`√(2/(fanIn+fanOut))`), matching the sigmoid/tanh activations used in those gates
+- **New:** `BiasVector` — 1D counterpart to `WeightMatrix` with per-scalar Adam optimizers; replaces repeated `number[] + Adam[]` pairs in `TransformerBlock`, `NetworkTransformer`, and `NetworkTransformerRL`
+- **New:** `defaultOptimizer` exported from `optimizers.ts` — single source of truth for the default `() => new SGD()` factory
+- **Refactor:** `NetworkN.train` and `trainWithDeltas` share extracted `_forwardAll()` and `_backpropLayers()` internals — eliminates ~50 lines of duplication
+- **Refactor:** `Transformer` backward methods now throw descriptive errors instead of crashing with a cryptic `TypeError` when called before `predict()`
+- **Refactor:** `NetworkTransformer.setWeights()` and `NetworkTransformerRL.setWeightsFlat()` use each component's own `setWeights()` instead of direct `.W` mutation
+
+### v0.2.5
+- Unified optimizer factories for `LSTMLayer`, `GRULayer`, `Conv1D` (per-scalar Adam/Momentum/SGD)
+- `NetworkN`: residual connections (`residual` option) and dropout (`dropoutRate`)
+- `Conv1D`: multi-channel input (`inputChannels`)
+- `NetworkTransformerRL`: configurable pooling (`avg` / `max` / `last` / `weighted`)
+- `Trainer`: weight decay, early stopping, classification metrics, gradient clipping support
+- `DataLoader`: validation split (`validationSplit` + `getValidationData()`)
+- `ModelSaver`: universal serialization via flat `getWeights()`/`setWeights()` for all classes
+- Gradient check test suite (`tests/GradientCheck.test.ts`)
+
 ## Possible improvements
 
 1. **Support for batches** in training to improve efficiency and gradient stability.

package/dist/index.d.mts

@@ -25,6 +25,7 @@ interface Optimizer {
     step(weight: number, gradient: number, lr: number): number;
 }
 type OptimizerFactory = () => Optimizer;
+declare const defaultOptimizer: OptimizerFactory;
 declare class SGD implements Optimizer {
     step(weight: number, gradient: number, lr: number): number;
 }
@@ -34,6 +35,13 @@ declare class Momentum implements Optimizer {
     constructor(beta?: number);
     step(weight: number, gradient: number, lr: number): number;
 }
+declare class ClipOptimizer implements Optimizer {
+    readonly inner: Optimizer;
+    readonly clipValue: number;
+    constructor(inner: Optimizer, clipValue: number);
+    step(weight: number, gradient: number, lr: number): number;
+}
+declare function ClippedOptimizerFactory(innerFactory: OptimizerFactory, clipValue: number): OptimizerFactory;
 declare class Adam implements Optimizer {
     readonly beta1: number;
     readonly beta2: number;
@@ -66,24 +74,35 @@ declare class Network {
     hiddenLayer: Layer;
     outputLayer: Layer;
     constructor(nInputs: number, nHidden: number, nOutputs: number);
-    predict(inputs: number[]): number;
+    predict(inputs: number[]): number[];
     train(inputs: number[], target: number, lr: number): number;
+    getWeights(): number[];
+    setWeights(weights: number[]): void;
 }
 
 interface NetworkNOptions {
     activations?: Activation[];
     optimizer?: OptimizerFactory;
+    residual?: boolean | ((layerIndex: number) => boolean);
+    dropoutRate?: number;
 }
 declare class NetworkN {
     readonly structure: number[];
     layers: Layer[];
+    private _dropouts;
+    private _residual;
     constructor(structure: number[], options?: NetworkNOptions);
-    predict(inputs: number[]): number[];
+    predict(inputs: number[], training?: boolean): number[];
     train(inputs: number[], targets: number[], lr: number): number;
     trainWithDeltas(inputs: number[], outputDeltas: number[], lr: number): void;
+    getWeights(): number[];
+    setWeights(weights: number[]): void;
+    private _shouldResidual;
+    private _forwardAll;
+    private _backpropLayers;
 }
 
-declare class Gate {
+declare class Gate$1 {
     W: number[][];
     b: number[];
     constructor(inputSize: number, hSize: number, initBias?: number);
@@ -94,12 +113,13 @@ declare class LSTMLayer {
     readonly hSize: number;
     h: number[];
     c: number[];
-    forgetGate: Gate;
-    inputGate: Gate;
-    cellGate: Gate;
-    outputGate: Gate;
+    forgetGate: Gate$1;
+    inputGate: Gate$1;
+    cellGate: Gate$1;
+    outputGate: Gate$1;
+    private _optimizers;
     private _traj;
-    constructor(inputSize: number, hiddenSize: number);
+    constructor(inputSize: number, hiddenSize: number, optimizerFactory?: OptimizerFactory);
     reset(): void;
     predict(inputs: number[]): number[];
     backprop(dh_seq: number[][], lr: number): void;
@@ -122,6 +142,8 @@ declare class LSTMLayer {
         };
     };
     setWeights(data: ReturnType<LSTMLayer["getWeights"]>): void;
+    getWeightsFlat(): number[];
+    setWeightsFlat(weights: number[]): void;
 }
 
 interface NetworkLSTMOptions {
@@ -163,6 +185,8 @@ declare class NetworkLSTM {
         }[][];
     };
     setWeights(data: ReturnType<NetworkLSTM["getWeights"]>): void;
+    getWeightsFlat(): number[];
+    setWeightsFlat(weights: number[]): void;
 }
 
 declare function matMul(A: number[][], B: number[][]): number[][];
@@ -174,38 +198,56 @@ declare class WeightMatrix {
     private opts;
     constructor(rows: number, cols: number);
     update(dW: number[][], lr: number, clipValue?: number): void;
+    getWeights(): number[];
+    setWeights(weights: number[]): void;
+}
+declare class BiasVector {
+    values: number[];
+    private opts;
+    constructor(size: number);
+    update(grad: number[], lr: number): void;
+    getWeights(): number[];
+    setWeights(weights: number[]): void;
 }
 declare class EmbeddingMatrix {
     W: number[][];
     constructor(vocabSize: number, d_model: number);
     get(idx: number): number[];
     update(idx: number, grad: number[], lr: number): void;
+    getWeights(): number[];
+    setWeights(weights: number[]): void;
 }
 
 declare class AttentionHead {
     readonly d_k: number;
     readonly d_v: number;
+    readonly causal: boolean;
     Wq: WeightMatrix;
     Wk: WeightMatrix;
     Wv: WeightMatrix;
     private cache;
-    constructor(d_model: number, d_k: number, d_v: number);
+    constructor(d_model: number, d_k: number, d_v: number, causal?: boolean);
     predict(X: number[][]): number[][];
     backward(dOut: number[][], lr: number): number[][];
     getAttentionWeights(): number[][] | null;
+    getWeights(): number[];
+    setWeights(weights: number[]): void;
 }
 
 declare class MultiHeadAttention {
     readonly nHeads: number;
     readonly d_model: number;
     readonly d_k: number;
+    readonly causal: boolean;
     heads: AttentionHead[];
     Wo: WeightMatrix;
     private _concat;
-    constructor(d_model: number, nHeads: number);
+    constructor(d_model: number, nHeads: number, causal?: boolean);
     predict(X: number[][]): number[][];
     backward(dOut: number[][], lr: number): number[][];
     getAttentionWeights(): (number[][] | null)[];
+    getWeights(): number[];
+    setWeights(weights: number[]): void;
 }
 
 declare class LayerNorm {
@@ -217,12 +259,15 @@ declare class LayerNorm {
     resetCache(seqLen: number): void;
     predictOne(x: number[], pos: number): number[];
     backwardOne(dOut: number[], pos: number, lr: number): number[];
+    getWeights(): number[];
+    setWeights(weights: number[]): void;
 }
 
 interface TransformerBlockOptions {
     d_model: number;
     nHeads: number;
     d_ff: number;
+    causal?: boolean;
 }
 declare class TransformerBlock {
     readonly d_model: number;
@@ -232,20 +277,20 @@ declare class TransformerBlock {
     norm2: LayerNorm;
     ff1: WeightMatrix;
     ff2: WeightMatrix;
-    b1: number[];
-    b2: number[];
-    private b1Opts;
-    private b2Opts;
+    b1: BiasVector;
+    b2: BiasVector;
     private _X;
     private _attnOut;
     private _h1;
     private _ff1Pre;
     private _ff1Out;
     private _ff2Out;
-    constructor({ d_model, nHeads, d_ff }: TransformerBlockOptions);
+    constructor({ d_model, nHeads, d_ff, causal }: TransformerBlockOptions);
     predict(X: number[][]): number[][];
     backward(dOut: number[][], lr: number): number[][];
     getAttentionWeights(): (number[][] | null)[];
+    getWeights(): number[];
+    setWeights(weights: number[]): void;
 }
 
 interface NetworkTransformerOptions {
@@ -265,12 +310,13 @@ declare class NetworkTransformer {
     posEmb: EmbeddingMatrix;
     blocks: TransformerBlock[];
     outputProj: WeightMatrix;
-    outputBias: number[];
-    private outBiasOpts;
+    outputBias: BiasVector;
     constructor(seqLen: number, options?: NetworkTransformerOptions);
     predict(tokens: number[]): number[];
     train(tokens: number[], targets: number[], lr: number, mask?: boolean[]): number;
     getAttentionWeights(): (number[][] | null)[][];
+    getWeights(): number[];
+    setWeights(weights: number[]): void;
     private _forward;
 }
 
@@ -280,6 +326,7 @@ interface NetworkTransformerRLOptions {
     d_ff?: number;
     nBlocks?: number;
     nActions?: number;
+    pooling?: 'avg' | 'max' | 'last' | 'weighted';
 }
 declare class NetworkTransformerRL {
     readonly seqLen: number;
@@ -289,14 +336,17 @@ declare class NetworkTransformerRL {
     inputProj: WeightMatrix;
     blocks: TransformerBlock[];
     outputProj: WeightMatrix;
-    outputBias: number[];
-    private outBiasOpts;
+    outputBias: BiasVector;
     private _projected;
+    private _pooling;
+    private _argmax;
     constructor(seqLen: number, inputDim: number, options?: NetworkTransformerRLOptions);
     predict(sequence: number[][]): number[];
     train(sequence: number[][], target: number[], lr: number): number;
     getAttentionWeights(): (number[][] | null)[][];
-    getWeights(): {
+    getWeightsFlat(): number[];
+    setWeightsFlat(weights: number[]): void;
+    getWeightsStructured(): {
         inputProj: number[][];
         blocks: {
             attn: {
@@ -323,9 +373,18 @@ declare class NetworkTransformerRL {
         outputProj: number[][];
         outputBias: number[];
     };
-    setWeights(data: ReturnType<NetworkTransformerRL['getWeights']>): void;
+    setWeightsStructured(data: ReturnType<NetworkTransformerRL['getWeightsStructured']>): void;
+    getWeights(): number[];
+    setWeights(weights: number[]): void;
     private _forward;
     private _pool;
+    private _poolAvg;
+    private _poolMax;
+    private _poolLast;
+    private _poolWeighted;
+    /** Returns the current pooling type for inspection. */
+    getPoolingType(): string;
+    private _distributePoolGradient;
 }
 
 declare function mse(predicted: number[], actual: number[]): number;
@@ -334,4 +393,204 @@ declare function mseDelta(predicted: number, actual: number): number;
 declare function crossEntropyDelta(predicted: number, actual: number): number;
 declare function crossEntropyDeltaRaw(predicted: number, actual: number): number;
 
-export { type Activation, Adam, AttentionHead, EmbeddingMatrix, LSTMLayer, Layer, LayerNorm, Momentum, MultiHeadAttention, Network, NetworkLSTM, type NetworkLSTMOptions, NetworkN, type NetworkNOptions, NetworkTransformer, type NetworkTransformerOptions, NetworkTransformerRL, type NetworkTransformerRLOptions, Neuron, NeuronN, type Optimizer, type OptimizerFactory, SGD, TransformerBlock, type TransformerBlockOptions, WeightMatrix, crossEntropy, crossEntropyDelta, crossEntropyDeltaRaw, elu, leakyRelu, linear, makeElu, makeLeakyRelu, matMul, mse, mseDelta, relu, sigmoid, softmax, softmaxBackward, tanh, transpose };
+declare class Dropout {
+    readonly rate: number;
+    private _mask;
+    constructor(rate: number);
+    forward(x: number[], training?: boolean): number[];
+    backward(dOut: number[]): number[];
+    resetMask(): void;
+    getWeights(): number[];
+    setWeights(_weights: number[]): void;
+}
+
+declare class Gate {
+    W: number[][];
+    b: number[];
+    constructor(inputSize: number, hSize: number, initBias?: number);
+    linear(combined: number[]): number[];
+}
+declare class GRULayer {
+    readonly inputSize: number;
+    readonly hSize: number;
+    h: number[];
+    resetGate: Gate;
+    updateGate: Gate;
+    newGate: Gate;
+    private _optimizers;
+    private _traj;
+    constructor(inputSize: number, hiddenSize: number, optimizerFactory?: OptimizerFactory);
+    reset(): void;
+    predict(inputs: number[]): number[];
+    backprop(dh_seq: number[][], lr: number): void;
+    getWeightsFlat(): number[];
+    setWeightsFlat(weights: number[]): void;
+    getWeights(): {
+        resetGate: {
+            W: number[][];
+            b: number[];
+        };
+        updateGate: {
+            W: number[][];
+            b: number[];
+        };
+        newGate: {
+            W: number[][];
+            b: number[];
+        };
+    };
+    setWeights(data: ReturnType<GRULayer["getWeights"]>): void;
+}
+
+declare class BatchNorm {
+    readonly dim: number;
+    readonly momentum: number;
+    gamma: number[];
+    beta: number[];
+    runningMean: number[];
+    runningVar: number[];
+    private _xNorm;
+    private _std;
+    constructor(dim: number, momentum?: number);
+    forward(x: number[]): number[];
+    backward(dOut: number[]): number[];
+    trainParams(dOut: number[], lr: number): void;
+    getWeights(): number[];
+    setWeights(weights: number[]): void;
+}
+
+declare class Conv1D {
+    readonly inputLength: number;
+    readonly kernelSize: number;
+    readonly filters: number;
+    readonly stride: number;
+    readonly padding: 'valid' | 'same';
+    readonly inputChannels: number;
+    kernels: number[][][];
+    biases: number[];
+    private _kOpts;
+    private _bOpts;
+    private _input;
+    private _paddedInput;
+    constructor(inputLength: number, kernelSize: number, filters: number, stride?: number, padding?: 'valid' | 'same', optimizerFactory?: OptimizerFactory, inputChannels?: number);
+    forward(input: number[] | number[][]): number[][];
+    backward(dOut: number[][], lr?: number): number[][];
+    getOutputLength(): number;
+    getWeights(): number[];
+    setWeights(weights: number[]): void;
+    private _normalizeInput;
+}
+
+interface DataPair {
+    inputs: number[][];
+    targets: number[][];
+}
+declare class DataLoader {
+    readonly data: DataPair;
+    readonly batchSize: number;
+    private _indices;
+    private _trainIndices;
+    private _valIndices;
+    private _pos;
+    private _validationSplit;
+    constructor(data: DataPair, batchSize?: number, validationSplit?: number);
+    shuffle(): void;
+    hasNext(): boolean;
+    next(): DataPair;
+    reset(): void;
+    get length(): number;
+    getValidationData(): DataPair;
+    get validationLength(): number;
+    static sequences(data: number[][], seqLen: number, validationSplit?: number): DataLoader;
+}
+
+interface TrainMetrics {
+    accuracy: number;
+    precision: number;
+    recall: number;
+    f1: number;
+}
+interface TrainerOptions {
+    epochs?: number;
+    lr?: number;
+    lrDecay?: number;
+    verbose?: boolean;
+    weightDecay?: number;
+    earlyStopping?: {
+        patience: number;
+        minDelta: number;
+    };
+    computeMetrics?: boolean;
+    clipValue?: number;
+}
+interface TrainDataset {
+    inputs: number[][];
+    targets: number[][];
+}
+interface TrainableNetwork {
+    train(inputs: number[], targets: number[], lr: number): number;
+}
+/** Extended interface for networks that support weight access and prediction.
+ *  Required for weightDecay, earlyStopping, and computeMetrics features. */
+interface TrainableNetworkWithWeights extends TrainableNetwork {
+    predict(inputs: number[]): number[];
+    getWeights(): number[];
+    setWeights(weights: number[]): void;
+}
+declare class Trainer {
+    readonly network: TrainableNetwork;
+    readonly epochs: number;
+    readonly lrInitial: number;
+    readonly lrDecay: number;
+    readonly verbose: boolean;
+    readonly weightDecay: number;
+    readonly clipValue: number;
+    private _history;
+    private _validationData?;
+    private _earlyStopping?;
+    private _bestLoss;
+    private _patienceCounter;
+    private _stopReason;
+    private _computeMetrics;
+    private _metrics;
+    constructor(network: TrainableNetwork, options?: TrainerOptions);
+    setValidationData(dataset: DataPair): void;
+    getBestLoss(): number;
+    getStopReason(): string;
+    getMetrics(): TrainMetrics[];
+    train(dataset: TrainDataset): number[];
+    getHistory(): number[];
+    /** Type guard: does this network support getWeights/setWeights/predict? */
+    private _hasWeights;
+    /** Mean squared error on a dataset (used for validation loss). */
+    private _computeLoss;
+    /** Heuristic: are targets classification-style (one-hot or single-class)? */
+    private _isClassification;
+    /** Compute classification metrics from predictions vs targets. */
+    private _computeMetricsArray;
+}
+
+declare class LRScheduler {
+    stepDecay(lr: number, epoch: number, dropRate: number, epochsDrop: number): number;
+    exponentialDecay(lr: number, epoch: number, decayRate: number): number;
+    plateauDecay(lr: number, currentLoss: number, history: number[], patience: number, factor: number): number;
+    cosineAnnealing(lr: number, epoch: number, maxEpochs: number, minLr?: number): number;
+}
+
+interface Serializable {
+    getWeights(): number[];
+    setWeights(weights: number[]): void;
+}
+declare class ModelSaver {
+    static toJSON(model: Serializable): string;
+    static fromJSON(model: Serializable, json: string): void;
+    static saveToFile(model: Serializable, path: string, writeFn: (path: string, data: string) => void): void;
+    static loadFromFile(model: Serializable, path: string, readFn: (path: string) => string): void;
+}
+
+declare function validateArray(arr: unknown, expectedLength: number, methodName: string): asserts arr is number[];
+declare function validateArrayMinLength(arr: unknown, minLength: number, methodName: string): asserts arr is number[];
+declare function validate2DArray(arr: unknown, expectedRows: number, expectedCols: number, methodName: string): asserts arr is number[][];
+declare function validateNumber(value: unknown, methodName: string): asserts value is number;
+
+export { type Activation, Adam, AttentionHead, BatchNorm, BiasVector, ClipOptimizer, ClippedOptimizerFactory, Conv1D, DataLoader, type DataPair, Dropout, EmbeddingMatrix, GRULayer, LRScheduler, LSTMLayer, Layer, LayerNorm, ModelSaver, Momentum, MultiHeadAttention, Network, NetworkLSTM, type NetworkLSTMOptions, NetworkN, type NetworkNOptions, NetworkTransformer, type NetworkTransformerOptions, NetworkTransformerRL, type NetworkTransformerRLOptions, Neuron, NeuronN, type Optimizer, type OptimizerFactory, SGD, type Serializable, type TrainDataset, type TrainMetrics, type TrainableNetwork, type TrainableNetworkWithWeights, Trainer, type TrainerOptions, TransformerBlock, type TransformerBlockOptions, WeightMatrix, crossEntropy, crossEntropyDelta, crossEntropyDeltaRaw, defaultOptimizer, elu, leakyRelu, linear, makeElu, makeLeakyRelu, matMul, mse, mseDelta, relu, sigmoid, softmax, softmaxBackward, tanh, transpose, validate2DArray, validateArray, validateArrayMinLength, validateNumber };