@dniskav/neuron 0.2.5 → 0.2.6

package/README.md

@@ -21,9 +21,11 @@ A minimal, dependency-free neural network library built from scratch in TypeScri
 | `AttentionHead` | Single scaled dot-product self-attention head (Q / K / V projections + backprop). |
 | `LayerNorm` | Layer normalization with learnable γ / β per feature. |
 | `WeightMatrix` | 2D weight matrix with per-scalar Adam optimizers. Optional per-element gradient clipping via `update(dW, lr, clipValue)`. |
+| `BiasVector` | 1D bias vector with per-scalar Adam optimizers. Companion to `WeightMatrix` for bias terms. |
 | `EmbeddingMatrix` | Lookup-table embedding matrix with SGD updates. |
 | `sigmoid` `relu` `tanh` `linear` | Built-in activation functions. |
-| `SGD` `Momentum` `Adam` | Optimizers. Each instance tracks its own state per weight. |
+| `SGD` `Momentum` `Adam` `ClipOptimizer` | Optimizers. Each instance tracks its own state per weight. `ClipOptimizer` wraps any optimizer with gradient clipping. |
+| `defaultOptimizer` | Default `OptimizerFactory` (`() => new SGD()`). Shared across `NeuronN`, `Layer`, `NetworkN`, `NetworkLSTM`. |
 | `mse` `crossEntropy` | Loss functions for evaluation and logging. |
 | `mseDelta` `crossEntropyDelta` | Output-layer delta functions for use with `trainWithDeltas`. |
 
@@ -83,8 +85,8 @@ for (let epoch = 0; epoch < 5000; epoch++) {
   }
 }
 
-console.log(net.predict([0, 1])); // ~0.97
-console.log(net.predict([1, 1])); // ~0.03
+console.log(net.predict([0, 1])[0]); // ~0.97
+console.log(net.predict([1, 1])[0]); // ~0.03
 ```
 
 ### NetworkN — deep network with custom architecture
@@ -307,6 +309,29 @@ const attnWeights = net.getAttentionWeights();
 // attnWeights[blockIdx][headIdx] → seqLen × seqLen matrix
 ```
 
+## Changelog
+
+### v0.2.6
+- **Fix:** `Network.predict` now returns `number[]` (consistent with all other network classes)
+- **Fix:** `Network.train` now uses the configured optimizer and `activation.dfn()` instead of hardcoded SGD and sigmoid derivative
+- **Fix:** `LayerNorm.backwardOne` now correctly uses pre-update γ when computing the input gradient
+- **Fix:** LSTM and GRU gate initialization corrected from He (`√(2/n)`) to Xavier fan-in+out (`√(2/(fanIn+fanOut))`), matching the sigmoid/tanh activations used in those gates
+- **New:** `BiasVector` — 1D counterpart to `WeightMatrix` with per-scalar Adam optimizers; replaces repeated `number[] + Adam[]` pairs in `TransformerBlock`, `NetworkTransformer`, and `NetworkTransformerRL`
+- **New:** `defaultOptimizer` exported from `optimizers.ts` — single source of truth for the default `() => new SGD()` factory
+- **Refactor:** `NetworkN.train` and `trainWithDeltas` share extracted `_forwardAll()` and `_backpropLayers()` internals — eliminates ~50 lines of duplication
+- **Refactor:** `Transformer` backward methods now throw descriptive errors instead of crashing with a cryptic `TypeError` when called before `predict()`
+- **Refactor:** `NetworkTransformer.setWeights()` and `NetworkTransformerRL.setWeightsFlat()` use each component's own `setWeights()` instead of direct `.W` mutation
+
+### v0.2.5
+- Unified optimizer factories for `LSTMLayer`, `GRULayer`, `Conv1D` (per-scalar Adam/Momentum/SGD)
+- `NetworkN`: residual connections (`residual` option) and dropout (`dropoutRate`)
+- `Conv1D`: multi-channel input (`inputChannels`)
+- `NetworkTransformerRL`: configurable pooling (`avg` / `max` / `last` / `weighted`)
+- `Trainer`: weight decay, early stopping, classification metrics, gradient clipping support
+- `DataLoader`: validation split (`validationSplit` + `getValidationData()`)
+- `ModelSaver`: universal serialization via flat `getWeights()`/`setWeights()` for all classes
+- Gradient check test suite (`tests/GradientCheck.test.ts`)
+
 ## Possible improvements
 
 1. **Support for batches** in training to improve efficiency and gradient stability.

package/dist/index.d.mts

@@ -25,6 +25,7 @@ interface Optimizer {
     step(weight: number, gradient: number, lr: number): number;
 }
 type OptimizerFactory = () => Optimizer;
+declare const defaultOptimizer: OptimizerFactory;
 declare class SGD implements Optimizer {
     step(weight: number, gradient: number, lr: number): number;
 }
@@ -73,7 +74,7 @@ declare class Network {
     hiddenLayer: Layer;
     outputLayer: Layer;
     constructor(nInputs: number, nHidden: number, nOutputs: number);
-    predict(inputs: number[]): number;
+    predict(inputs: number[]): number[];
     train(inputs: number[], target: number, lr: number): number;
     getWeights(): number[];
     setWeights(weights: number[]): void;
@@ -97,6 +98,8 @@ declare class NetworkN {
     getWeights(): number[];
     setWeights(weights: number[]): void;
     private _shouldResidual;
+    private _forwardAll;
+    private _backpropLayers;
 }
 
 declare class Gate$1 {
@@ -198,6 +201,14 @@ declare class WeightMatrix {
     getWeights(): number[];
     setWeights(weights: number[]): void;
 }
+declare class BiasVector {
+    values: number[];
+    private opts;
+    constructor(size: number);
+    update(grad: number[], lr: number): void;
+    getWeights(): number[];
+    setWeights(weights: number[]): void;
+}
 declare class EmbeddingMatrix {
     W: number[][];
     constructor(vocabSize: number, d_model: number);
@@ -266,10 +277,8 @@ declare class TransformerBlock {
     norm2: LayerNorm;
     ff1: WeightMatrix;
     ff2: WeightMatrix;
-    b1: number[];
-    b2: number[];
-    private b1Opts;
-    private b2Opts;
+    b1: BiasVector;
+    b2: BiasVector;
     private _X;
     private _attnOut;
     private _h1;
@@ -301,8 +310,7 @@ declare class NetworkTransformer {
     posEmb: EmbeddingMatrix;
     blocks: TransformerBlock[];
     outputProj: WeightMatrix;
-    outputBias: number[];
-    private outBiasOpts;
+    outputBias: BiasVector;
     constructor(seqLen: number, options?: NetworkTransformerOptions);
     predict(tokens: number[]): number[];
     train(tokens: number[], targets: number[], lr: number, mask?: boolean[]): number;
@@ -328,8 +336,7 @@ declare class NetworkTransformerRL {
     inputProj: WeightMatrix;
     blocks: TransformerBlock[];
     outputProj: WeightMatrix;
-    outputBias: number[];
-    private outBiasOpts;
+    outputBias: BiasVector;
     private _projected;
     private _pooling;
     private _argmax;
@@ -586,4 +593,4 @@ declare function validateArrayMinLength(arr: unknown, minLength: number, methodN
 declare function validate2DArray(arr: unknown, expectedRows: number, expectedCols: number, methodName: string): asserts arr is number[][];
 declare function validateNumber(value: unknown, methodName: string): asserts value is number;
 
-export { type Activation, Adam, AttentionHead, BatchNorm, ClipOptimizer, ClippedOptimizerFactory, Conv1D, DataLoader, type DataPair, Dropout, EmbeddingMatrix, GRULayer, LRScheduler, LSTMLayer, Layer, LayerNorm, ModelSaver, Momentum, MultiHeadAttention, Network, NetworkLSTM, type NetworkLSTMOptions, NetworkN, type NetworkNOptions, NetworkTransformer, type NetworkTransformerOptions, NetworkTransformerRL, type NetworkTransformerRLOptions, Neuron, NeuronN, type Optimizer, type OptimizerFactory, SGD, type Serializable, type TrainDataset, type TrainMetrics, type TrainableNetwork, type TrainableNetworkWithWeights, Trainer, type TrainerOptions, TransformerBlock, type TransformerBlockOptions, WeightMatrix, crossEntropy, crossEntropyDelta, crossEntropyDeltaRaw, elu, leakyRelu, linear, makeElu, makeLeakyRelu, matMul, mse, mseDelta, relu, sigmoid, softmax, softmaxBackward, tanh, transpose, validate2DArray, validateArray, validateArrayMinLength, validateNumber };
+export { type Activation, Adam, AttentionHead, BatchNorm, BiasVector, ClipOptimizer, ClippedOptimizerFactory, Conv1D, DataLoader, type DataPair, Dropout, EmbeddingMatrix, GRULayer, LRScheduler, LSTMLayer, Layer, LayerNorm, ModelSaver, Momentum, MultiHeadAttention, Network, NetworkLSTM, type NetworkLSTMOptions, NetworkN, type NetworkNOptions, NetworkTransformer, type NetworkTransformerOptions, NetworkTransformerRL, type NetworkTransformerRLOptions, Neuron, NeuronN, type Optimizer, type OptimizerFactory, SGD, type Serializable, type TrainDataset, type TrainMetrics, type TrainableNetwork, type TrainableNetworkWithWeights, Trainer, type TrainerOptions, TransformerBlock, type TransformerBlockOptions, WeightMatrix, crossEntropy, crossEntropyDelta, crossEntropyDeltaRaw, defaultOptimizer, elu, leakyRelu, linear, makeElu, makeLeakyRelu, matMul, mse, mseDelta, relu, sigmoid, softmax, softmaxBackward, tanh, transpose, validate2DArray, validateArray, validateArrayMinLength, validateNumber };

package/dist/index.d.ts

@@ -25,6 +25,7 @@ interface Optimizer {
     step(weight: number, gradient: number, lr: number): number;
 }
 type OptimizerFactory = () => Optimizer;
+declare const defaultOptimizer: OptimizerFactory;
 declare class SGD implements Optimizer {
     step(weight: number, gradient: number, lr: number): number;
 }
@@ -73,7 +74,7 @@ declare class Network {
     hiddenLayer: Layer;
     outputLayer: Layer;
     constructor(nInputs: number, nHidden: number, nOutputs: number);
-    predict(inputs: number[]): number;
+    predict(inputs: number[]): number[];
     train(inputs: number[], target: number, lr: number): number;
     getWeights(): number[];
     setWeights(weights: number[]): void;
@@ -97,6 +98,8 @@ declare class NetworkN {
     getWeights(): number[];
     setWeights(weights: number[]): void;
     private _shouldResidual;
+    private _forwardAll;
+    private _backpropLayers;
 }
 
 declare class Gate$1 {
@@ -198,6 +201,14 @@ declare class WeightMatrix {
     getWeights(): number[];
     setWeights(weights: number[]): void;
 }
+declare class BiasVector {
+    values: number[];
+    private opts;
+    constructor(size: number);
+    update(grad: number[], lr: number): void;
+    getWeights(): number[];
+    setWeights(weights: number[]): void;
+}
 declare class EmbeddingMatrix {
     W: number[][];
     constructor(vocabSize: number, d_model: number);
@@ -266,10 +277,8 @@ declare class TransformerBlock {
     norm2: LayerNorm;
     ff1: WeightMatrix;
     ff2: WeightMatrix;
-    b1: number[];
-    b2: number[];
-    private b1Opts;
-    private b2Opts;
+    b1: BiasVector;
+    b2: BiasVector;
     private _X;
     private _attnOut;
     private _h1;
@@ -301,8 +310,7 @@ declare class NetworkTransformer {
     posEmb: EmbeddingMatrix;
     blocks: TransformerBlock[];
     outputProj: WeightMatrix;
-    outputBias: number[];
-    private outBiasOpts;
+    outputBias: BiasVector;
     constructor(seqLen: number, options?: NetworkTransformerOptions);
     predict(tokens: number[]): number[];
     train(tokens: number[], targets: number[], lr: number, mask?: boolean[]): number;
@@ -328,8 +336,7 @@ declare class NetworkTransformerRL {
     inputProj: WeightMatrix;
     blocks: TransformerBlock[];
     outputProj: WeightMatrix;
-    outputBias: number[];
-    private outBiasOpts;
+    outputBias: BiasVector;
     private _projected;
     private _pooling;
     private _argmax;
@@ -586,4 +593,4 @@ declare function validateArrayMinLength(arr: unknown, minLength: number, methodN
 declare function validate2DArray(arr: unknown, expectedRows: number, expectedCols: number, methodName: string): asserts arr is number[][];
 declare function validateNumber(value: unknown, methodName: string): asserts value is number;
 
-export { type Activation, Adam, AttentionHead, BatchNorm, ClipOptimizer, ClippedOptimizerFactory, Conv1D, DataLoader, type DataPair, Dropout, EmbeddingMatrix, GRULayer, LRScheduler, LSTMLayer, Layer, LayerNorm, ModelSaver, Momentum, MultiHeadAttention, Network, NetworkLSTM, type NetworkLSTMOptions, NetworkN, type NetworkNOptions, NetworkTransformer, type NetworkTransformerOptions, NetworkTransformerRL, type NetworkTransformerRLOptions, Neuron, NeuronN, type Optimizer, type OptimizerFactory, SGD, type Serializable, type TrainDataset, type TrainMetrics, type TrainableNetwork, type TrainableNetworkWithWeights, Trainer, type TrainerOptions, TransformerBlock, type TransformerBlockOptions, WeightMatrix, crossEntropy, crossEntropyDelta, crossEntropyDeltaRaw, elu, leakyRelu, linear, makeElu, makeLeakyRelu, matMul, mse, mseDelta, relu, sigmoid, softmax, softmaxBackward, tanh, transpose, validate2DArray, validateArray, validateArrayMinLength, validateNumber };
+export { type Activation, Adam, AttentionHead, BatchNorm, BiasVector, ClipOptimizer, ClippedOptimizerFactory, Conv1D, DataLoader, type DataPair, Dropout, EmbeddingMatrix, GRULayer, LRScheduler, LSTMLayer, Layer, LayerNorm, ModelSaver, Momentum, MultiHeadAttention, Network, NetworkLSTM, type NetworkLSTMOptions, NetworkN, type NetworkNOptions, NetworkTransformer, type NetworkTransformerOptions, NetworkTransformerRL, type NetworkTransformerRLOptions, Neuron, NeuronN, type Optimizer, type OptimizerFactory, SGD, type Serializable, type TrainDataset, type TrainMetrics, type TrainableNetwork, type TrainableNetworkWithWeights, Trainer, type TrainerOptions, TransformerBlock, type TransformerBlockOptions, WeightMatrix, crossEntropy, crossEntropyDelta, crossEntropyDeltaRaw, defaultOptimizer, elu, leakyRelu, linear, makeElu, makeLeakyRelu, matMul, mse, mseDelta, relu, sigmoid, softmax, softmaxBackward, tanh, transpose, validate2DArray, validateArray, validateArrayMinLength, validateNumber };