@dniskav/neuron 0.2.5 → 0.2.7

package/README.md

@@ -3,6 +3,28 @@
 
 A minimal, dependency-free neural network library built from scratch in TypeScript. Designed for learning and experimentation — every line of math is readable.
 
+Each class is a building block for the next: from a single neuron to a full Transformer with causal attention.
+
+```mermaid
+graph TD
+    A["Neuron\n1 input · 1 weight · 1 bias"]
+    B["NeuronN\nN inputs · Xavier init · configurable activation"]
+    C["Layer\ngroup of NeuronN sharing the same inputs"]
+    D["Network\nhidden + output · backprop"]
+    E["NetworkN\narbitrary depth · define as [inputs, ...hidden, outputs]"]
+    F["LSTMLayer\nrecurrent · hidden + cell state · BPTT"]
+    G["NetworkLSTM\nLSTM + dense layers · sequence memory"]
+    H["AttentionHead\nQ · K · V · scaled dot-product"]
+    I["MultiHeadAttention\nN heads in parallel"]
+    J["TransformerBlock\nattention + FFN + LayerNorm × 2 + residuals"]
+    K["NetworkTransformer\nembeddings → blocks → per-token logits"]
+    L["NetworkTransformerRL\ncontinuous projection → causal attention → Q-values"]
+
+    A --> B --> C --> D --> E
+    E --> F --> G
+    E --> H --> I --> J --> K --> L
+```
+
 ## What's inside
 
 | Export | Description |
@@ -21,9 +43,11 @@ A minimal, dependency-free neural network library built from scratch in TypeScri
 | `AttentionHead` | Single scaled dot-product self-attention head (Q / K / V projections + backprop). |
 | `LayerNorm` | Layer normalization with learnable γ / β per feature. |
 | `WeightMatrix` | 2D weight matrix with per-scalar Adam optimizers. Optional per-element gradient clipping via `update(dW, lr, clipValue)`. |
+| `BiasVector` | 1D bias vector with per-scalar Adam optimizers. Companion to `WeightMatrix` for bias terms. |
 | `EmbeddingMatrix` | Lookup-table embedding matrix with SGD updates. |
 | `sigmoid` `relu` `tanh` `linear` | Built-in activation functions. |
-| `SGD` `Momentum` `Adam` | Optimizers. Each instance tracks its own state per weight. |
+| `SGD` `Momentum` `Adam` `ClipOptimizer` | Optimizers. Each instance tracks its own state per weight. `ClipOptimizer` wraps any optimizer with gradient clipping. |
+| `defaultOptimizer` | Default `OptimizerFactory` (`() => new SGD()`). Shared across `NeuronN`, `Layer`, `NetworkN`, `NetworkLSTM`. |
 | `mse` `crossEntropy` | Loss functions for evaluation and logging. |
 | `mseDelta` `crossEntropyDelta` | Output-layer delta functions for use with `trainWithDeltas`. |
 
@@ -83,8 +107,8 @@ for (let epoch = 0; epoch < 5000; epoch++) {
   }
 }
 
-console.log(net.predict([0, 1])); // ~0.97
-console.log(net.predict([1, 1])); // ~0.03
+console.log(net.predict([0, 1])[0]); // ~0.97
+console.log(net.predict([1, 1])[0]); // ~0.03
 ```
 
 ### NetworkN — deep network with custom architecture
@@ -307,6 +331,32 @@ const attnWeights = net.getAttentionWeights();
 // attnWeights[blockIdx][headIdx] → seqLen × seqLen matrix
 ```
 
+## Changelog
+
+### v0.2.7
+- **Docs:** Added architecture diagram to README — visual progression from `Neuron` to `NetworkTransformerRL`
+
+### v0.2.6
+- **Fix:** `Network.predict` now returns `number[]` (consistent with all other network classes)
+- **Fix:** `Network.train` now uses the configured optimizer and `activation.dfn()` instead of hardcoded SGD and sigmoid derivative
+- **Fix:** `LayerNorm.backwardOne` now correctly uses pre-update γ when computing the input gradient
+- **Fix:** LSTM and GRU gate initialization corrected from He (`√(2/n)`) to Xavier fan-in+out (`√(2/(fanIn+fanOut))`), matching the sigmoid/tanh activations used in those gates
+- **New:** `BiasVector` — 1D counterpart to `WeightMatrix` with per-scalar Adam optimizers; replaces repeated `number[] + Adam[]` pairs in `TransformerBlock`, `NetworkTransformer`, and `NetworkTransformerRL`
+- **New:** `defaultOptimizer` exported from `optimizers.ts` — single source of truth for the default `() => new SGD()` factory
+- **Refactor:** `NetworkN.train` and `trainWithDeltas` share extracted `_forwardAll()` and `_backpropLayers()` internals — eliminates ~50 lines of duplication
+- **Refactor:** `Transformer` backward methods now throw descriptive errors instead of crashing with a cryptic `TypeError` when called before `predict()`
+- **Refactor:** `NetworkTransformer.setWeights()` and `NetworkTransformerRL.setWeightsFlat()` use each component's own `setWeights()` instead of direct `.W` mutation
+
+### v0.2.5
+- Unified optimizer factories for `LSTMLayer`, `GRULayer`, `Conv1D` (per-scalar Adam/Momentum/SGD)
+- `NetworkN`: residual connections (`residual` option) and dropout (`dropoutRate`)
+- `Conv1D`: multi-channel input (`inputChannels`)
+- `NetworkTransformerRL`: configurable pooling (`avg` / `max` / `last` / `weighted`)
+- `Trainer`: weight decay, early stopping, classification metrics, gradient clipping support
+- `DataLoader`: validation split (`validationSplit` + `getValidationData()`)
+- `ModelSaver`: universal serialization via flat `getWeights()`/`setWeights()` for all classes
+- Gradient check test suite (`tests/GradientCheck.test.ts`)
+
 ## Possible improvements
 
 1. **Support for batches** in training to improve efficiency and gradient stability.

package/dist/index.d.mts

@@ -25,6 +25,7 @@ interface Optimizer {
     step(weight: number, gradient: number, lr: number): number;
 }
 type OptimizerFactory = () => Optimizer;
+declare const defaultOptimizer: OptimizerFactory;
 declare class SGD implements Optimizer {
     step(weight: number, gradient: number, lr: number): number;
 }
@@ -73,7 +74,7 @@ declare class Network {
     hiddenLayer: Layer;
     outputLayer: Layer;
     constructor(nInputs: number, nHidden: number, nOutputs: number);
-    predict(inputs: number[]): number;
+    predict(inputs: number[]): number[];
     train(inputs: number[], target: number, lr: number): number;
     getWeights(): number[];
     setWeights(weights: number[]): void;
@@ -97,6 +98,8 @@ declare class NetworkN {
     getWeights(): number[];
     setWeights(weights: number[]): void;
     private _shouldResidual;
+    private _forwardAll;
+    private _backpropLayers;
 }
 
 declare class Gate$1 {
@@ -198,6 +201,14 @@ declare class WeightMatrix {
     getWeights(): number[];
     setWeights(weights: number[]): void;
 }
+declare class BiasVector {
+    values: number[];
+    private opts;
+    constructor(size: number);
+    update(grad: number[], lr: number): void;
+    getWeights(): number[];
+    setWeights(weights: number[]): void;
+}
 declare class EmbeddingMatrix {
     W: number[][];
     constructor(vocabSize: number, d_model: number);
@@ -266,10 +277,8 @@ declare class TransformerBlock {
     norm2: LayerNorm;
     ff1: WeightMatrix;
     ff2: WeightMatrix;
-    b1: number[];
-    b2: number[];
-    private b1Opts;
-    private b2Opts;
+    b1: BiasVector;
+    b2: BiasVector;
     private _X;
     private _attnOut;
     private _h1;
@@ -301,8 +310,7 @@ declare class NetworkTransformer {
     posEmb: EmbeddingMatrix;
     blocks: TransformerBlock[];
     outputProj: WeightMatrix;
-    outputBias: number[];
-    private outBiasOpts;
+    outputBias: BiasVector;
     constructor(seqLen: number, options?: NetworkTransformerOptions);
     predict(tokens: number[]): number[];
     train(tokens: number[], targets: number[], lr: number, mask?: boolean[]): number;
@@ -328,8 +336,7 @@ declare class NetworkTransformerRL {
     inputProj: WeightMatrix;
     blocks: TransformerBlock[];
     outputProj: WeightMatrix;
-    outputBias: number[];
-    private outBiasOpts;
+    outputBias: BiasVector;
     private _projected;
     private _pooling;
     private _argmax;
@@ -586,4 +593,4 @@ declare function validateArrayMinLength(arr: unknown, minLength: number, methodN
 declare function validate2DArray(arr: unknown, expectedRows: number, expectedCols: number, methodName: string): asserts arr is number[][];
 declare function validateNumber(value: unknown, methodName: string): asserts value is number;
 
-export { type Activation, Adam, AttentionHead, BatchNorm, ClipOptimizer, ClippedOptimizerFactory, Conv1D, DataLoader, type DataPair, Dropout, EmbeddingMatrix, GRULayer, LRScheduler, LSTMLayer, Layer, LayerNorm, ModelSaver, Momentum, MultiHeadAttention, Network, NetworkLSTM, type NetworkLSTMOptions, NetworkN, type NetworkNOptions, NetworkTransformer, type NetworkTransformerOptions, NetworkTransformerRL, type NetworkTransformerRLOptions, Neuron, NeuronN, type Optimizer, type OptimizerFactory, SGD, type Serializable, type TrainDataset, type TrainMetrics, type TrainableNetwork, type TrainableNetworkWithWeights, Trainer, type TrainerOptions, TransformerBlock, type TransformerBlockOptions, WeightMatrix, crossEntropy, crossEntropyDelta, crossEntropyDeltaRaw, elu, leakyRelu, linear, makeElu, makeLeakyRelu, matMul, mse, mseDelta, relu, sigmoid, softmax, softmaxBackward, tanh, transpose, validate2DArray, validateArray, validateArrayMinLength, validateNumber };
+export { type Activation, Adam, AttentionHead, BatchNorm, BiasVector, ClipOptimizer, ClippedOptimizerFactory, Conv1D, DataLoader, type DataPair, Dropout, EmbeddingMatrix, GRULayer, LRScheduler, LSTMLayer, Layer, LayerNorm, ModelSaver, Momentum, MultiHeadAttention, Network, NetworkLSTM, type NetworkLSTMOptions, NetworkN, type NetworkNOptions, NetworkTransformer, type NetworkTransformerOptions, NetworkTransformerRL, type NetworkTransformerRLOptions, Neuron, NeuronN, type Optimizer, type OptimizerFactory, SGD, type Serializable, type TrainDataset, type TrainMetrics, type TrainableNetwork, type TrainableNetworkWithWeights, Trainer, type TrainerOptions, TransformerBlock, type TransformerBlockOptions, WeightMatrix, crossEntropy, crossEntropyDelta, crossEntropyDeltaRaw, defaultOptimizer, elu, leakyRelu, linear, makeElu, makeLeakyRelu, matMul, mse, mseDelta, relu, sigmoid, softmax, softmaxBackward, tanh, transpose, validate2DArray, validateArray, validateArrayMinLength, validateNumber };

package/dist/index.d.ts

@@ -25,6 +25,7 @@ interface Optimizer {
     step(weight: number, gradient: number, lr: number): number;
 }
 type OptimizerFactory = () => Optimizer;
+declare const defaultOptimizer: OptimizerFactory;
 declare class SGD implements Optimizer {
     step(weight: number, gradient: number, lr: number): number;
 }
@@ -73,7 +74,7 @@ declare class Network {
     hiddenLayer: Layer;
     outputLayer: Layer;
     constructor(nInputs: number, nHidden: number, nOutputs: number);
-    predict(inputs: number[]): number;
+    predict(inputs: number[]): number[];
     train(inputs: number[], target: number, lr: number): number;
     getWeights(): number[];
     setWeights(weights: number[]): void;
@@ -97,6 +98,8 @@ declare class NetworkN {
     getWeights(): number[];
     setWeights(weights: number[]): void;
     private _shouldResidual;
+    private _forwardAll;
+    private _backpropLayers;
 }
 
 declare class Gate$1 {
@@ -198,6 +201,14 @@ declare class WeightMatrix {
     getWeights(): number[];
     setWeights(weights: number[]): void;
 }
+declare class BiasVector {
+    values: number[];
+    private opts;
+    constructor(size: number);
+    update(grad: number[], lr: number): void;
+    getWeights(): number[];
+    setWeights(weights: number[]): void;
+}
 declare class EmbeddingMatrix {
     W: number[][];
     constructor(vocabSize: number, d_model: number);
@@ -266,10 +277,8 @@ declare class TransformerBlock {
     norm2: LayerNorm;
     ff1: WeightMatrix;
     ff2: WeightMatrix;
-    b1: number[];
-    b2: number[];
-    private b1Opts;
-    private b2Opts;
+    b1: BiasVector;
+    b2: BiasVector;
     private _X;
     private _attnOut;
     private _h1;
@@ -301,8 +310,7 @@ declare class NetworkTransformer {
     posEmb: EmbeddingMatrix;
     blocks: TransformerBlock[];
     outputProj: WeightMatrix;
-    outputBias: number[];
-    private outBiasOpts;
+    outputBias: BiasVector;
     constructor(seqLen: number, options?: NetworkTransformerOptions);
     predict(tokens: number[]): number[];
     train(tokens: number[], targets: number[], lr: number, mask?: boolean[]): number;
@@ -328,8 +336,7 @@ declare class NetworkTransformerRL {
     inputProj: WeightMatrix;
     blocks: TransformerBlock[];
     outputProj: WeightMatrix;
-    outputBias: number[];
-    private outBiasOpts;
+    outputBias: BiasVector;
     private _projected;
     private _pooling;
     private _argmax;
@@ -586,4 +593,4 @@ declare function validateArrayMinLength(arr: unknown, minLength: number, methodN
 declare function validate2DArray(arr: unknown, expectedRows: number, expectedCols: number, methodName: string): asserts arr is number[][];
 declare function validateNumber(value: unknown, methodName: string): asserts value is number;
 
-export { type Activation, Adam, AttentionHead, BatchNorm, ClipOptimizer, ClippedOptimizerFactory, Conv1D, DataLoader, type DataPair, Dropout, EmbeddingMatrix, GRULayer, LRScheduler, LSTMLayer, Layer, LayerNorm, ModelSaver, Momentum, MultiHeadAttention, Network, NetworkLSTM, type NetworkLSTMOptions, NetworkN, type NetworkNOptions, NetworkTransformer, type NetworkTransformerOptions, NetworkTransformerRL, type NetworkTransformerRLOptions, Neuron, NeuronN, type Optimizer, type OptimizerFactory, SGD, type Serializable, type TrainDataset, type TrainMetrics, type TrainableNetwork, type TrainableNetworkWithWeights, Trainer, type TrainerOptions, TransformerBlock, type TransformerBlockOptions, WeightMatrix, crossEntropy, crossEntropyDelta, crossEntropyDeltaRaw, elu, leakyRelu, linear, makeElu, makeLeakyRelu, matMul, mse, mseDelta, relu, sigmoid, softmax, softmaxBackward, tanh, transpose, validate2DArray, validateArray, validateArrayMinLength, validateNumber };
+export { type Activation, Adam, AttentionHead, BatchNorm, BiasVector, ClipOptimizer, ClippedOptimizerFactory, Conv1D, DataLoader, type DataPair, Dropout, EmbeddingMatrix, GRULayer, LRScheduler, LSTMLayer, Layer, LayerNorm, ModelSaver, Momentum, MultiHeadAttention, Network, NetworkLSTM, type NetworkLSTMOptions, NetworkN, type NetworkNOptions, NetworkTransformer, type NetworkTransformerOptions, NetworkTransformerRL, type NetworkTransformerRLOptions, Neuron, NeuronN, type Optimizer, type OptimizerFactory, SGD, type Serializable, type TrainDataset, type TrainMetrics, type TrainableNetwork, type TrainableNetworkWithWeights, Trainer, type TrainerOptions, TransformerBlock, type TransformerBlockOptions, WeightMatrix, crossEntropy, crossEntropyDelta, crossEntropyDeltaRaw, defaultOptimizer, elu, leakyRelu, linear, makeElu, makeLeakyRelu, matMul, mse, mseDelta, relu, sigmoid, softmax, softmaxBackward, tanh, transpose, validate2DArray, validateArray, validateArrayMinLength, validateNumber };