@dniskav/neuron 0.1.6 → 0.2.1

package/README.md

@@ -14,6 +14,13 @@ A minimal, dependency-free neural network library built from scratch in TypeScri
 | `NetworkN` | Deep network of arbitrary depth. Define your architecture as `[inputs, ...hidden, outputs]`. |
 | `LSTMLayer` | Recurrent layer with persistent hidden and cell state. Learns sequences via BPTT. |
 | `NetworkLSTM` | Wraps an `LSTMLayer` + dense layers. Maintains memory across steps within an episode. |
+| `NetworkTransformer` | Full token-classification Transformer: embeddings → N blocks → per-token logits. |
+| `TransformerBlock` | One Transformer block: multi-head attention + FFN + LayerNorm × 2 with residuals. |
+| `MultiHeadAttention` | N parallel attention heads concatenated and projected to `d_model`. |
+| `AttentionHead` | Single scaled dot-product self-attention head (Q / K / V projections + backprop). |
+| `LayerNorm` | Layer normalization with learnable γ / β per feature. |
+| `WeightMatrix` | 2D weight matrix with per-scalar Adam optimizers. Optional per-element gradient clipping via `update(dW, lr, clipValue)`. |
+| `EmbeddingMatrix` | Lookup-table embedding matrix with SGD updates. |
 | `sigmoid` `relu` `tanh` `linear` | Built-in activation functions. |
 | `SGD` `Momentum` `Adam` | Optimizers. Each instance tracks its own state per weight. |
 | `mse` `crossEntropy` | Loss functions for evaluation and logging. |
@@ -230,10 +237,44 @@ npm run dev     # watch mode
 
 If you are an AI agent or LLM working with this codebase, read [AGENTS.md](AGENTS.md) first. It contains the full class hierarchy, design constraints, and what this library does not do.
 
+### NetworkTransformer — self-attention over sequences
+
+```ts
+import { NetworkTransformer } from "@dniskav/neuron";
+
+// Sudoku solver: 81 cells (tokens), values 0–9, predict digit 1–9 per cell
+const net = new NetworkTransformer(81, {
+  vocabSize: 10,   // digits 0–9
+  d_model:   64,   // embedding / hidden dimension
+  nHeads:    4,    // attention heads (d_k = d_model / nHeads = 16)
+  d_ff:      128,  // FFN hidden size
+  nBlocks:   4,    // number of transformer blocks
+  nClasses:  9,    // output classes per token (digits 1–9)
+});
+
+// tokens: 81 cell values (0 = empty)
+const puzzle   = [5,3,0, 0,7,0, 0,0,0, ...];
+const targets  = [...];   // 81*9 one-hot values
+const mask     = puzzle.map(v => v === 0);   // only train on empty cells
+
+const loss = net.train(puzzle, targets, 0.001, mask);
+// loss is cross-entropy (not MSE) — decreases from ~2.2 toward 0 as training progresses
+const logits = net.predict(puzzle);   // 729 logits (81 × 9)
+
+// Attention weights from all blocks for visualization
+const weights = net.getAttentionWeights();
+// weights[blockIdx][headIdx]  → seqLen × seqLen matrix
+```
+
+Each head in each block learns a different type of relationship (row, column,
+3×3 box). The network figures this out by itself through training.
+
 ## Possible improvements
 
-1. **Support for batches** in training to improve efficiency.
-2. **Improve documentation** with more advanced examples and use cases.
+1. **Support for batches** in training to improve efficiency and gradient stability.
+2. **Global gradient norm clipping** — `WeightMatrix.update` supports per-element clipping; a utility to clip across all matrices by total norm would be more principled.
+3. **Learning rate warmup** — standard practice for Transformers; ramp LR from 0 to target over the first N steps.
+4. **Pre-norm architecture** — LayerNorm before the residual add (instead of after) is more stable for deep stacks.
 
 ## License
 

package/dist/index.d.mts

@@ -165,10 +165,119 @@ declare class NetworkLSTM {
     setWeights(data: ReturnType<NetworkLSTM["getWeights"]>): void;
 }
 
+declare function matMul(A: number[][], B: number[][]): number[][];
+declare function transpose(A: number[][]): number[][];
+declare function softmax(row: number[]): number[];
+declare function softmaxBackward(dS: number[], s: number[]): number[];
+declare class WeightMatrix {
+    W: number[][];
+    private opts;
+    constructor(rows: number, cols: number);
+    update(dW: number[][], lr: number, clipValue?: number): void;
+}
+declare class EmbeddingMatrix {
+    W: number[][];
+    constructor(vocabSize: number, d_model: number);
+    get(idx: number): number[];
+    update(idx: number, grad: number[], lr: number): void;
+}
+
+declare class AttentionHead {
+    readonly d_k: number;
+    readonly d_v: number;
+    Wq: WeightMatrix;
+    Wk: WeightMatrix;
+    Wv: WeightMatrix;
+    private cache;
+    constructor(d_model: number, d_k: number, d_v: number);
+    predict(X: number[][]): number[][];
+    backward(dOut: number[][], lr: number): number[][];
+    getAttentionWeights(): number[][] | null;
+}
+
+declare class MultiHeadAttention {
+    readonly nHeads: number;
+    readonly d_model: number;
+    readonly d_k: number;
+    heads: AttentionHead[];
+    Wo: WeightMatrix;
+    private _concat;
+    constructor(d_model: number, nHeads: number);
+    predict(X: number[][]): number[][];
+    backward(dOut: number[][], lr: number): number[][];
+    getAttentionWeights(): (number[][] | null)[];
+}
+
+declare class LayerNorm {
+    gamma: number[];
+    beta: number[];
+    private readonly eps;
+    private _cache;
+    constructor(dim: number);
+    resetCache(seqLen: number): void;
+    predictOne(x: number[], pos: number): number[];
+    backwardOne(dOut: number[], pos: number, lr: number): number[];
+}
+
+interface TransformerBlockOptions {
+    d_model: number;
+    nHeads: number;
+    d_ff: number;
+}
+declare class TransformerBlock {
+    readonly d_model: number;
+    readonly d_ff: number;
+    attn: MultiHeadAttention;
+    norm1: LayerNorm;
+    norm2: LayerNorm;
+    ff1: WeightMatrix;
+    ff2: WeightMatrix;
+    b1: number[];
+    b2: number[];
+    private b1Opts;
+    private b2Opts;
+    private _X;
+    private _attnOut;
+    private _h1;
+    private _ff1Pre;
+    private _ff1Out;
+    private _ff2Out;
+    constructor({ d_model, nHeads, d_ff }: TransformerBlockOptions);
+    predict(X: number[][]): number[][];
+    backward(dOut: number[][], lr: number): number[][];
+    getAttentionWeights(): (number[][] | null)[];
+}
+
+interface NetworkTransformerOptions {
+    vocabSize?: number;
+    d_model?: number;
+    nHeads?: number;
+    d_ff?: number;
+    nBlocks?: number;
+    nClasses?: number;
+}
+declare class NetworkTransformer {
+    readonly seqLen: number;
+    readonly vocabSize: number;
+    readonly d_model: number;
+    readonly nClasses: number;
+    tokenEmb: EmbeddingMatrix;
+    posEmb: EmbeddingMatrix;
+    blocks: TransformerBlock[];
+    outputProj: WeightMatrix;
+    outputBias: number[];
+    private outBiasOpts;
+    constructor(seqLen: number, options?: NetworkTransformerOptions);
+    predict(tokens: number[]): number[];
+    train(tokens: number[], targets: number[], lr: number, mask?: boolean[]): number;
+    getAttentionWeights(): (number[][] | null)[][];
+    private _forward;
+}
+
 declare function mse(predicted: number[], actual: number[]): number;
 declare function crossEntropy(predicted: number[], actual: number[]): number;
 declare function mseDelta(predicted: number, actual: number): number;
 declare function crossEntropyDelta(predicted: number, actual: number): number;
 declare function crossEntropyDeltaRaw(predicted: number, actual: number): number;
 
-export { type Activation, Adam, LSTMLayer, Layer, Momentum, Network, NetworkLSTM, type NetworkLSTMOptions, NetworkN, type NetworkNOptions, Neuron, NeuronN, type Optimizer, type OptimizerFactory, SGD, crossEntropy, crossEntropyDelta, crossEntropyDeltaRaw, elu, leakyRelu, linear, makeElu, makeLeakyRelu, mse, mseDelta, relu, sigmoid, tanh };
+export { type Activation, Adam, AttentionHead, EmbeddingMatrix, LSTMLayer, Layer, LayerNorm, Momentum, MultiHeadAttention, Network, NetworkLSTM, type NetworkLSTMOptions, NetworkN, type NetworkNOptions, NetworkTransformer, type NetworkTransformerOptions, Neuron, NeuronN, type Optimizer, type OptimizerFactory, SGD, TransformerBlock, type TransformerBlockOptions, WeightMatrix, crossEntropy, crossEntropyDelta, crossEntropyDeltaRaw, elu, leakyRelu, linear, makeElu, makeLeakyRelu, matMul, mse, mseDelta, relu, sigmoid, softmax, softmaxBackward, tanh, transpose };

package/dist/index.d.ts

@@ -165,10 +165,119 @@ declare class NetworkLSTM {
     setWeights(data: ReturnType<NetworkLSTM["getWeights"]>): void;
 }
 
+declare function matMul(A: number[][], B: number[][]): number[][];
+declare function transpose(A: number[][]): number[][];
+declare function softmax(row: number[]): number[];
+declare function softmaxBackward(dS: number[], s: number[]): number[];
+declare class WeightMatrix {
+    W: number[][];
+    private opts;
+    constructor(rows: number, cols: number);
+    update(dW: number[][], lr: number, clipValue?: number): void;
+}
+declare class EmbeddingMatrix {
+    W: number[][];
+    constructor(vocabSize: number, d_model: number);
+    get(idx: number): number[];
+    update(idx: number, grad: number[], lr: number): void;
+}
+
+declare class AttentionHead {
+    readonly d_k: number;
+    readonly d_v: number;
+    Wq: WeightMatrix;
+    Wk: WeightMatrix;
+    Wv: WeightMatrix;
+    private cache;
+    constructor(d_model: number, d_k: number, d_v: number);
+    predict(X: number[][]): number[][];
+    backward(dOut: number[][], lr: number): number[][];
+    getAttentionWeights(): number[][] | null;
+}
+
+declare class MultiHeadAttention {
+    readonly nHeads: number;
+    readonly d_model: number;
+    readonly d_k: number;
+    heads: AttentionHead[];
+    Wo: WeightMatrix;
+    private _concat;
+    constructor(d_model: number, nHeads: number);
+    predict(X: number[][]): number[][];
+    backward(dOut: number[][], lr: number): number[][];
+    getAttentionWeights(): (number[][] | null)[];
+}
+
+declare class LayerNorm {
+    gamma: number[];
+    beta: number[];
+    private readonly eps;
+    private _cache;
+    constructor(dim: number);
+    resetCache(seqLen: number): void;
+    predictOne(x: number[], pos: number): number[];
+    backwardOne(dOut: number[], pos: number, lr: number): number[];
+}
+
+interface TransformerBlockOptions {
+    d_model: number;
+    nHeads: number;
+    d_ff: number;
+}
+declare class TransformerBlock {
+    readonly d_model: number;
+    readonly d_ff: number;
+    attn: MultiHeadAttention;
+    norm1: LayerNorm;
+    norm2: LayerNorm;
+    ff1: WeightMatrix;
+    ff2: WeightMatrix;
+    b1: number[];
+    b2: number[];
+    private b1Opts;
+    private b2Opts;
+    private _X;
+    private _attnOut;
+    private _h1;
+    private _ff1Pre;
+    private _ff1Out;
+    private _ff2Out;
+    constructor({ d_model, nHeads, d_ff }: TransformerBlockOptions);
+    predict(X: number[][]): number[][];
+    backward(dOut: number[][], lr: number): number[][];
+    getAttentionWeights(): (number[][] | null)[];
+}
+
+interface NetworkTransformerOptions {
+    vocabSize?: number;
+    d_model?: number;
+    nHeads?: number;
+    d_ff?: number;
+    nBlocks?: number;
+    nClasses?: number;
+}
+declare class NetworkTransformer {
+    readonly seqLen: number;
+    readonly vocabSize: number;
+    readonly d_model: number;
+    readonly nClasses: number;
+    tokenEmb: EmbeddingMatrix;
+    posEmb: EmbeddingMatrix;
+    blocks: TransformerBlock[];
+    outputProj: WeightMatrix;
+    outputBias: number[];
+    private outBiasOpts;
+    constructor(seqLen: number, options?: NetworkTransformerOptions);
+    predict(tokens: number[]): number[];
+    train(tokens: number[], targets: number[], lr: number, mask?: boolean[]): number;
+    getAttentionWeights(): (number[][] | null)[][];
+    private _forward;
+}
+
 declare function mse(predicted: number[], actual: number[]): number;
 declare function crossEntropy(predicted: number[], actual: number[]): number;
 declare function mseDelta(predicted: number, actual: number): number;
 declare function crossEntropyDelta(predicted: number, actual: number): number;
 declare function crossEntropyDeltaRaw(predicted: number, actual: number): number;
 
-export { type Activation, Adam, LSTMLayer, Layer, Momentum, Network, NetworkLSTM, type NetworkLSTMOptions, NetworkN, type NetworkNOptions, Neuron, NeuronN, type Optimizer, type OptimizerFactory, SGD, crossEntropy, crossEntropyDelta, crossEntropyDeltaRaw, elu, leakyRelu, linear, makeElu, makeLeakyRelu, mse, mseDelta, relu, sigmoid, tanh };
+export { type Activation, Adam, AttentionHead, EmbeddingMatrix, LSTMLayer, Layer, LayerNorm, Momentum, MultiHeadAttention, Network, NetworkLSTM, type NetworkLSTMOptions, NetworkN, type NetworkNOptions, NetworkTransformer, type NetworkTransformerOptions, Neuron, NeuronN, type Optimizer, type OptimizerFactory, SGD, TransformerBlock, type TransformerBlockOptions, WeightMatrix, crossEntropy, crossEntropyDelta, crossEntropyDeltaRaw, elu, leakyRelu, linear, makeElu, makeLeakyRelu, matMul, mse, mseDelta, relu, sigmoid, softmax, softmaxBackward, tanh, transpose };