@dniskav/neuron 0.2.1 → 0.2.3

package/README.md

@@ -15,6 +15,7 @@ A minimal, dependency-free neural network library built from scratch in TypeScri
 | `LSTMLayer` | Recurrent layer with persistent hidden and cell state. Learns sequences via BPTT. |
 | `NetworkLSTM` | Wraps an `LSTMLayer` + dense layers. Maintains memory across steps within an episode. |
 | `NetworkTransformer` | Full token-classification Transformer: embeddings → N blocks → per-token logits. |
+| `NetworkTransformerRL` | Transformer for RL agents: continuous input projection → causal attention → Q-values. Remembers the last N steps. |
 | `TransformerBlock` | One Transformer block: multi-head attention + FFN + LayerNorm × 2 with residuals. |
 | `MultiHeadAttention` | N parallel attention heads concatenated and projected to `d_model`. |
 | `AttentionHead` | Single scaled dot-product self-attention head (Q / K / V projections + backprop). |
@@ -269,6 +270,43 @@ const weights = net.getAttentionWeights();
 Each head in each block learns a different type of relationship (row, column,
 3×3 box). The network figures this out by itself through training.
 
+### NetworkTransformerRL — Transformer for reinforcement learning
+
+`NetworkTransformerRL` uses causal self-attention over a sliding window of past states to output Q-values. Unlike `NetworkLSTM`, the agent attends to specific past moments rather than compressing them into a single hidden vector.
+
+```ts
+import { NetworkTransformerRL } from "@dniskav/neuron";
+
+// Agent sees the last 8 steps, each step is a 7-value sensor vector → 4 actions
+const net = new NetworkTransformerRL(8, 7, {
+  d_model:  32,
+  nHeads:   2,
+  d_ff:     64,
+  nBlocks:  2,
+  nActions: 4,
+});
+
+// Each step: feed the last N states as a sequence
+const sequence = getLastNStates();      // number[][] — shape: [8, 7]
+const qValues  = net.predict(sequence); // number[4]
+
+// Q-learning update: train toward Bellman target
+const action  = argmax(qValues);
+const reward  = env.step(action);
+const targets = qValues.slice();
+targets[action] = reward + 0.99 * Math.max(...net.predict(nextSequence));
+
+const loss = net.train(sequence, targets, 0.001);
+```
+
+The last step in the sequence gets 2× pooling weight — the most recent state contributes more to the decision.
+
+```ts
+// Inspect what the agent is attending to
+const attnWeights = net.getAttentionWeights();
+// attnWeights[blockIdx][headIdx] → seqLen × seqLen matrix
+```
+
 ## Possible improvements
 
 1. **Support for batches** in training to improve efficiency and gradient stability.

package/dist/index.d.mts

@@ -274,10 +274,64 @@ declare class NetworkTransformer {
     private _forward;
 }
 
+interface NetworkTransformerRLOptions {
+    d_model?: number;
+    nHeads?: number;
+    d_ff?: number;
+    nBlocks?: number;
+    nActions?: number;
+}
+declare class NetworkTransformerRL {
+    readonly seqLen: number;
+    readonly inputDim: number;
+    readonly d_model: number;
+    readonly nActions: number;
+    inputProj: WeightMatrix;
+    blocks: TransformerBlock[];
+    outputProj: WeightMatrix;
+    outputBias: number[];
+    private outBiasOpts;
+    private _projected;
+    constructor(seqLen: number, inputDim: number, options?: NetworkTransformerRLOptions);
+    predict(sequence: number[][]): number[];
+    train(sequence: number[][], target: number[], lr: number): number;
+    getAttentionWeights(): (number[][] | null)[][];
+    getWeights(): {
+        inputProj: number[][];
+        blocks: {
+            attn: {
+                heads: {
+                    Wq: number[][];
+                    Wk: number[][];
+                    Wv: number[][];
+                }[];
+                Wo: number[][];
+            };
+            norm1: {
+                gamma: number[];
+                beta: number[];
+            };
+            norm2: {
+                gamma: number[];
+                beta: number[];
+            };
+            ff1: number[][];
+            ff2: number[][];
+            b1: number[];
+            b2: number[];
+        }[];
+        outputProj: number[][];
+        outputBias: number[];
+    };
+    setWeights(data: ReturnType<NetworkTransformerRL['getWeights']>): void;
+    private _forward;
+    private _pool;
+}
+
 declare function mse(predicted: number[], actual: number[]): number;
 declare function crossEntropy(predicted: number[], actual: number[]): number;
 declare function mseDelta(predicted: number, actual: number): number;
 declare function crossEntropyDelta(predicted: number, actual: number): number;
 declare function crossEntropyDeltaRaw(predicted: number, actual: number): number;
 
-export { type Activation, Adam, AttentionHead, EmbeddingMatrix, LSTMLayer, Layer, LayerNorm, Momentum, MultiHeadAttention, Network, NetworkLSTM, type NetworkLSTMOptions, NetworkN, type NetworkNOptions, NetworkTransformer, type NetworkTransformerOptions, Neuron, NeuronN, type Optimizer, type OptimizerFactory, SGD, TransformerBlock, type TransformerBlockOptions, WeightMatrix, crossEntropy, crossEntropyDelta, crossEntropyDeltaRaw, elu, leakyRelu, linear, makeElu, makeLeakyRelu, matMul, mse, mseDelta, relu, sigmoid, softmax, softmaxBackward, tanh, transpose };
+export { type Activation, Adam, AttentionHead, EmbeddingMatrix, LSTMLayer, Layer, LayerNorm, Momentum, MultiHeadAttention, Network, NetworkLSTM, type NetworkLSTMOptions, NetworkN, type NetworkNOptions, NetworkTransformer, type NetworkTransformerOptions, NetworkTransformerRL, type NetworkTransformerRLOptions, Neuron, NeuronN, type Optimizer, type OptimizerFactory, SGD, TransformerBlock, type TransformerBlockOptions, WeightMatrix, crossEntropy, crossEntropyDelta, crossEntropyDeltaRaw, elu, leakyRelu, linear, makeElu, makeLeakyRelu, matMul, mse, mseDelta, relu, sigmoid, softmax, softmaxBackward, tanh, transpose };

package/dist/index.d.ts

@@ -274,10 +274,64 @@ declare class NetworkTransformer {
     private _forward;
 }
 
+interface NetworkTransformerRLOptions {
+    d_model?: number;
+    nHeads?: number;
+    d_ff?: number;
+    nBlocks?: number;
+    nActions?: number;
+}
+declare class NetworkTransformerRL {
+    readonly seqLen: number;
+    readonly inputDim: number;
+    readonly d_model: number;
+    readonly nActions: number;
+    inputProj: WeightMatrix;
+    blocks: TransformerBlock[];
+    outputProj: WeightMatrix;
+    outputBias: number[];
+    private outBiasOpts;
+    private _projected;
+    constructor(seqLen: number, inputDim: number, options?: NetworkTransformerRLOptions);
+    predict(sequence: number[][]): number[];
+    train(sequence: number[][], target: number[], lr: number): number;
+    getAttentionWeights(): (number[][] | null)[][];
+    getWeights(): {
+        inputProj: number[][];
+        blocks: {
+            attn: {
+                heads: {
+                    Wq: number[][];
+                    Wk: number[][];
+                    Wv: number[][];
+                }[];
+                Wo: number[][];
+            };
+            norm1: {
+                gamma: number[];
+                beta: number[];
+            };
+            norm2: {
+                gamma: number[];
+                beta: number[];
+            };
+            ff1: number[][];
+            ff2: number[][];
+            b1: number[];
+            b2: number[];
+        }[];
+        outputProj: number[][];
+        outputBias: number[];
+    };
+    setWeights(data: ReturnType<NetworkTransformerRL['getWeights']>): void;
+    private _forward;
+    private _pool;
+}
+
 declare function mse(predicted: number[], actual: number[]): number;
 declare function crossEntropy(predicted: number[], actual: number[]): number;
 declare function mseDelta(predicted: number, actual: number): number;
 declare function crossEntropyDelta(predicted: number, actual: number): number;
 declare function crossEntropyDeltaRaw(predicted: number, actual: number): number;
 
-export { type Activation, Adam, AttentionHead, EmbeddingMatrix, LSTMLayer, Layer, LayerNorm, Momentum, MultiHeadAttention, Network, NetworkLSTM, type NetworkLSTMOptions, NetworkN, type NetworkNOptions, NetworkTransformer, type NetworkTransformerOptions, Neuron, NeuronN, type Optimizer, type OptimizerFactory, SGD, TransformerBlock, type TransformerBlockOptions, WeightMatrix, crossEntropy, crossEntropyDelta, crossEntropyDeltaRaw, elu, leakyRelu, linear, makeElu, makeLeakyRelu, matMul, mse, mseDelta, relu, sigmoid, softmax, softmaxBackward, tanh, transpose };
+export { type Activation, Adam, AttentionHead, EmbeddingMatrix, LSTMLayer, Layer, LayerNorm, Momentum, MultiHeadAttention, Network, NetworkLSTM, type NetworkLSTMOptions, NetworkN, type NetworkNOptions, NetworkTransformer, type NetworkTransformerOptions, NetworkTransformerRL, type NetworkTransformerRLOptions, Neuron, NeuronN, type Optimizer, type OptimizerFactory, SGD, TransformerBlock, type TransformerBlockOptions, WeightMatrix, crossEntropy, crossEntropyDelta, crossEntropyDeltaRaw, elu, leakyRelu, linear, makeElu, makeLeakyRelu, matMul, mse, mseDelta, relu, sigmoid, softmax, softmaxBackward, tanh, transpose };

package/dist/index.js

@@ -32,6 +32,7 @@ __export(index_exports, {
   NetworkLSTM: () => NetworkLSTM,
   NetworkN: () => NetworkN,
   NetworkTransformer: () => NetworkTransformer,
+  NetworkTransformerRL: () => NetworkTransformerRL,
   Neuron: () => Neuron,
   NeuronN: () => NeuronN,
   SGD: () => SGD,
@@ -1098,6 +1099,172 @@ var NetworkTransformer = class {
   }
 };
 
+// src/NetworkTransformerRL.ts
+var NetworkTransformerRL = class {
+  constructor(seqLen, inputDim, options = {}) {
+    // Forward caches para backprop
+    this._projected = null;
+    const {
+      d_model = 32,
+      nHeads = 2,
+      d_ff = 64,
+      nBlocks = 2,
+      nActions = 2
+    } = options;
+    this.seqLen = seqLen;
+    this.inputDim = inputDim;
+    this.d_model = d_model;
+    this.nActions = nActions;
+    this.inputProj = new WeightMatrix(d_model, inputDim);
+    this.blocks = Array.from(
+      { length: nBlocks },
+      () => new TransformerBlock({ d_model, nHeads, d_ff })
+    );
+    this.outputProj = new WeightMatrix(nActions, d_model);
+    this.outputBias = new Array(nActions).fill(0);
+    this.outBiasOpts = Array.from({ length: nActions }, () => new Adam());
+  }
+  // ── Forward ────────────────────────────────────────────────────────────────
+  // sequence: seqLen × inputDim → nActions Q-values
+  predict(sequence) {
+    const h = this._forward(sequence);
+    const pooled = this._pool(h);
+    return this.outputProj.W.map(
+      (row, c) => row.reduce((s, w, m) => s + w * pooled[m], this.outputBias[c])
+    );
+  }
+  // ── Training ────────────────────────────────────────────────────────────────
+  // sequence: seqLen × inputDim
+  // target:   nActions Q-values (one-hot style para Q-learning)
+  // lr:       learning rate
+  // Returns: MSE loss
+  train(sequence, target, lr) {
+    const h = this._forward(sequence);
+    const pooled = this._pool(h);
+    const pred = this.outputProj.W.map(
+      (row, c) => row.reduce((s, w, m) => s + w * pooled[m], this.outputBias[c])
+    );
+    const n = this.nActions;
+    let loss = 0;
+    for (let c = 0; c < n; c++) {
+      const diff = pred[c] - target[c];
+      loss += diff * diff;
+    }
+    loss /= n;
+    const dPred = pred.map((p, c) => 2 * (p - target[c]) / n);
+    const dPooled = Array.from(
+      { length: this.d_model },
+      (_, m) => dPred.reduce((s, dp, c) => s + dp * this.outputProj.W[c][m], 0)
+    );
+    const dWout = Array.from(
+      { length: this.nActions },
+      (_, c) => Array.from(
+        { length: this.d_model },
+        (_2, m) => dPred[c] * pooled[m]
+      )
+    );
+    const dBout = dPred.slice();
+    this.outputProj.update(dWout, lr);
+    for (let c = 0; c < this.nActions; c++)
+      this.outputBias[c] = this.outBiasOpts[c].step(this.outputBias[c], dBout[c], lr);
+    let dH = Array.from(
+      { length: this.seqLen },
+      (_, i) => dPooled.map((v) => v / this.seqLen)
+      // Gradiente dividido entre posiciones
+    );
+    for (let b = this.blocks.length - 1; b >= 0; b--)
+      dH = this.blocks[b].backward(dH, lr);
+    for (let i = 0; i < this.seqLen; i++) {
+      const dInputProj = Array.from(
+        { length: this.d_model },
+        (_, k) => Array.from(
+          { length: this.inputDim },
+          (_2, m) => dH[i][k] * sequence[i][m]
+        )
+      );
+      this.inputProj.update(dInputProj, lr);
+    }
+    return loss;
+  }
+  // Attention weights from every block for visualization.
+  getAttentionWeights() {
+    return this.blocks.map((b) => b.getAttentionWeights());
+  }
+  // ── Serialization ──────────────────────────────────────────────────────────
+  getWeights() {
+    return {
+      inputProj: this.inputProj.W.map((r) => [...r]),
+      blocks: this.blocks.map((b) => ({
+        attn: {
+          heads: b.attn.heads.map((h) => ({
+            Wq: h.Wq.W.map((r) => [...r]),
+            Wk: h.Wk.W.map((r) => [...r]),
+            Wv: h.Wv.W.map((r) => [...r])
+          })),
+          Wo: b.attn.Wo.W.map((r) => [...r])
+        },
+        norm1: { gamma: [...b.norm1.gamma], beta: [...b.norm1.beta] },
+        norm2: { gamma: [...b.norm2.gamma], beta: [...b.norm2.beta] },
+        ff1: b.ff1.W.map((r) => [...r]),
+        ff2: b.ff2.W.map((r) => [...r]),
+        b1: [...b.b1],
+        b2: [...b.b2]
+      })),
+      outputProj: this.outputProj.W.map((r) => [...r]),
+      outputBias: [...this.outputBias]
+    };
+  }
+  setWeights(data) {
+    data.inputProj.forEach((row, i) => {
+      this.inputProj.W[i] = [...row];
+    });
+    data.blocks.forEach((bd, b) => {
+      const blk = this.blocks[b];
+      bd.attn.heads.forEach((hd, h) => {
+        blk.attn.heads[h].Wq.W = hd.Wq.map((r) => [...r]);
+        blk.attn.heads[h].Wk.W = hd.Wk.map((r) => [...r]);
+        blk.attn.heads[h].Wv.W = hd.Wv.map((r) => [...r]);
+      });
+      blk.attn.Wo.W = bd.attn.Wo.map((r) => [...r]);
+      blk.norm1.gamma = [...bd.norm1.gamma];
+      blk.norm1.beta = [...bd.norm1.beta];
+      blk.norm2.gamma = [...bd.norm2.gamma];
+      blk.norm2.beta = [...bd.norm2.beta];
+      blk.ff1.W = bd.ff1.map((r) => [...r]);
+      blk.ff2.W = bd.ff2.map((r) => [...r]);
+      blk.b1 = [...bd.b1];
+      blk.b2 = [...bd.b2];
+    });
+    this.outputProj.W = data.outputProj.map((r) => [...r]);
+    this.outputBias = [...data.outputBias];
+  }
+  // ── Internal ────────────────────────────────────────────────────────────────
+  _forward(sequence) {
+    let h = sequence.map(
+      (step) => this.inputProj.W.map(
+        (row, k) => row.reduce((s, w, m) => s + w * step[m], 0)
+      )
+    );
+    for (const block of this.blocks)
+      h = block.predict(h);
+    this._projected = h;
+    return h;
+  }
+  _pool(h) {
+    const weights = Array.from(
+      { length: this.seqLen },
+      (_, i) => i === this.seqLen - 1 ? 2 : 1
+    );
+    const totalWeight = weights.reduce((a, b) => a + b, 0);
+    return Array.from({ length: this.d_model }, (_, m) => {
+      let sum = 0;
+      for (let i = 0; i < this.seqLen; i++)
+        sum += weights[i] * h[i][m];
+      return sum / totalWeight;
+    });
+  }
+};
+
 // src/losses.ts
 function mse(predicted, actual) {
   return predicted.reduce((sum, p, i) => sum + (actual[i] - p) ** 2, 0) / predicted.length;
@@ -1134,6 +1301,7 @@ function crossEntropyDeltaRaw(predicted, actual) {
   NetworkLSTM,
   NetworkN,
   NetworkTransformer,
+  NetworkTransformerRL,
   Neuron,
   NeuronN,
   SGD,

package/dist/index.mjs

@@ -1039,6 +1039,172 @@ var NetworkTransformer = class {
   }
 };
 
+// src/NetworkTransformerRL.ts
+var NetworkTransformerRL = class {
+  constructor(seqLen, inputDim, options = {}) {
+    // Forward caches para backprop
+    this._projected = null;
+    const {
+      d_model = 32,
+      nHeads = 2,
+      d_ff = 64,
+      nBlocks = 2,
+      nActions = 2
+    } = options;
+    this.seqLen = seqLen;
+    this.inputDim = inputDim;
+    this.d_model = d_model;
+    this.nActions = nActions;
+    this.inputProj = new WeightMatrix(d_model, inputDim);
+    this.blocks = Array.from(
+      { length: nBlocks },
+      () => new TransformerBlock({ d_model, nHeads, d_ff })
+    );
+    this.outputProj = new WeightMatrix(nActions, d_model);
+    this.outputBias = new Array(nActions).fill(0);
+    this.outBiasOpts = Array.from({ length: nActions }, () => new Adam());
+  }
+  // ── Forward ────────────────────────────────────────────────────────────────
+  // sequence: seqLen × inputDim → nActions Q-values
+  predict(sequence) {
+    const h = this._forward(sequence);
+    const pooled = this._pool(h);
+    return this.outputProj.W.map(
+      (row, c) => row.reduce((s, w, m) => s + w * pooled[m], this.outputBias[c])
+    );
+  }
+  // ── Training ────────────────────────────────────────────────────────────────
+  // sequence: seqLen × inputDim
+  // target:   nActions Q-values (one-hot style para Q-learning)
+  // lr:       learning rate
+  // Returns: MSE loss
+  train(sequence, target, lr) {
+    const h = this._forward(sequence);
+    const pooled = this._pool(h);
+    const pred = this.outputProj.W.map(
+      (row, c) => row.reduce((s, w, m) => s + w * pooled[m], this.outputBias[c])
+    );
+    const n = this.nActions;
+    let loss = 0;
+    for (let c = 0; c < n; c++) {
+      const diff = pred[c] - target[c];
+      loss += diff * diff;
+    }
+    loss /= n;
+    const dPred = pred.map((p, c) => 2 * (p - target[c]) / n);
+    const dPooled = Array.from(
+      { length: this.d_model },
+      (_, m) => dPred.reduce((s, dp, c) => s + dp * this.outputProj.W[c][m], 0)
+    );
+    const dWout = Array.from(
+      { length: this.nActions },
+      (_, c) => Array.from(
+        { length: this.d_model },
+        (_2, m) => dPred[c] * pooled[m]
+      )
+    );
+    const dBout = dPred.slice();
+    this.outputProj.update(dWout, lr);
+    for (let c = 0; c < this.nActions; c++)
+      this.outputBias[c] = this.outBiasOpts[c].step(this.outputBias[c], dBout[c], lr);
+    let dH = Array.from(
+      { length: this.seqLen },
+      (_, i) => dPooled.map((v) => v / this.seqLen)
+      // Gradiente dividido entre posiciones
+    );
+    for (let b = this.blocks.length - 1; b >= 0; b--)
+      dH = this.blocks[b].backward(dH, lr);
+    for (let i = 0; i < this.seqLen; i++) {
+      const dInputProj = Array.from(
+        { length: this.d_model },
+        (_, k) => Array.from(
+          { length: this.inputDim },
+          (_2, m) => dH[i][k] * sequence[i][m]
+        )
+      );
+      this.inputProj.update(dInputProj, lr);
+    }
+    return loss;
+  }
+  // Attention weights from every block for visualization.
+  getAttentionWeights() {
+    return this.blocks.map((b) => b.getAttentionWeights());
+  }
+  // ── Serialization ──────────────────────────────────────────────────────────
+  getWeights() {
+    return {
+      inputProj: this.inputProj.W.map((r) => [...r]),
+      blocks: this.blocks.map((b) => ({
+        attn: {
+          heads: b.attn.heads.map((h) => ({
+            Wq: h.Wq.W.map((r) => [...r]),
+            Wk: h.Wk.W.map((r) => [...r]),
+            Wv: h.Wv.W.map((r) => [...r])
+          })),
+          Wo: b.attn.Wo.W.map((r) => [...r])
+        },
+        norm1: { gamma: [...b.norm1.gamma], beta: [...b.norm1.beta] },
+        norm2: { gamma: [...b.norm2.gamma], beta: [...b.norm2.beta] },
+        ff1: b.ff1.W.map((r) => [...r]),
+        ff2: b.ff2.W.map((r) => [...r]),
+        b1: [...b.b1],
+        b2: [...b.b2]
+      })),
+      outputProj: this.outputProj.W.map((r) => [...r]),
+      outputBias: [...this.outputBias]
+    };
+  }
+  setWeights(data) {
+    data.inputProj.forEach((row, i) => {
+      this.inputProj.W[i] = [...row];
+    });
+    data.blocks.forEach((bd, b) => {
+      const blk = this.blocks[b];
+      bd.attn.heads.forEach((hd, h) => {
+        blk.attn.heads[h].Wq.W = hd.Wq.map((r) => [...r]);
+        blk.attn.heads[h].Wk.W = hd.Wk.map((r) => [...r]);
+        blk.attn.heads[h].Wv.W = hd.Wv.map((r) => [...r]);
+      });
+      blk.attn.Wo.W = bd.attn.Wo.map((r) => [...r]);
+      blk.norm1.gamma = [...bd.norm1.gamma];
+      blk.norm1.beta = [...bd.norm1.beta];
+      blk.norm2.gamma = [...bd.norm2.gamma];
+      blk.norm2.beta = [...bd.norm2.beta];
+      blk.ff1.W = bd.ff1.map((r) => [...r]);
+      blk.ff2.W = bd.ff2.map((r) => [...r]);
+      blk.b1 = [...bd.b1];
+      blk.b2 = [...bd.b2];
+    });
+    this.outputProj.W = data.outputProj.map((r) => [...r]);
+    this.outputBias = [...data.outputBias];
+  }
+  // ── Internal ────────────────────────────────────────────────────────────────
+  _forward(sequence) {
+    let h = sequence.map(
+      (step) => this.inputProj.W.map(
+        (row, k) => row.reduce((s, w, m) => s + w * step[m], 0)
+      )
+    );
+    for (const block of this.blocks)
+      h = block.predict(h);
+    this._projected = h;
+    return h;
+  }
+  _pool(h) {
+    const weights = Array.from(
+      { length: this.seqLen },
+      (_, i) => i === this.seqLen - 1 ? 2 : 1
+    );
+    const totalWeight = weights.reduce((a, b) => a + b, 0);
+    return Array.from({ length: this.d_model }, (_, m) => {
+      let sum = 0;
+      for (let i = 0; i < this.seqLen; i++)
+        sum += weights[i] * h[i][m];
+      return sum / totalWeight;
+    });
+  }
+};
+
 // src/losses.ts
 function mse(predicted, actual) {
   return predicted.reduce((sum, p, i) => sum + (actual[i] - p) ** 2, 0) / predicted.length;
@@ -1074,6 +1240,7 @@ export {
   NetworkLSTM,
   NetworkN,
   NetworkTransformer,
+  NetworkTransformerRL,
   Neuron,
   NeuronN,
   SGD,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dniskav/neuron",
-  "version": "0.2.1",
+  "version": "0.2.3",
   "description": "Minimal neural network from scratch — neuron, layer, network, backpropagation. No dependencies.",
   "main": "dist/index.js",
   "module": "dist/index.mjs",