@dniskav/neuron 0.2.0 → 0.2.1

package/README.md

@@ -19,7 +19,7 @@ A minimal, dependency-free neural network library built from scratch in TypeScri
 | `MultiHeadAttention` | N parallel attention heads concatenated and projected to `d_model`. |
 | `AttentionHead` | Single scaled dot-product self-attention head (Q / K / V projections + backprop). |
 | `LayerNorm` | Layer normalization with learnable γ / β per feature. |
-| `WeightMatrix` | 2D weight matrix with per-scalar Adam optimizers. |
+| `WeightMatrix` | 2D weight matrix with per-scalar Adam optimizers. Optional per-element gradient clipping via `update(dW, lr, clipValue)`. |
 | `EmbeddingMatrix` | Lookup-table embedding matrix with SGD updates. |
 | `sigmoid` `relu` `tanh` `linear` | Built-in activation functions. |
 | `SGD` `Momentum` `Adam` | Optimizers. Each instance tracks its own state per weight. |
@@ -258,6 +258,7 @@ const targets  = [...];   // 81*9 one-hot values
 const mask     = puzzle.map(v => v === 0);   // only train on empty cells
 
 const loss = net.train(puzzle, targets, 0.001, mask);
+// loss is cross-entropy (not MSE) — decreases from ~2.2 toward 0 as training progresses
 const logits = net.predict(puzzle);   // 729 logits (81 × 9)
 
 // Attention weights from all blocks for visualization
@@ -270,8 +271,10 @@ Each head in each block learns a different type of relationship (row, column,
 
 ## Possible improvements
 
-1. **Support for batches** in training to improve efficiency.
-2. **Improve documentation** with more advanced examples and use cases.
+1. **Support for batches** in training to improve efficiency and gradient stability.
+2. **Global gradient norm clipping** — `WeightMatrix.update` supports per-element clipping; a utility to clip across all matrices by total norm would be more principled.
+3. **Learning rate warmup** — standard practice for Transformers; ramp LR from 0 to target over the first N steps.
+4. **Pre-norm architecture** — LayerNorm before the residual add (instead of after) is more stable for deep stacks.
 
 ## License
 

package/dist/index.d.mts

@@ -173,7 +173,7 @@ declare class WeightMatrix {
     W: number[][];
     private opts;
     constructor(rows: number, cols: number);
-    update(dW: number[][], lr: number): void;
+    update(dW: number[][], lr: number, clipValue?: number): void;
 }
 declare class EmbeddingMatrix {
     W: number[][];

package/dist/index.d.ts

@@ -173,7 +173,7 @@ declare class WeightMatrix {
     W: number[][];
     private opts;
     constructor(rows: number, cols: number);
-    update(dW: number[][], lr: number): void;
+    update(dW: number[][], lr: number, clipValue?: number): void;
 }
 declare class EmbeddingMatrix {
     W: number[][];

package/dist/index.js

@@ -579,10 +579,15 @@ var WeightMatrix = class {
     );
   }
   // Apply pre-computed gradient (same shape as W).
-  update(dW, lr) {
+  // clipValue: optional per-element gradient clipping before the Adam step.
+  // Prevents gradient explosion in deep networks (e.g. Transformers without
+  // global norm clipping). Pass e.g. 1.0 to clip to [-1, 1].
+  update(dW, lr, clipValue = Infinity) {
     for (let i = 0; i < this.W.length; i++)
-      for (let j = 0; j < this.W[0].length; j++)
-        this.W[i][j] = this.opts[i][j].step(this.W[i][j], dW[i][j], lr);
+      for (let j = 0; j < this.W[0].length; j++) {
+        const g = isFinite(clipValue) ? Math.max(-clipValue, Math.min(clipValue, dW[i][j])) : dW[i][j];
+        this.W[i][j] = this.opts[i][j].step(this.W[i][j], g, lr);
+      }
   }
 };
 var EmbeddingMatrix = class {
@@ -1036,14 +1041,14 @@ var NetworkTransformer = class {
     const dLogits = Array.from({ length: this.seqLen }, (_, i) => {
       if (mask && !mask[i]) return new Array(this.nClasses).fill(0);
       count++;
-      return Array.from({ length: this.nClasses }, (_2, c) => {
+      const probs = softmax(logits[i]);
+      for (let c = 0; c < this.nClasses; c++) {
         const t = targets[i * this.nClasses + c];
-        const p = logits[i][c];
-        loss += (p - t) ** 2;
-        return 2 * (p - t);
-      });
+        if (t > 0) loss -= Math.log(Math.max(probs[c], 1e-7));
+      }
+      return probs.map((p, c) => p - targets[i * this.nClasses + c]);
     });
-    if (count > 0) loss /= count * this.nClasses;
+    if (count > 0) loss /= count;
     const dH = Array.from(
       { length: this.seqLen },
       (_, i) => Array.from(

package/dist/index.mjs

@@ -520,10 +520,15 @@ var WeightMatrix = class {
     );
   }
   // Apply pre-computed gradient (same shape as W).
-  update(dW, lr) {
+  // clipValue: optional per-element gradient clipping before the Adam step.
+  // Prevents gradient explosion in deep networks (e.g. Transformers without
+  // global norm clipping). Pass e.g. 1.0 to clip to [-1, 1].
+  update(dW, lr, clipValue = Infinity) {
     for (let i = 0; i < this.W.length; i++)
-      for (let j = 0; j < this.W[0].length; j++)
-        this.W[i][j] = this.opts[i][j].step(this.W[i][j], dW[i][j], lr);
+      for (let j = 0; j < this.W[0].length; j++) {
+        const g = isFinite(clipValue) ? Math.max(-clipValue, Math.min(clipValue, dW[i][j])) : dW[i][j];
+        this.W[i][j] = this.opts[i][j].step(this.W[i][j], g, lr);
+      }
   }
 };
 var EmbeddingMatrix = class {
@@ -977,14 +982,14 @@ var NetworkTransformer = class {
     const dLogits = Array.from({ length: this.seqLen }, (_, i) => {
       if (mask && !mask[i]) return new Array(this.nClasses).fill(0);
       count++;
-      return Array.from({ length: this.nClasses }, (_2, c) => {
+      const probs = softmax(logits[i]);
+      for (let c = 0; c < this.nClasses; c++) {
         const t = targets[i * this.nClasses + c];
-        const p = logits[i][c];
-        loss += (p - t) ** 2;
-        return 2 * (p - t);
-      });
+        if (t > 0) loss -= Math.log(Math.max(probs[c], 1e-7));
+      }
+      return probs.map((p, c) => p - targets[i * this.nClasses + c]);
     });
-    if (count > 0) loss /= count * this.nClasses;
+    if (count > 0) loss /= count;
     const dH = Array.from(
       { length: this.seqLen },
       (_, i) => Array.from(

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dniskav/neuron",
-  "version": "0.2.0",
+  "version": "0.2.1",
   "description": "Minimal neural network from scratch — neuron, layer, network, backpropagation. No dependencies.",
   "main": "dist/index.js",
   "module": "dist/index.mjs",