@dniskav/neuron 0.1.4 → 0.1.5

package/README.md

@@ -1,5 +1,3 @@
-# @dniskav/neuron
-
 [![npm](https://img.shields.io/npm/v/@dniskav/neuron)](https://www.npmjs.com/package/@dniskav/neuron)
 [![license](https://img.shields.io/npm/l/@dniskav/neuron)](LICENSE)
 
@@ -7,15 +5,19 @@ A minimal, dependency-free neural network library built from scratch in TypeScri
 
 ## What's inside
 
-| Class | Description |
-|-------|-------------|
+| Export | Description |
+|--------|-------------|
 | `Neuron` | Single-input neuron. The simplest possible unit: one weight, one bias. |
-| `NeuronN` | N-input neuron with Xavier initialization and sigmoid activation. |
+| `NeuronN` | N-input neuron with Xavier initialization and configurable activation. |
 | `Layer` | A group of `NeuronN` neurons that share the same inputs. |
 | `Network` | Two-layer network (hidden + output) with backpropagation. |
 | `NetworkN` | Deep network of arbitrary depth. Define your architecture as `[inputs, ...hidden, outputs]`. |
 | `LSTMLayer` | Recurrent layer with persistent hidden and cell state. Learns sequences via BPTT. |
 | `NetworkLSTM` | Wraps an `LSTMLayer` + dense layers. Maintains memory across steps within an episode. |
+| `sigmoid` `relu` `tanh` `linear` | Built-in activation functions. |
+| `SGD` `Momentum` `Adam` | Optimizers. Each instance tracks its own state per weight. |
+| `mse` `crossEntropy` | Loss functions for evaluation and logging. |
+| `mseDelta` `crossEntropyDelta` | Output-layer delta functions for use with `trainWithDeltas`. |
 
 ## Install
 
@@ -92,12 +94,73 @@ net.train([0.5, 0.3, 0.8], [1, 0], 0.05);
 const [out1, out2] = net.predict([0.5, 0.3, 0.8]);
 ```
 
+### Activations — ReLU, tanh, and more
+
+Pass an activation per layer. The last layer typically uses `sigmoid` for binary output or `linear` for regression.
+
+```ts
+import { NetworkN, relu, sigmoid } from "@dniskav/neuron";
+
+const net = new NetworkN([3, 64, 32, 1], {
+  activations: [relu, relu, sigmoid],
+});
+```
+
+Available: `sigmoid`, `relu`, `tanh`, `linear`.
+
+### Optimizers — Adam, Momentum, SGD
+
+Pass an optimizer factory. Each weight gets its own instance with independent state.
+
+```ts
+import { NetworkN, relu, sigmoid, Adam } from "@dniskav/neuron";
+
+const net = new NetworkN([2, 64, 1], {
+  activations: [relu, sigmoid],
+  optimizer: () => new Adam(),          // default: beta1=0.9, beta2=0.999
+});
+
+// Momentum example
+import { Momentum } from "@dniskav/neuron";
+const net2 = new NetworkN([2, 32, 1], {
+  optimizer: () => new Momentum(0.9),
+});
+```
+
+Optimizers also work in `NetworkLSTM` (applied to the dense layers):
+
+```ts
+import { NetworkLSTM, relu, Adam } from "@dniskav/neuron";
+
+const net = new NetworkLSTM(1, 8, [4, 1], {
+  denseActivation: relu,
+  optimizer: () => new Adam(0.001),
+});
+```
+
+### Loss utilities
+
+```ts
+import { mse, crossEntropy } from "@dniskav/neuron";
+
+const predicted = net.predict([0.5, 0.3]);
+console.log(mse(predicted, [1, 0]));
+console.log(crossEntropy(predicted, [1, 0]));
+```
+
 ### trainWithDeltas — custom loss / physics-based gradients
 
 `NetworkN` also exposes `trainWithDeltas` for when you compute your own output-layer deltas (e.g., from a physics simulation or a custom loss function):
 
 ```ts
-net.trainWithDeltas(inputs, [0.4, -0.2], 0.05);
+import { NetworkN, mseDelta } from "@dniskav/neuron";
+
+const net = new NetworkN([3, 16, 2]);
+const pred = net.predict(inputs);
+
+// Compute deltas manually using a helper, or from any external signal
+const deltas = pred.map((p, i) => mseDelta(p, targets[i]));
+net.trainWithDeltas(inputs, deltas, 0.01);
 ```
 
 ### NetworkLSTM — recurrent network with memory
@@ -143,17 +206,19 @@ The network learns to count steps using its hidden state — no external counter
 
 ## How it works
 
-Every class uses **sigmoid** as its activation function and **gradient descent** to update weights:
+Each class applies an **activation function** to the weighted sum of inputs and uses **gradient descent** to update weights:
 
 ```
-weight += lr × error × input
-bias   += lr × error
+weight += lr × delta × input
+bias   += lr × delta
 ```
 
-`NetworkN` implements full **backpropagation** across all layers, propagating deltas from the output back to the first layer using the chain rule.
+`NetworkN` implements full **backpropagation** across all layers, propagating deltas from the output back to the first layer using the chain rule. The derivative of the chosen activation is applied at each layer.
 
 `NeuronN` uses simplified **Xavier initialization** — weights start in `[-√(1/n), +√(1/n)]` — so gradients flow well from the start of training.
 
+When an **optimizer** is used (e.g., Adam), the raw gradient is passed to the optimizer instead of being applied directly. Each weight maintains its own optimizer state (velocity, moments).
+
 ## Build
 
 ```bash
@@ -165,6 +230,11 @@ npm run dev     # watch mode
 
 If you are an AI agent or LLM working with this codebase, read [AGENTS.md](AGENTS.md) first. It contains the full class hierarchy, design constraints, and what this library does not do.
 
+## Possible improvements
+
+1. **Support for batches** in training to improve efficiency.
+2. **Improve documentation** with more advanced examples and use cases.
+
 ## License
 
 MIT

package/dist/index.d.mts

@@ -6,17 +6,53 @@ declare class Neuron {
     train(input: number, target: number, lr: number): void;
 }
 
+interface Activation {
+    fn(x: number): number;
+    dfn(out: number): number;
+}
+declare const sigmoid: Activation;
+declare const tanh: Activation;
+declare const relu: Activation;
+declare const linear: Activation;
+
+interface Optimizer {
+    step(weight: number, gradient: number, lr: number): number;
+}
+type OptimizerFactory = () => Optimizer;
+declare class SGD implements Optimizer {
+    step(weight: number, gradient: number, lr: number): number;
+}
+declare class Momentum implements Optimizer {
+    readonly beta: number;
+    private v;
+    constructor(beta?: number);
+    step(weight: number, gradient: number, lr: number): number;
+}
+declare class Adam implements Optimizer {
+    readonly beta1: number;
+    readonly beta2: number;
+    readonly epsilon: number;
+    private m;
+    private v;
+    private t;
+    constructor(beta1?: number, beta2?: number, epsilon?: number);
+    step(weight: number, gradient: number, lr: number): number;
+}
+
 declare class NeuronN {
     weights: number[];
     bias: number;
-    constructor(nInputs: number);
+    readonly activation: Activation;
+    private _opts;
+    constructor(nInputs: number, activation?: Activation, optimizerFactory?: OptimizerFactory);
     predict(inputs: number[]): number;
+    _update(weightGrads: number[], biasGrad: number, lr: number): void;
     train(inputs: number[], target: number, lr: number): void;
 }
 
 declare class Layer {
     neurons: NeuronN[];
-    constructor(nNeurons: number, nInputs: number);
+    constructor(nNeurons: number, nInputs: number, activation?: Activation, optimizerFactory?: OptimizerFactory);
     predict(inputs: number[]): number[];
 }
 
@@ -28,10 +64,14 @@ declare class Network {
     train(inputs: number[], target: number, lr: number): number;
 }
 
+interface NetworkNOptions {
+    activations?: Activation[];
+    optimizer?: OptimizerFactory;
+}
 declare class NetworkN {
     readonly structure: number[];
     layers: Layer[];
-    constructor(structure: number[]);
+    constructor(structure: number[], options?: NetworkNOptions);
     predict(inputs: number[]): number[];
     train(inputs: number[], targets: number[], lr: number): number;
     trainWithDeltas(inputs: number[], outputDeltas: number[], lr: number): void;
@@ -78,13 +118,17 @@ declare class LSTMLayer {
     setWeights(data: ReturnType<LSTMLayer["getWeights"]>): void;
 }
 
+interface NetworkLSTMOptions {
+    denseActivation?: Activation;
+    optimizer?: OptimizerFactory;
+}
 declare class NetworkLSTM {
     readonly inputSize: number;
     readonly hiddenSize: number;
     lstm: LSTMLayer;
     denseLayers: Layer[];
     private _acts;
-    constructor(inputSize: number, hiddenSize: number, denseStructure: number[]);
+    constructor(inputSize: number, hiddenSize: number, denseStructure: number[], options?: NetworkLSTMOptions);
     resetState(): void;
     predict(inputs: number[]): number[];
     train(targets: number[][], lr: number): void;
@@ -115,4 +159,10 @@ declare class NetworkLSTM {
     setWeights(data: ReturnType<NetworkLSTM["getWeights"]>): void;
 }
 
-export { LSTMLayer, Layer, Network, NetworkLSTM, NetworkN, Neuron, NeuronN };
+declare function mse(predicted: number[], actual: number[]): number;
+declare function crossEntropy(predicted: number[], actual: number[]): number;
+declare function mseDelta(predicted: number, actual: number): number;
+declare function crossEntropyDelta(predicted: number, actual: number): number;
+declare function crossEntropyDeltaRaw(predicted: number, actual: number): number;
+
+export { type Activation, Adam, LSTMLayer, Layer, Momentum, Network, NetworkLSTM, type NetworkLSTMOptions, NetworkN, type NetworkNOptions, Neuron, NeuronN, type Optimizer, type OptimizerFactory, SGD, crossEntropy, crossEntropyDelta, crossEntropyDeltaRaw, linear, mse, mseDelta, relu, sigmoid, tanh };

package/dist/index.d.ts

@@ -6,17 +6,53 @@ declare class Neuron {
     train(input: number, target: number, lr: number): void;
 }
 
+interface Activation {
+    fn(x: number): number;
+    dfn(out: number): number;
+}
+declare const sigmoid: Activation;
+declare const tanh: Activation;
+declare const relu: Activation;
+declare const linear: Activation;
+
+interface Optimizer {
+    step(weight: number, gradient: number, lr: number): number;
+}
+type OptimizerFactory = () => Optimizer;
+declare class SGD implements Optimizer {
+    step(weight: number, gradient: number, lr: number): number;
+}
+declare class Momentum implements Optimizer {
+    readonly beta: number;
+    private v;
+    constructor(beta?: number);
+    step(weight: number, gradient: number, lr: number): number;
+}
+declare class Adam implements Optimizer {
+    readonly beta1: number;
+    readonly beta2: number;
+    readonly epsilon: number;
+    private m;
+    private v;
+    private t;
+    constructor(beta1?: number, beta2?: number, epsilon?: number);
+    step(weight: number, gradient: number, lr: number): number;
+}
+
 declare class NeuronN {
     weights: number[];
     bias: number;
-    constructor(nInputs: number);
+    readonly activation: Activation;
+    private _opts;
+    constructor(nInputs: number, activation?: Activation, optimizerFactory?: OptimizerFactory);
     predict(inputs: number[]): number;
+    _update(weightGrads: number[], biasGrad: number, lr: number): void;
     train(inputs: number[], target: number, lr: number): void;
 }
 
 declare class Layer {
     neurons: NeuronN[];
-    constructor(nNeurons: number, nInputs: number);
+    constructor(nNeurons: number, nInputs: number, activation?: Activation, optimizerFactory?: OptimizerFactory);
     predict(inputs: number[]): number[];
 }
 
@@ -28,10 +64,14 @@ declare class Network {
     train(inputs: number[], target: number, lr: number): number;
 }
 
+interface NetworkNOptions {
+    activations?: Activation[];
+    optimizer?: OptimizerFactory;
+}
 declare class NetworkN {
     readonly structure: number[];
     layers: Layer[];
-    constructor(structure: number[]);
+    constructor(structure: number[], options?: NetworkNOptions);
     predict(inputs: number[]): number[];
     train(inputs: number[], targets: number[], lr: number): number;
     trainWithDeltas(inputs: number[], outputDeltas: number[], lr: number): void;
@@ -78,13 +118,17 @@ declare class LSTMLayer {
     setWeights(data: ReturnType<LSTMLayer["getWeights"]>): void;
 }
 
+interface NetworkLSTMOptions {
+    denseActivation?: Activation;
+    optimizer?: OptimizerFactory;
+}
 declare class NetworkLSTM {
     readonly inputSize: number;
     readonly hiddenSize: number;
     lstm: LSTMLayer;
     denseLayers: Layer[];
     private _acts;
-    constructor(inputSize: number, hiddenSize: number, denseStructure: number[]);
+    constructor(inputSize: number, hiddenSize: number, denseStructure: number[], options?: NetworkLSTMOptions);
     resetState(): void;
     predict(inputs: number[]): number[];
     train(targets: number[][], lr: number): void;
@@ -115,4 +159,10 @@ declare class NetworkLSTM {
     setWeights(data: ReturnType<NetworkLSTM["getWeights"]>): void;
 }
 
-export { LSTMLayer, Layer, Network, NetworkLSTM, NetworkN, Neuron, NeuronN };
+declare function mse(predicted: number[], actual: number[]): number;
+declare function crossEntropy(predicted: number[], actual: number[]): number;
+declare function mseDelta(predicted: number, actual: number): number;
+declare function crossEntropyDelta(predicted: number, actual: number): number;
+declare function crossEntropyDeltaRaw(predicted: number, actual: number): number;
+
+export { type Activation, Adam, LSTMLayer, Layer, Momentum, Network, NetworkLSTM, type NetworkLSTMOptions, NetworkN, type NetworkNOptions, Neuron, NeuronN, type Optimizer, type OptimizerFactory, SGD, crossEntropy, crossEntropyDelta, crossEntropyDeltaRaw, linear, mse, mseDelta, relu, sigmoid, tanh };

package/dist/index.js

@@ -20,13 +20,25 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
 // src/index.ts
 var index_exports = {};
 __export(index_exports, {
+  Adam: () => Adam,
   LSTMLayer: () => LSTMLayer,
   Layer: () => Layer,
+  Momentum: () => Momentum,
   Network: () => Network,
   NetworkLSTM: () => NetworkLSTM,
   NetworkN: () => NetworkN,
   Neuron: () => Neuron,
-  NeuronN: () => NeuronN
+  NeuronN: () => NeuronN,
+  SGD: () => SGD,
+  crossEntropy: () => crossEntropy,
+  crossEntropyDelta: () => crossEntropyDelta,
+  crossEntropyDeltaRaw: () => crossEntropyDeltaRaw,
+  linear: () => linear,
+  mse: () => mse,
+  mseDelta: () => mseDelta,
+  relu: () => relu,
+  sigmoid: () => sigmoid2,
+  tanh: () => tanh
 });
 module.exports = __toCommonJS(index_exports);
 
@@ -50,32 +62,97 @@ var Neuron = class {
   }
 };
 
+// src/activations.ts
+var sigmoid2 = {
+  fn: (x) => 1 / (1 + Math.exp(-x)),
+  dfn: (out) => out * (1 - out)
+};
+var tanh = {
+  fn: (x) => {
+    const e = Math.exp(2 * x);
+    return (e - 1) / (e + 1);
+  },
+  dfn: (out) => 1 - out * out
+};
+var relu = {
+  fn: (x) => Math.max(0, x),
+  dfn: (out) => out > 0 ? 1 : 0
+};
+var linear = {
+  fn: (x) => x,
+  dfn: () => 1
+};
+
+// src/optimizers.ts
+var SGD = class {
+  step(weight, gradient, lr) {
+    return weight + lr * gradient;
+  }
+};
+var Momentum = class {
+  constructor(beta = 0.9) {
+    this.beta = beta;
+    this.v = 0;
+  }
+  step(weight, gradient, lr) {
+    this.v = this.beta * this.v + lr * gradient;
+    return weight + this.v;
+  }
+};
+var Adam = class {
+  constructor(beta1 = 0.9, beta2 = 0.999, epsilon = 1e-8) {
+    this.beta1 = beta1;
+    this.beta2 = beta2;
+    this.epsilon = epsilon;
+    this.m = 0;
+    this.v = 0;
+    this.t = 0;
+  }
+  step(weight, gradient, lr) {
+    this.t++;
+    this.m = this.beta1 * this.m + (1 - this.beta1) * gradient;
+    this.v = this.beta2 * this.v + (1 - this.beta2) * gradient * gradient;
+    const mHat = this.m / (1 - Math.pow(this.beta1, this.t));
+    const vHat = this.v / (1 - Math.pow(this.beta2, this.t));
+    return weight + lr * mHat / (Math.sqrt(vHat) + this.epsilon);
+  }
+};
+
 // src/NeuronN.ts
-function sigmoid2(x) {
-  return 1 / (1 + Math.exp(-x));
-}
+var defaultOptimizer = () => new SGD();
 var NeuronN = class {
-  constructor(nInputs) {
+  constructor(nInputs, activation = sigmoid2, optimizerFactory = defaultOptimizer) {
     const limit = Math.sqrt(1 / nInputs);
     this.weights = Array.from({ length: nInputs }, () => (Math.random() * 2 - 1) * limit);
     this.bias = 0;
+    this.activation = activation;
+    this._opts = Array.from({ length: nInputs + 1 }, optimizerFactory);
   }
   predict(inputs) {
     const sum = inputs.reduce((acc, e, i) => acc + e * this.weights[i], this.bias);
-    return sigmoid2(sum);
+    return this.activation.fn(sum);
+  }
+  // Apply pre-computed gradients via the optimizer.
+  // Called internally by Layer / NetworkN / NetworkLSTM during backprop.
+  _update(weightGrads, biasGrad, lr) {
+    this.weights = this.weights.map((w, i) => this._opts[i].step(w, weightGrads[i], lr));
+    this.bias = this._opts[this.weights.length].step(this.bias, biasGrad, lr);
   }
   train(inputs, target, lr) {
     const prediction = this.predict(inputs);
     const error = target - prediction;
-    this.weights = this.weights.map((w, i) => w + lr * error * inputs[i]);
-    this.bias += lr * error;
+    this._update(inputs.map((inp) => error * inp), error, lr);
   }
 };
 
 // src/Layer.ts
+var defaultOptimizer2 = () => new SGD();
 var Layer = class {
-  constructor(nNeurons, nInputs) {
-    this.neurons = Array.from({ length: nNeurons }, () => new NeuronN(nInputs));
+  constructor(nNeurons, nInputs, activation = sigmoid2, optimizerFactory = defaultOptimizer2) {
+    this.neurons = Array.from(
+      { length: nNeurons },
+      () => new NeuronN(nInputs, activation, optimizerFactory)
+    );
   }
   predict(inputs) {
     return this.neurons.map((n) => n.predict(inputs));
@@ -115,12 +192,16 @@ var Network = class {
 };
 
 // src/NetworkN.ts
+var defaultOptimizer3 = () => new SGD();
 var NetworkN = class {
-  constructor(structure) {
+  constructor(structure, options = {}) {
     this.structure = structure;
+    const nLayers = structure.length - 1;
+    const activations = options.activations ?? Array.from({ length: nLayers }, () => sigmoid2);
+    const optimizer = options.optimizer ?? defaultOptimizer3;
     this.layers = [];
     for (let i = 1; i < structure.length; i++) {
-      this.layers.push(new Layer(structure[i], structure[i - 1]));
+      this.layers.push(new Layer(structure[i], structure[i - 1], activations[i - 1], optimizer));
     }
   }
   predict(inputs) {
@@ -132,17 +213,18 @@ var NetworkN = class {
     const act = [inputs];
     for (const layer of this.layers) act.push(layer.predict(act[act.length - 1]));
     const pred = act[act.length - 1];
-    let deltas = pred.map((p, i) => (targets[i] - p) * p * (1 - p));
+    const outAct = this.layers[this.layers.length - 1].neurons[0].activation;
+    let deltas = pred.map((p, i) => (targets[i] - p) * outAct.dfn(p));
     for (let l = this.layers.length - 1; l >= 0; l--) {
       const layer = this.layers[l];
       const layerIn = act[l];
+      const prevAct = l > 0 ? this.layers[l - 1].neurons[0].activation : null;
       const prevDeltas = layerIn.map((out, j) => {
         const errProp = layer.neurons.reduce((s, n, k) => s + deltas[k] * n.weights[j], 0);
-        return errProp * out * (1 - out);
+        return prevAct ? errProp * prevAct.dfn(out) : errProp;
       });
       layer.neurons.forEach((n, k) => {
-        n.weights = n.weights.map((w, j) => w + lr * deltas[k] * layerIn[j]);
-        n.bias += lr * deltas[k];
+        n._update(layerIn.map((inp) => deltas[k] * inp), deltas[k], lr);
       });
       deltas = prevDeltas;
     }
@@ -157,13 +239,13 @@ var NetworkN = class {
     for (let l = this.layers.length - 1; l >= 0; l--) {
       const layer = this.layers[l];
       const layerIn = act[l];
+      const prevAct = l > 0 ? this.layers[l - 1].neurons[0].activation : null;
       const prevDeltas = layerIn.map((out, j) => {
         const errProp = layer.neurons.reduce((s, n, k) => s + deltas[k] * n.weights[j], 0);
-        return errProp * out * (1 - out);
+        return prevAct ? errProp * prevAct.dfn(out) : errProp;
       });
       layer.neurons.forEach((n, k) => {
-        n.weights = n.weights.map((w, j) => w + lr * deltas[k] * layerIn[j]);
-        n.bias += lr * deltas[k];
+        n._update(layerIn.map((inp) => deltas[k] * inp), deltas[k], lr);
       });
       deltas = prevDeltas;
     }
@@ -174,7 +256,7 @@ var NetworkN = class {
 function sigmoid3(x) {
   return 1 / (1 + Math.exp(-x));
 }
-function tanh(x) {
+function tanh2(x) {
   const e = Math.exp(2 * x);
   return (e - 1) / (e + 1);
 }
@@ -223,10 +305,10 @@ var LSTMLayer = class {
     const zo = this.outputGate.linear(combined);
     const zf_a = zf.map(sigmoid3);
     const zi_a = zi.map(sigmoid3);
-    const zg_a = zg.map(tanh);
+    const zg_a = zg.map(tanh2);
     const zo_a = zo.map(sigmoid3);
     const c = c_prev.map((cv, k) => zf_a[k] * cv + zi_a[k] * zg_a[k]);
-    const h = zo_a.map((o, k) => o * tanh(c[k]));
+    const h = zo_a.map((o, k) => o * tanh2(c[k]));
     this._traj.push({ combined, c_prev, zf, zf_a, zi, zi_a, zg, zg_a, zo, zo_a, c, h });
     this.h = h;
     this.c = c;
@@ -254,7 +336,7 @@ var LSTMLayer = class {
     for (let t = T - 1; t >= 0; t--) {
       const s = this._traj[t];
       const dh = dh_seq[t].map((d, k) => d + dh_next[k]);
-      const tanh_c = s.c.map(tanh);
+      const tanh_c = s.c.map(tanh2);
       const do_a = dh.map((d, k) => d * tanh_c[k]);
       const dc = dh.map(
         (d, k) => d * s.zo_a[k] * (1 - tanh_c[k] ** 2) + dc_next[k]
@@ -323,16 +405,19 @@ var LSTMLayer = class {
 };
 
 // src/NetworkLSTM.ts
+var defaultOptimizer4 = () => new SGD();
 var NetworkLSTM = class {
   // [T][layer+1][neuron]
-  constructor(inputSize, hiddenSize, denseStructure) {
+  constructor(inputSize, hiddenSize, denseStructure, options = {}) {
     this.inputSize = inputSize;
     this.hiddenSize = hiddenSize;
     this.lstm = new LSTMLayer(inputSize, hiddenSize);
+    const activation = options.denseActivation ?? sigmoid2;
+    const optimizer = options.optimizer ?? defaultOptimizer4;
     this.denseLayers = [];
     const sizes = [hiddenSize, ...denseStructure];
     for (let i = 1; i < sizes.length; i++) {
-      this.denseLayers.push(new Layer(sizes[i], sizes[i - 1]));
+      this.denseLayers.push(new Layer(sizes[i], sizes[i - 1], activation, optimizer));
     }
     this._acts = [];
   }
@@ -365,14 +450,16 @@ var NetworkLSTM = class {
     for (let t = 0; t < T; t++) {
       const acts = this._acts[t];
       const pred = acts[acts.length - 1];
-      let deltas = pred.map((p, i) => (targets[t][i] - p) * p * (1 - p));
+      const outAct = this.denseLayers[this.denseLayers.length - 1].neurons[0].activation;
+      let deltas = pred.map((p, i) => (targets[t][i] - p) * outAct.dfn(p));
       for (let l = this.denseLayers.length - 1; l >= 0; l--) {
         const layer = this.denseLayers[l];
         const layerIn = acts[l];
         const grad = denseGrads[l];
+        const prevAct = l > 0 ? this.denseLayers[l - 1].neurons[0].activation : null;
         const prevDeltas = layerIn.map((out, j) => {
           const errProp = layer.neurons.reduce((s, n, k) => s + deltas[k] * n.weights[j], 0);
-          return l === 0 ? errProp : errProp * out * (1 - out);
+          return prevAct ? errProp * prevAct.dfn(out) : errProp;
         });
         layer.neurons.forEach((n, k) => {
           n.weights.forEach((_, j) => {
@@ -388,8 +475,11 @@ var NetworkLSTM = class {
       const layer = this.denseLayers[l];
       const grad = denseGrads[l];
       layer.neurons.forEach((n, k) => {
-        n.weights = n.weights.map((w, j) => w + lr / T * grad.dW[k][j]);
-        n.bias += lr / T * grad.db[k];
+        n._update(
+          grad.dW[k].map((g) => g / T),
+          grad.db[k] / T,
+          lr
+        );
       });
     }
     this.lstm.backprop(dh_seq, lr);
@@ -414,13 +504,48 @@ var NetworkLSTM = class {
     });
   }
 };
+
+// src/losses.ts
+function mse(predicted, actual) {
+  return predicted.reduce((sum, p, i) => sum + (actual[i] - p) ** 2, 0) / predicted.length;
+}
+function crossEntropy(predicted, actual) {
+  const eps = 1e-15;
+  return -predicted.reduce((sum, p, i) => {
+    const clipped = Math.max(eps, Math.min(1 - eps, p));
+    return sum + actual[i] * Math.log(clipped) + (1 - actual[i]) * Math.log(1 - clipped);
+  }, 0) / predicted.length;
+}
+function mseDelta(predicted, actual) {
+  return actual - predicted;
+}
+function crossEntropyDelta(predicted, actual) {
+  return actual - predicted;
+}
+function crossEntropyDeltaRaw(predicted, actual) {
+  const eps = 1e-15;
+  const p = Math.max(eps, Math.min(1 - eps, predicted));
+  return actual / p - (1 - actual) / (1 - p);
+}
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
+  Adam,
   LSTMLayer,
   Layer,
+  Momentum,
   Network,
   NetworkLSTM,
   NetworkN,
   Neuron,
-  NeuronN
+  NeuronN,
+  SGD,
+  crossEntropy,
+  crossEntropyDelta,
+  crossEntropyDeltaRaw,
+  linear,
+  mse,
+  mseDelta,
+  relu,
+  sigmoid,
+  tanh
 });

package/dist/index.mjs

@@ -18,32 +18,97 @@ var Neuron = class {
   }
 };
 
+// src/activations.ts
+var sigmoid2 = {
+  fn: (x) => 1 / (1 + Math.exp(-x)),
+  dfn: (out) => out * (1 - out)
+};
+var tanh = {
+  fn: (x) => {
+    const e = Math.exp(2 * x);
+    return (e - 1) / (e + 1);
+  },
+  dfn: (out) => 1 - out * out
+};
+var relu = {
+  fn: (x) => Math.max(0, x),
+  dfn: (out) => out > 0 ? 1 : 0
+};
+var linear = {
+  fn: (x) => x,
+  dfn: () => 1
+};
+
+// src/optimizers.ts
+var SGD = class {
+  step(weight, gradient, lr) {
+    return weight + lr * gradient;
+  }
+};
+var Momentum = class {
+  constructor(beta = 0.9) {
+    this.beta = beta;
+    this.v = 0;
+  }
+  step(weight, gradient, lr) {
+    this.v = this.beta * this.v + lr * gradient;
+    return weight + this.v;
+  }
+};
+var Adam = class {
+  constructor(beta1 = 0.9, beta2 = 0.999, epsilon = 1e-8) {
+    this.beta1 = beta1;
+    this.beta2 = beta2;
+    this.epsilon = epsilon;
+    this.m = 0;
+    this.v = 0;
+    this.t = 0;
+  }
+  step(weight, gradient, lr) {
+    this.t++;
+    this.m = this.beta1 * this.m + (1 - this.beta1) * gradient;
+    this.v = this.beta2 * this.v + (1 - this.beta2) * gradient * gradient;
+    const mHat = this.m / (1 - Math.pow(this.beta1, this.t));
+    const vHat = this.v / (1 - Math.pow(this.beta2, this.t));
+    return weight + lr * mHat / (Math.sqrt(vHat) + this.epsilon);
+  }
+};
+
 // src/NeuronN.ts
-function sigmoid2(x) {
-  return 1 / (1 + Math.exp(-x));
-}
+var defaultOptimizer = () => new SGD();
 var NeuronN = class {
-  constructor(nInputs) {
+  constructor(nInputs, activation = sigmoid2, optimizerFactory = defaultOptimizer) {
     const limit = Math.sqrt(1 / nInputs);
     this.weights = Array.from({ length: nInputs }, () => (Math.random() * 2 - 1) * limit);
     this.bias = 0;
+    this.activation = activation;
+    this._opts = Array.from({ length: nInputs + 1 }, optimizerFactory);
   }
   predict(inputs) {
     const sum = inputs.reduce((acc, e, i) => acc + e * this.weights[i], this.bias);
-    return sigmoid2(sum);
+    return this.activation.fn(sum);
+  }
+  // Apply pre-computed gradients via the optimizer.
+  // Called internally by Layer / NetworkN / NetworkLSTM during backprop.
+  _update(weightGrads, biasGrad, lr) {
+    this.weights = this.weights.map((w, i) => this._opts[i].step(w, weightGrads[i], lr));
+    this.bias = this._opts[this.weights.length].step(this.bias, biasGrad, lr);
   }
   train(inputs, target, lr) {
     const prediction = this.predict(inputs);
     const error = target - prediction;
-    this.weights = this.weights.map((w, i) => w + lr * error * inputs[i]);
-    this.bias += lr * error;
+    this._update(inputs.map((inp) => error * inp), error, lr);
   }
 };
 
 // src/Layer.ts
+var defaultOptimizer2 = () => new SGD();
 var Layer = class {
-  constructor(nNeurons, nInputs) {
-    this.neurons = Array.from({ length: nNeurons }, () => new NeuronN(nInputs));
+  constructor(nNeurons, nInputs, activation = sigmoid2, optimizerFactory = defaultOptimizer2) {
+    this.neurons = Array.from(
+      { length: nNeurons },
+      () => new NeuronN(nInputs, activation, optimizerFactory)
+    );
   }
   predict(inputs) {
     return this.neurons.map((n) => n.predict(inputs));
@@ -83,12 +148,16 @@ var Network = class {
 };
 
 // src/NetworkN.ts
+var defaultOptimizer3 = () => new SGD();
 var NetworkN = class {
-  constructor(structure) {
+  constructor(structure, options = {}) {
     this.structure = structure;
+    const nLayers = structure.length - 1;
+    const activations = options.activations ?? Array.from({ length: nLayers }, () => sigmoid2);
+    const optimizer = options.optimizer ?? defaultOptimizer3;
     this.layers = [];
     for (let i = 1; i < structure.length; i++) {
-      this.layers.push(new Layer(structure[i], structure[i - 1]));
+      this.layers.push(new Layer(structure[i], structure[i - 1], activations[i - 1], optimizer));
     }
   }
   predict(inputs) {
@@ -100,17 +169,18 @@ var NetworkN = class {
     const act = [inputs];
     for (const layer of this.layers) act.push(layer.predict(act[act.length - 1]));
     const pred = act[act.length - 1];
-    let deltas = pred.map((p, i) => (targets[i] - p) * p * (1 - p));
+    const outAct = this.layers[this.layers.length - 1].neurons[0].activation;
+    let deltas = pred.map((p, i) => (targets[i] - p) * outAct.dfn(p));
     for (let l = this.layers.length - 1; l >= 0; l--) {
       const layer = this.layers[l];
       const layerIn = act[l];
+      const prevAct = l > 0 ? this.layers[l - 1].neurons[0].activation : null;
       const prevDeltas = layerIn.map((out, j) => {
         const errProp = layer.neurons.reduce((s, n, k) => s + deltas[k] * n.weights[j], 0);
-        return errProp * out * (1 - out);
+        return prevAct ? errProp * prevAct.dfn(out) : errProp;
       });
       layer.neurons.forEach((n, k) => {
-        n.weights = n.weights.map((w, j) => w + lr * deltas[k] * layerIn[j]);
-        n.bias += lr * deltas[k];
+        n._update(layerIn.map((inp) => deltas[k] * inp), deltas[k], lr);
       });
       deltas = prevDeltas;
     }
@@ -125,13 +195,13 @@ var NetworkN = class {
     for (let l = this.layers.length - 1; l >= 0; l--) {
       const layer = this.layers[l];
       const layerIn = act[l];
+      const prevAct = l > 0 ? this.layers[l - 1].neurons[0].activation : null;
       const prevDeltas = layerIn.map((out, j) => {
         const errProp = layer.neurons.reduce((s, n, k) => s + deltas[k] * n.weights[j], 0);
-        return errProp * out * (1 - out);
+        return prevAct ? errProp * prevAct.dfn(out) : errProp;
       });
       layer.neurons.forEach((n, k) => {
-        n.weights = n.weights.map((w, j) => w + lr * deltas[k] * layerIn[j]);
-        n.bias += lr * deltas[k];
+        n._update(layerIn.map((inp) => deltas[k] * inp), deltas[k], lr);
       });
       deltas = prevDeltas;
     }
@@ -142,7 +212,7 @@ var NetworkN = class {
 function sigmoid3(x) {
   return 1 / (1 + Math.exp(-x));
 }
-function tanh(x) {
+function tanh2(x) {
   const e = Math.exp(2 * x);
   return (e - 1) / (e + 1);
 }
@@ -191,10 +261,10 @@ var LSTMLayer = class {
     const zo = this.outputGate.linear(combined);
     const zf_a = zf.map(sigmoid3);
     const zi_a = zi.map(sigmoid3);
-    const zg_a = zg.map(tanh);
+    const zg_a = zg.map(tanh2);
     const zo_a = zo.map(sigmoid3);
     const c = c_prev.map((cv, k) => zf_a[k] * cv + zi_a[k] * zg_a[k]);
-    const h = zo_a.map((o, k) => o * tanh(c[k]));
+    const h = zo_a.map((o, k) => o * tanh2(c[k]));
     this._traj.push({ combined, c_prev, zf, zf_a, zi, zi_a, zg, zg_a, zo, zo_a, c, h });
     this.h = h;
     this.c = c;
@@ -222,7 +292,7 @@ var LSTMLayer = class {
     for (let t = T - 1; t >= 0; t--) {
       const s = this._traj[t];
       const dh = dh_seq[t].map((d, k) => d + dh_next[k]);
-      const tanh_c = s.c.map(tanh);
+      const tanh_c = s.c.map(tanh2);
       const do_a = dh.map((d, k) => d * tanh_c[k]);
       const dc = dh.map(
         (d, k) => d * s.zo_a[k] * (1 - tanh_c[k] ** 2) + dc_next[k]
@@ -291,16 +361,19 @@ var LSTMLayer = class {
 };
 
 // src/NetworkLSTM.ts
+var defaultOptimizer4 = () => new SGD();
 var NetworkLSTM = class {
   // [T][layer+1][neuron]
-  constructor(inputSize, hiddenSize, denseStructure) {
+  constructor(inputSize, hiddenSize, denseStructure, options = {}) {
     this.inputSize = inputSize;
     this.hiddenSize = hiddenSize;
     this.lstm = new LSTMLayer(inputSize, hiddenSize);
+    const activation = options.denseActivation ?? sigmoid2;
+    const optimizer = options.optimizer ?? defaultOptimizer4;
     this.denseLayers = [];
     const sizes = [hiddenSize, ...denseStructure];
     for (let i = 1; i < sizes.length; i++) {
-      this.denseLayers.push(new Layer(sizes[i], sizes[i - 1]));
+      this.denseLayers.push(new Layer(sizes[i], sizes[i - 1], activation, optimizer));
     }
     this._acts = [];
   }
@@ -333,14 +406,16 @@ var NetworkLSTM = class {
     for (let t = 0; t < T; t++) {
       const acts = this._acts[t];
       const pred = acts[acts.length - 1];
-      let deltas = pred.map((p, i) => (targets[t][i] - p) * p * (1 - p));
+      const outAct = this.denseLayers[this.denseLayers.length - 1].neurons[0].activation;
+      let deltas = pred.map((p, i) => (targets[t][i] - p) * outAct.dfn(p));
       for (let l = this.denseLayers.length - 1; l >= 0; l--) {
         const layer = this.denseLayers[l];
         const layerIn = acts[l];
         const grad = denseGrads[l];
+        const prevAct = l > 0 ? this.denseLayers[l - 1].neurons[0].activation : null;
         const prevDeltas = layerIn.map((out, j) => {
           const errProp = layer.neurons.reduce((s, n, k) => s + deltas[k] * n.weights[j], 0);
-          return l === 0 ? errProp : errProp * out * (1 - out);
+          return prevAct ? errProp * prevAct.dfn(out) : errProp;
         });
         layer.neurons.forEach((n, k) => {
           n.weights.forEach((_, j) => {
@@ -356,8 +431,11 @@ var NetworkLSTM = class {
       const layer = this.denseLayers[l];
       const grad = denseGrads[l];
       layer.neurons.forEach((n, k) => {
-        n.weights = n.weights.map((w, j) => w + lr / T * grad.dW[k][j]);
-        n.bias += lr / T * grad.db[k];
+        n._update(
+          grad.dW[k].map((g) => g / T),
+          grad.db[k] / T,
+          lr
+        );
       });
     }
     this.lstm.backprop(dh_seq, lr);
@@ -382,12 +460,47 @@ var NetworkLSTM = class {
     });
   }
 };
+
+// src/losses.ts
+function mse(predicted, actual) {
+  return predicted.reduce((sum, p, i) => sum + (actual[i] - p) ** 2, 0) / predicted.length;
+}
+function crossEntropy(predicted, actual) {
+  const eps = 1e-15;
+  return -predicted.reduce((sum, p, i) => {
+    const clipped = Math.max(eps, Math.min(1 - eps, p));
+    return sum + actual[i] * Math.log(clipped) + (1 - actual[i]) * Math.log(1 - clipped);
+  }, 0) / predicted.length;
+}
+function mseDelta(predicted, actual) {
+  return actual - predicted;
+}
+function crossEntropyDelta(predicted, actual) {
+  return actual - predicted;
+}
+function crossEntropyDeltaRaw(predicted, actual) {
+  const eps = 1e-15;
+  const p = Math.max(eps, Math.min(1 - eps, predicted));
+  return actual / p - (1 - actual) / (1 - p);
+}
 export {
+  Adam,
   LSTMLayer,
   Layer,
+  Momentum,
   Network,
   NetworkLSTM,
   NetworkN,
   Neuron,
-  NeuronN
+  NeuronN,
+  SGD,
+  crossEntropy,
+  crossEntropyDelta,
+  crossEntropyDeltaRaw,
+  linear,
+  mse,
+  mseDelta,
+  relu,
+  sigmoid2 as sigmoid,
+  tanh
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dniskav/neuron",
-  "version": "0.1.4",
+  "version": "0.1.5",
   "description": "Minimal neural network from scratch — neuron, layer, network, backpropagation. No dependencies.",
   "main": "dist/index.js",
   "module": "dist/index.mjs",