@dniskav/neuron 0.1.4 → 0.1.6

package/README.md

@@ -1,5 +1,3 @@
-# @dniskav/neuron
-
 [![npm](https://img.shields.io/npm/v/@dniskav/neuron)](https://www.npmjs.com/package/@dniskav/neuron)
 [![license](https://img.shields.io/npm/l/@dniskav/neuron)](LICENSE)
 
@@ -7,15 +5,19 @@ A minimal, dependency-free neural network library built from scratch in TypeScri
 
 ## What's inside
 
-| Class | Description |
-|-------|-------------|
+| Export | Description |
+|--------|-------------|
 | `Neuron` | Single-input neuron. The simplest possible unit: one weight, one bias. |
-| `NeuronN` | N-input neuron with Xavier initialization and sigmoid activation. |
+| `NeuronN` | N-input neuron with Xavier initialization and configurable activation. |
 | `Layer` | A group of `NeuronN` neurons that share the same inputs. |
 | `Network` | Two-layer network (hidden + output) with backpropagation. |
 | `NetworkN` | Deep network of arbitrary depth. Define your architecture as `[inputs, ...hidden, outputs]`. |
 | `LSTMLayer` | Recurrent layer with persistent hidden and cell state. Learns sequences via BPTT. |
 | `NetworkLSTM` | Wraps an `LSTMLayer` + dense layers. Maintains memory across steps within an episode. |
+| `sigmoid` `relu` `tanh` `linear` | Built-in activation functions. |
+| `SGD` `Momentum` `Adam` | Optimizers. Each instance tracks its own state per weight. |
+| `mse` `crossEntropy` | Loss functions for evaluation and logging. |
+| `mseDelta` `crossEntropyDelta` | Output-layer delta functions for use with `trainWithDeltas`. |
 
 ## Install
 
@@ -92,12 +94,73 @@ net.train([0.5, 0.3, 0.8], [1, 0], 0.05);
 const [out1, out2] = net.predict([0.5, 0.3, 0.8]);
 ```
 
+### Activations — ReLU, tanh, and more
+
+Pass an activation per layer. The last layer typically uses `sigmoid` for binary output or `linear` for regression.
+
+```ts
+import { NetworkN, relu, sigmoid } from "@dniskav/neuron";
+
+const net = new NetworkN([3, 64, 32, 1], {
+  activations: [relu, relu, sigmoid],
+});
+```
+
+Available: `sigmoid`, `relu`, `tanh`, `linear`.
+
+### Optimizers — Adam, Momentum, SGD
+
+Pass an optimizer factory. Each weight gets its own instance with independent state.
+
+```ts
+import { NetworkN, relu, sigmoid, Adam } from "@dniskav/neuron";
+
+const net = new NetworkN([2, 64, 1], {
+  activations: [relu, sigmoid],
+  optimizer: () => new Adam(),          // default: beta1=0.9, beta2=0.999
+});
+
+// Momentum example
+import { Momentum } from "@dniskav/neuron";
+const net2 = new NetworkN([2, 32, 1], {
+  optimizer: () => new Momentum(0.9),
+});
+```
+
+Optimizers also work in `NetworkLSTM` (applied to the dense layers):
+
+```ts
+import { NetworkLSTM, relu, Adam } from "@dniskav/neuron";
+
+const net = new NetworkLSTM(1, 8, [4, 1], {
+  denseActivation: relu,
+  optimizer: () => new Adam(0.001),
+});
+```
+
+### Loss utilities
+
+```ts
+import { mse, crossEntropy } from "@dniskav/neuron";
+
+const predicted = net.predict([0.5, 0.3]);
+console.log(mse(predicted, [1, 0]));
+console.log(crossEntropy(predicted, [1, 0]));
+```
+
 ### trainWithDeltas — custom loss / physics-based gradients
 
 `NetworkN` also exposes `trainWithDeltas` for when you compute your own output-layer deltas (e.g., from a physics simulation or a custom loss function):
 
 ```ts
-net.trainWithDeltas(inputs, [0.4, -0.2], 0.05);
+import { NetworkN, mseDelta } from "@dniskav/neuron";
+
+const net = new NetworkN([3, 16, 2]);
+const pred = net.predict(inputs);
+
+// Compute deltas manually using a helper, or from any external signal
+const deltas = pred.map((p, i) => mseDelta(p, targets[i]));
+net.trainWithDeltas(inputs, deltas, 0.01);
 ```
 
 ### NetworkLSTM — recurrent network with memory
@@ -143,17 +206,19 @@ The network learns to count steps using its hidden state — no external counter
 
 ## How it works
 
-Every class uses **sigmoid** as its activation function and **gradient descent** to update weights:
+Each class applies an **activation function** to the weighted sum of inputs and uses **gradient descent** to update weights:
 
 ```
-weight += lr × error × input
-bias   += lr × error
+weight += lr × delta × input
+bias   += lr × delta
 ```
 
-`NetworkN` implements full **backpropagation** across all layers, propagating deltas from the output back to the first layer using the chain rule.
+`NetworkN` implements full **backpropagation** across all layers, propagating deltas from the output back to the first layer using the chain rule. The derivative of the chosen activation is applied at each layer.
 
 `NeuronN` uses simplified **Xavier initialization** — weights start in `[-√(1/n), +√(1/n)]` — so gradients flow well from the start of training.
 
+When an **optimizer** is used (e.g., Adam), the raw gradient is passed to the optimizer instead of being applied directly. Each weight maintains its own optimizer state (velocity, moments).
+
 ## Build
 
 ```bash
@@ -165,6 +230,11 @@ npm run dev     # watch mode
 
 If you are an AI agent or LLM working with this codebase, read [AGENTS.md](AGENTS.md) first. It contains the full class hierarchy, design constraints, and what this library does not do.
 
+## Possible improvements
+
+1. **Support for batches** in training to improve efficiency.
+2. **Improve documentation** with more advanced examples and use cases.
+
 ## License
 
 MIT

package/dist/index.d.mts

@@ -6,17 +6,59 @@ declare class Neuron {
     train(input: number, target: number, lr: number): void;
 }
 
+interface Activation {
+    fn(x: number): number;
+    dfn(out: number): number;
+}
+declare const sigmoid: Activation;
+declare const tanh: Activation;
+declare const relu: Activation;
+declare const linear: Activation;
+declare function makeLeakyRelu(alpha?: number): Activation;
+/** Leaky ReLU with the standard α = 0.01. */
+declare const leakyRelu: Activation;
+declare function makeElu(alpha?: number): Activation;
+/** ELU with the standard α = 1.0. */
+declare const elu: Activation;
+
+interface Optimizer {
+    step(weight: number, gradient: number, lr: number): number;
+}
+type OptimizerFactory = () => Optimizer;
+declare class SGD implements Optimizer {
+    step(weight: number, gradient: number, lr: number): number;
+}
+declare class Momentum implements Optimizer {
+    readonly beta: number;
+    private v;
+    constructor(beta?: number);
+    step(weight: number, gradient: number, lr: number): number;
+}
+declare class Adam implements Optimizer {
+    readonly beta1: number;
+    readonly beta2: number;
+    readonly epsilon: number;
+    private m;
+    private v;
+    private t;
+    constructor(beta1?: number, beta2?: number, epsilon?: number);
+    step(weight: number, gradient: number, lr: number): number;
+}
+
 declare class NeuronN {
     weights: number[];
     bias: number;
-    constructor(nInputs: number);
+    readonly activation: Activation;
+    private _opts;
+    constructor(nInputs: number, activation?: Activation, optimizerFactory?: OptimizerFactory);
     predict(inputs: number[]): number;
+    _update(weightGrads: number[], biasGrad: number, lr: number): void;
     train(inputs: number[], target: number, lr: number): void;
 }
 
 declare class Layer {
     neurons: NeuronN[];
-    constructor(nNeurons: number, nInputs: number);
+    constructor(nNeurons: number, nInputs: number, activation?: Activation, optimizerFactory?: OptimizerFactory);
     predict(inputs: number[]): number[];
 }
 
@@ -28,10 +70,14 @@ declare class Network {
     train(inputs: number[], target: number, lr: number): number;
 }
 
+interface NetworkNOptions {
+    activations?: Activation[];
+    optimizer?: OptimizerFactory;
+}
 declare class NetworkN {
     readonly structure: number[];
     layers: Layer[];
-    constructor(structure: number[]);
+    constructor(structure: number[], options?: NetworkNOptions);
     predict(inputs: number[]): number[];
     train(inputs: number[], targets: number[], lr: number): number;
     trainWithDeltas(inputs: number[], outputDeltas: number[], lr: number): void;
@@ -78,13 +124,17 @@ declare class LSTMLayer {
     setWeights(data: ReturnType<LSTMLayer["getWeights"]>): void;
 }
 
+interface NetworkLSTMOptions {
+    denseActivation?: Activation;
+    optimizer?: OptimizerFactory;
+}
 declare class NetworkLSTM {
     readonly inputSize: number;
     readonly hiddenSize: number;
     lstm: LSTMLayer;
     denseLayers: Layer[];
     private _acts;
-    constructor(inputSize: number, hiddenSize: number, denseStructure: number[]);
+    constructor(inputSize: number, hiddenSize: number, denseStructure: number[], options?: NetworkLSTMOptions);
     resetState(): void;
     predict(inputs: number[]): number[];
     train(targets: number[][], lr: number): void;
@@ -115,4 +165,10 @@ declare class NetworkLSTM {
     setWeights(data: ReturnType<NetworkLSTM["getWeights"]>): void;
 }
 
-export { LSTMLayer, Layer, Network, NetworkLSTM, NetworkN, Neuron, NeuronN };
+declare function mse(predicted: number[], actual: number[]): number;
+declare function crossEntropy(predicted: number[], actual: number[]): number;
+declare function mseDelta(predicted: number, actual: number): number;
+declare function crossEntropyDelta(predicted: number, actual: number): number;
+declare function crossEntropyDeltaRaw(predicted: number, actual: number): number;
+
+export { type Activation, Adam, LSTMLayer, Layer, Momentum, Network, NetworkLSTM, type NetworkLSTMOptions, NetworkN, type NetworkNOptions, Neuron, NeuronN, type Optimizer, type OptimizerFactory, SGD, crossEntropy, crossEntropyDelta, crossEntropyDeltaRaw, elu, leakyRelu, linear, makeElu, makeLeakyRelu, mse, mseDelta, relu, sigmoid, tanh };

package/dist/index.d.ts

@@ -6,17 +6,59 @@ declare class Neuron {
     train(input: number, target: number, lr: number): void;
 }
 
+interface Activation {
+    fn(x: number): number;
+    dfn(out: number): number;
+}
+declare const sigmoid: Activation;
+declare const tanh: Activation;
+declare const relu: Activation;
+declare const linear: Activation;
+declare function makeLeakyRelu(alpha?: number): Activation;
+/** Leaky ReLU with the standard α = 0.01. */
+declare const leakyRelu: Activation;
+declare function makeElu(alpha?: number): Activation;
+/** ELU with the standard α = 1.0. */
+declare const elu: Activation;
+
+interface Optimizer {
+    step(weight: number, gradient: number, lr: number): number;
+}
+type OptimizerFactory = () => Optimizer;
+declare class SGD implements Optimizer {
+    step(weight: number, gradient: number, lr: number): number;
+}
+declare class Momentum implements Optimizer {
+    readonly beta: number;
+    private v;
+    constructor(beta?: number);
+    step(weight: number, gradient: number, lr: number): number;
+}
+declare class Adam implements Optimizer {
+    readonly beta1: number;
+    readonly beta2: number;
+    readonly epsilon: number;
+    private m;
+    private v;
+    private t;
+    constructor(beta1?: number, beta2?: number, epsilon?: number);
+    step(weight: number, gradient: number, lr: number): number;
+}
+
 declare class NeuronN {
     weights: number[];
     bias: number;
-    constructor(nInputs: number);
+    readonly activation: Activation;
+    private _opts;
+    constructor(nInputs: number, activation?: Activation, optimizerFactory?: OptimizerFactory);
     predict(inputs: number[]): number;
+    _update(weightGrads: number[], biasGrad: number, lr: number): void;
     train(inputs: number[], target: number, lr: number): void;
 }
 
 declare class Layer {
     neurons: NeuronN[];
-    constructor(nNeurons: number, nInputs: number);
+    constructor(nNeurons: number, nInputs: number, activation?: Activation, optimizerFactory?: OptimizerFactory);
     predict(inputs: number[]): number[];
 }
 
@@ -28,10 +70,14 @@ declare class Network {
     train(inputs: number[], target: number, lr: number): number;
 }
 
+interface NetworkNOptions {
+    activations?: Activation[];
+    optimizer?: OptimizerFactory;
+}
 declare class NetworkN {
     readonly structure: number[];
     layers: Layer[];
-    constructor(structure: number[]);
+    constructor(structure: number[], options?: NetworkNOptions);
     predict(inputs: number[]): number[];
     train(inputs: number[], targets: number[], lr: number): number;
     trainWithDeltas(inputs: number[], outputDeltas: number[], lr: number): void;
@@ -78,13 +124,17 @@ declare class LSTMLayer {
     setWeights(data: ReturnType<LSTMLayer["getWeights"]>): void;
 }
 
+interface NetworkLSTMOptions {
+    denseActivation?: Activation;
+    optimizer?: OptimizerFactory;
+}
 declare class NetworkLSTM {
     readonly inputSize: number;
     readonly hiddenSize: number;
     lstm: LSTMLayer;
     denseLayers: Layer[];
     private _acts;
-    constructor(inputSize: number, hiddenSize: number, denseStructure: number[]);
+    constructor(inputSize: number, hiddenSize: number, denseStructure: number[], options?: NetworkLSTMOptions);
     resetState(): void;
     predict(inputs: number[]): number[];
     train(targets: number[][], lr: number): void;
@@ -115,4 +165,10 @@ declare class NetworkLSTM {
     setWeights(data: ReturnType<NetworkLSTM["getWeights"]>): void;
 }
 
-export { LSTMLayer, Layer, Network, NetworkLSTM, NetworkN, Neuron, NeuronN };
+declare function mse(predicted: number[], actual: number[]): number;
+declare function crossEntropy(predicted: number[], actual: number[]): number;
+declare function mseDelta(predicted: number, actual: number): number;
+declare function crossEntropyDelta(predicted: number, actual: number): number;
+declare function crossEntropyDeltaRaw(predicted: number, actual: number): number;
+
+export { type Activation, Adam, LSTMLayer, Layer, Momentum, Network, NetworkLSTM, type NetworkLSTMOptions, NetworkN, type NetworkNOptions, Neuron, NeuronN, type Optimizer, type OptimizerFactory, SGD, crossEntropy, crossEntropyDelta, crossEntropyDeltaRaw, elu, leakyRelu, linear, makeElu, makeLeakyRelu, mse, mseDelta, relu, sigmoid, tanh };

package/dist/index.js

@@ -20,13 +20,29 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
 // src/index.ts
 var index_exports = {};
 __export(index_exports, {
+  Adam: () => Adam,
   LSTMLayer: () => LSTMLayer,
   Layer: () => Layer,
+  Momentum: () => Momentum,
   Network: () => Network,
   NetworkLSTM: () => NetworkLSTM,
   NetworkN: () => NetworkN,
   Neuron: () => Neuron,
-  NeuronN: () => NeuronN
+  NeuronN: () => NeuronN,
+  SGD: () => SGD,
+  crossEntropy: () => crossEntropy,
+  crossEntropyDelta: () => crossEntropyDelta,
+  crossEntropyDeltaRaw: () => crossEntropyDeltaRaw,
+  elu: () => elu,
+  leakyRelu: () => leakyRelu,
+  linear: () => linear,
+  makeElu: () => makeElu,
+  makeLeakyRelu: () => makeLeakyRelu,
+  mse: () => mse,
+  mseDelta: () => mseDelta,
+  relu: () => relu,
+  sigmoid: () => sigmoid2,
+  tanh: () => tanh
 });
 module.exports = __toCommonJS(index_exports);
 
@@ -50,32 +66,111 @@ var Neuron = class {
   }
 };
 
-// src/NeuronN.ts
-function sigmoid2(x) {
-  return 1 / (1 + Math.exp(-x));
+// src/activations.ts
+var sigmoid2 = {
+  fn: (x) => 1 / (1 + Math.exp(-x)),
+  dfn: (out) => out * (1 - out)
+};
+var tanh = {
+  fn: (x) => {
+    const e = Math.exp(2 * x);
+    return (e - 1) / (e + 1);
+  },
+  dfn: (out) => 1 - out * out
+};
+var relu = {
+  fn: (x) => Math.max(0, x),
+  dfn: (out) => out > 0 ? 1 : 0
+};
+var linear = {
+  fn: (x) => x,
+  dfn: () => 1
+};
+function makeLeakyRelu(alpha = 0.01) {
+  return {
+    fn: (x) => x > 0 ? x : alpha * x,
+    dfn: (out) => out > 0 ? 1 : alpha
+  };
+}
+var leakyRelu = makeLeakyRelu(0.01);
+function makeElu(alpha = 1) {
+  return {
+    fn: (x) => x > 0 ? x : alpha * (Math.exp(x) - 1),
+    dfn: (out) => out > 0 ? 1 : out + alpha
+  };
 }
+var elu = makeElu(1);
+
+// src/optimizers.ts
+var SGD = class {
+  step(weight, gradient, lr) {
+    return weight + lr * gradient;
+  }
+};
+var Momentum = class {
+  constructor(beta = 0.9) {
+    this.beta = beta;
+    this.v = 0;
+  }
+  step(weight, gradient, lr) {
+    this.v = this.beta * this.v + lr * gradient;
+    return weight + this.v;
+  }
+};
+var Adam = class {
+  constructor(beta1 = 0.9, beta2 = 0.999, epsilon = 1e-8) {
+    this.beta1 = beta1;
+    this.beta2 = beta2;
+    this.epsilon = epsilon;
+    this.m = 0;
+    this.v = 0;
+    this.t = 0;
+  }
+  step(weight, gradient, lr) {
+    this.t++;
+    this.m = this.beta1 * this.m + (1 - this.beta1) * gradient;
+    this.v = this.beta2 * this.v + (1 - this.beta2) * gradient * gradient;
+    const mHat = this.m / (1 - Math.pow(this.beta1, this.t));
+    const vHat = this.v / (1 - Math.pow(this.beta2, this.t));
+    return weight + lr * mHat / (Math.sqrt(vHat) + this.epsilon);
+  }
+};
+
+// src/NeuronN.ts
+var defaultOptimizer = () => new SGD();
 var NeuronN = class {
-  constructor(nInputs) {
+  constructor(nInputs, activation = sigmoid2, optimizerFactory = defaultOptimizer) {
     const limit = Math.sqrt(1 / nInputs);
     this.weights = Array.from({ length: nInputs }, () => (Math.random() * 2 - 1) * limit);
     this.bias = 0;
+    this.activation = activation;
+    this._opts = Array.from({ length: nInputs + 1 }, optimizerFactory);
   }
   predict(inputs) {
     const sum = inputs.reduce((acc, e, i) => acc + e * this.weights[i], this.bias);
-    return sigmoid2(sum);
+    return this.activation.fn(sum);
+  }
+  // Apply pre-computed gradients via the optimizer.
+  // Called internally by Layer / NetworkN / NetworkLSTM during backprop.
+  _update(weightGrads, biasGrad, lr) {
+    this.weights = this.weights.map((w, i) => this._opts[i].step(w, weightGrads[i], lr));
+    this.bias = this._opts[this.weights.length].step(this.bias, biasGrad, lr);
   }
   train(inputs, target, lr) {
     const prediction = this.predict(inputs);
     const error = target - prediction;
-    this.weights = this.weights.map((w, i) => w + lr * error * inputs[i]);
-    this.bias += lr * error;
+    this._update(inputs.map((inp) => error * inp), error, lr);
   }
 };
 
 // src/Layer.ts
+var defaultOptimizer2 = () => new SGD();
 var Layer = class {
-  constructor(nNeurons, nInputs) {
-    this.neurons = Array.from({ length: nNeurons }, () => new NeuronN(nInputs));
+  constructor(nNeurons, nInputs, activation = sigmoid2, optimizerFactory = defaultOptimizer2) {
+    this.neurons = Array.from(
+      { length: nNeurons },
+      () => new NeuronN(nInputs, activation, optimizerFactory)
+    );
   }
   predict(inputs) {
     return this.neurons.map((n) => n.predict(inputs));
@@ -115,12 +210,16 @@ var Network = class {
 };
 
 // src/NetworkN.ts
+var defaultOptimizer3 = () => new SGD();
 var NetworkN = class {
-  constructor(structure) {
+  constructor(structure, options = {}) {
     this.structure = structure;
+    const nLayers = structure.length - 1;
+    const activations = options.activations ?? Array.from({ length: nLayers }, () => sigmoid2);
+    const optimizer = options.optimizer ?? defaultOptimizer3;
     this.layers = [];
     for (let i = 1; i < structure.length; i++) {
-      this.layers.push(new Layer(structure[i], structure[i - 1]));
+      this.layers.push(new Layer(structure[i], structure[i - 1], activations[i - 1], optimizer));
     }
   }
   predict(inputs) {
@@ -132,17 +231,18 @@ var NetworkN = class {
     const act = [inputs];
     for (const layer of this.layers) act.push(layer.predict(act[act.length - 1]));
     const pred = act[act.length - 1];
-    let deltas = pred.map((p, i) => (targets[i] - p) * p * (1 - p));
+    const outAct = this.layers[this.layers.length - 1].neurons[0].activation;
+    let deltas = pred.map((p, i) => (targets[i] - p) * outAct.dfn(p));
     for (let l = this.layers.length - 1; l >= 0; l--) {
       const layer = this.layers[l];
       const layerIn = act[l];
+      const prevAct = l > 0 ? this.layers[l - 1].neurons[0].activation : null;
       const prevDeltas = layerIn.map((out, j) => {
         const errProp = layer.neurons.reduce((s, n, k) => s + deltas[k] * n.weights[j], 0);
-        return errProp * out * (1 - out);
+        return prevAct ? errProp * prevAct.dfn(out) : errProp;
       });
       layer.neurons.forEach((n, k) => {
-        n.weights = n.weights.map((w, j) => w + lr * deltas[k] * layerIn[j]);
-        n.bias += lr * deltas[k];
+        n._update(layerIn.map((inp) => deltas[k] * inp), deltas[k], lr);
       });
       deltas = prevDeltas;
     }
@@ -157,13 +257,13 @@ var NetworkN = class {
     for (let l = this.layers.length - 1; l >= 0; l--) {
       const layer = this.layers[l];
       const layerIn = act[l];
+      const prevAct = l > 0 ? this.layers[l - 1].neurons[0].activation : null;
       const prevDeltas = layerIn.map((out, j) => {
         const errProp = layer.neurons.reduce((s, n, k) => s + deltas[k] * n.weights[j], 0);
-        return errProp * out * (1 - out);
+        return prevAct ? errProp * prevAct.dfn(out) : errProp;
       });
       layer.neurons.forEach((n, k) => {
-        n.weights = n.weights.map((w, j) => w + lr * deltas[k] * layerIn[j]);
-        n.bias += lr * deltas[k];
+        n._update(layerIn.map((inp) => deltas[k] * inp), deltas[k], lr);
       });
       deltas = prevDeltas;
     }
@@ -174,7 +274,7 @@ var NetworkN = class {
 function sigmoid3(x) {
   return 1 / (1 + Math.exp(-x));
 }
-function tanh(x) {
+function tanh2(x) {
   const e = Math.exp(2 * x);
   return (e - 1) / (e + 1);
 }
@@ -223,10 +323,10 @@ var LSTMLayer = class {
     const zo = this.outputGate.linear(combined);
     const zf_a = zf.map(sigmoid3);
     const zi_a = zi.map(sigmoid3);
-    const zg_a = zg.map(tanh);
+    const zg_a = zg.map(tanh2);
     const zo_a = zo.map(sigmoid3);
     const c = c_prev.map((cv, k) => zf_a[k] * cv + zi_a[k] * zg_a[k]);
-    const h = zo_a.map((o, k) => o * tanh(c[k]));
+    const h = zo_a.map((o, k) => o * tanh2(c[k]));
     this._traj.push({ combined, c_prev, zf, zf_a, zi, zi_a, zg, zg_a, zo, zo_a, c, h });
     this.h = h;
     this.c = c;
@@ -254,7 +354,7 @@ var LSTMLayer = class {
     for (let t = T - 1; t >= 0; t--) {
       const s = this._traj[t];
       const dh = dh_seq[t].map((d, k) => d + dh_next[k]);
-      const tanh_c = s.c.map(tanh);
+      const tanh_c = s.c.map(tanh2);
       const do_a = dh.map((d, k) => d * tanh_c[k]);
       const dc = dh.map(
         (d, k) => d * s.zo_a[k] * (1 - tanh_c[k] ** 2) + dc_next[k]
@@ -323,16 +423,19 @@ var LSTMLayer = class {
 };
 
 // src/NetworkLSTM.ts
+var defaultOptimizer4 = () => new SGD();
 var NetworkLSTM = class {
   // [T][layer+1][neuron]
-  constructor(inputSize, hiddenSize, denseStructure) {
+  constructor(inputSize, hiddenSize, denseStructure, options = {}) {
     this.inputSize = inputSize;
     this.hiddenSize = hiddenSize;
     this.lstm = new LSTMLayer(inputSize, hiddenSize);
+    const activation = options.denseActivation ?? sigmoid2;
+    const optimizer = options.optimizer ?? defaultOptimizer4;
     this.denseLayers = [];
     const sizes = [hiddenSize, ...denseStructure];
     for (let i = 1; i < sizes.length; i++) {
-      this.denseLayers.push(new Layer(sizes[i], sizes[i - 1]));
+      this.denseLayers.push(new Layer(sizes[i], sizes[i - 1], activation, optimizer));
     }
     this._acts = [];
   }
@@ -365,14 +468,16 @@ var NetworkLSTM = class {
     for (let t = 0; t < T; t++) {
       const acts = this._acts[t];
       const pred = acts[acts.length - 1];
-      let deltas = pred.map((p, i) => (targets[t][i] - p) * p * (1 - p));
+      const outAct = this.denseLayers[this.denseLayers.length - 1].neurons[0].activation;
+      let deltas = pred.map((p, i) => (targets[t][i] - p) * outAct.dfn(p));
       for (let l = this.denseLayers.length - 1; l >= 0; l--) {
         const layer = this.denseLayers[l];
         const layerIn = acts[l];
         const grad = denseGrads[l];
+        const prevAct = l > 0 ? this.denseLayers[l - 1].neurons[0].activation : null;
         const prevDeltas = layerIn.map((out, j) => {
           const errProp = layer.neurons.reduce((s, n, k) => s + deltas[k] * n.weights[j], 0);
-          return l === 0 ? errProp : errProp * out * (1 - out);
+          return prevAct ? errProp * prevAct.dfn(out) : errProp;
         });
         layer.neurons.forEach((n, k) => {
           n.weights.forEach((_, j) => {
@@ -388,8 +493,11 @@ var NetworkLSTM = class {
       const layer = this.denseLayers[l];
       const grad = denseGrads[l];
       layer.neurons.forEach((n, k) => {
-        n.weights = n.weights.map((w, j) => w + lr / T * grad.dW[k][j]);
-        n.bias += lr / T * grad.db[k];
+        n._update(
+          grad.dW[k].map((g) => g / T),
+          grad.db[k] / T,
+          lr
+        );
       });
     }
     this.lstm.backprop(dh_seq, lr);
@@ -414,13 +522,52 @@ var NetworkLSTM = class {
     });
   }
 };
+
+// src/losses.ts
+function mse(predicted, actual) {
+  return predicted.reduce((sum, p, i) => sum + (actual[i] - p) ** 2, 0) / predicted.length;
+}
+function crossEntropy(predicted, actual) {
+  const eps = 1e-15;
+  return -predicted.reduce((sum, p, i) => {
+    const clipped = Math.max(eps, Math.min(1 - eps, p));
+    return sum + actual[i] * Math.log(clipped) + (1 - actual[i]) * Math.log(1 - clipped);
+  }, 0) / predicted.length;
+}
+function mseDelta(predicted, actual) {
+  return actual - predicted;
+}
+function crossEntropyDelta(predicted, actual) {
+  return actual - predicted;
+}
+function crossEntropyDeltaRaw(predicted, actual) {
+  const eps = 1e-15;
+  const p = Math.max(eps, Math.min(1 - eps, predicted));
+  return actual / p - (1 - actual) / (1 - p);
+}
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
+  Adam,
   LSTMLayer,
   Layer,
+  Momentum,
   Network,
   NetworkLSTM,
   NetworkN,
   Neuron,
-  NeuronN
+  NeuronN,
+  SGD,
+  crossEntropy,
+  crossEntropyDelta,
+  crossEntropyDeltaRaw,
+  elu,
+  leakyRelu,
+  linear,
+  makeElu,
+  makeLeakyRelu,
+  mse,
+  mseDelta,
+  relu,
+  sigmoid,
+  tanh
 });

package/dist/index.mjs

@@ -18,32 +18,111 @@ var Neuron = class {
   }
 };
 
-// src/NeuronN.ts
-function sigmoid2(x) {
-  return 1 / (1 + Math.exp(-x));
+// src/activations.ts
+var sigmoid2 = {
+  fn: (x) => 1 / (1 + Math.exp(-x)),
+  dfn: (out) => out * (1 - out)
+};
+var tanh = {
+  fn: (x) => {
+    const e = Math.exp(2 * x);
+    return (e - 1) / (e + 1);
+  },
+  dfn: (out) => 1 - out * out
+};
+var relu = {
+  fn: (x) => Math.max(0, x),
+  dfn: (out) => out > 0 ? 1 : 0
+};
+var linear = {
+  fn: (x) => x,
+  dfn: () => 1
+};
+function makeLeakyRelu(alpha = 0.01) {
+  return {
+    fn: (x) => x > 0 ? x : alpha * x,
+    dfn: (out) => out > 0 ? 1 : alpha
+  };
+}
+var leakyRelu = makeLeakyRelu(0.01);
+function makeElu(alpha = 1) {
+  return {
+    fn: (x) => x > 0 ? x : alpha * (Math.exp(x) - 1),
+    dfn: (out) => out > 0 ? 1 : out + alpha
+  };
 }
+var elu = makeElu(1);
+
+// src/optimizers.ts
+var SGD = class {
+  step(weight, gradient, lr) {
+    return weight + lr * gradient;
+  }
+};
+var Momentum = class {
+  constructor(beta = 0.9) {
+    this.beta = beta;
+    this.v = 0;
+  }
+  step(weight, gradient, lr) {
+    this.v = this.beta * this.v + lr * gradient;
+    return weight + this.v;
+  }
+};
+var Adam = class {
+  constructor(beta1 = 0.9, beta2 = 0.999, epsilon = 1e-8) {
+    this.beta1 = beta1;
+    this.beta2 = beta2;
+    this.epsilon = epsilon;
+    this.m = 0;
+    this.v = 0;
+    this.t = 0;
+  }
+  step(weight, gradient, lr) {
+    this.t++;
+    this.m = this.beta1 * this.m + (1 - this.beta1) * gradient;
+    this.v = this.beta2 * this.v + (1 - this.beta2) * gradient * gradient;
+    const mHat = this.m / (1 - Math.pow(this.beta1, this.t));
+    const vHat = this.v / (1 - Math.pow(this.beta2, this.t));
+    return weight + lr * mHat / (Math.sqrt(vHat) + this.epsilon);
+  }
+};
+
+// src/NeuronN.ts
+var defaultOptimizer = () => new SGD();
 var NeuronN = class {
-  constructor(nInputs) {
+  constructor(nInputs, activation = sigmoid2, optimizerFactory = defaultOptimizer) {
     const limit = Math.sqrt(1 / nInputs);
     this.weights = Array.from({ length: nInputs }, () => (Math.random() * 2 - 1) * limit);
     this.bias = 0;
+    this.activation = activation;
+    this._opts = Array.from({ length: nInputs + 1 }, optimizerFactory);
   }
   predict(inputs) {
     const sum = inputs.reduce((acc, e, i) => acc + e * this.weights[i], this.bias);
-    return sigmoid2(sum);
+    return this.activation.fn(sum);
+  }
+  // Apply pre-computed gradients via the optimizer.
+  // Called internally by Layer / NetworkN / NetworkLSTM during backprop.
+  _update(weightGrads, biasGrad, lr) {
+    this.weights = this.weights.map((w, i) => this._opts[i].step(w, weightGrads[i], lr));
+    this.bias = this._opts[this.weights.length].step(this.bias, biasGrad, lr);
   }
   train(inputs, target, lr) {
     const prediction = this.predict(inputs);
     const error = target - prediction;
-    this.weights = this.weights.map((w, i) => w + lr * error * inputs[i]);
-    this.bias += lr * error;
+    this._update(inputs.map((inp) => error * inp), error, lr);
   }
 };
 
 // src/Layer.ts
+var defaultOptimizer2 = () => new SGD();
 var Layer = class {
-  constructor(nNeurons, nInputs) {
-    this.neurons = Array.from({ length: nNeurons }, () => new NeuronN(nInputs));
+  constructor(nNeurons, nInputs, activation = sigmoid2, optimizerFactory = defaultOptimizer2) {
+    this.neurons = Array.from(
+      { length: nNeurons },
+      () => new NeuronN(nInputs, activation, optimizerFactory)
+    );
   }
   predict(inputs) {
     return this.neurons.map((n) => n.predict(inputs));
@@ -83,12 +162,16 @@ var Network = class {
 };
 
 // src/NetworkN.ts
+var defaultOptimizer3 = () => new SGD();
 var NetworkN = class {
-  constructor(structure) {
+  constructor(structure, options = {}) {
     this.structure = structure;
+    const nLayers = structure.length - 1;
+    const activations = options.activations ?? Array.from({ length: nLayers }, () => sigmoid2);
+    const optimizer = options.optimizer ?? defaultOptimizer3;
     this.layers = [];
     for (let i = 1; i < structure.length; i++) {
-      this.layers.push(new Layer(structure[i], structure[i - 1]));
+      this.layers.push(new Layer(structure[i], structure[i - 1], activations[i - 1], optimizer));
     }
   }
   predict(inputs) {
@@ -100,17 +183,18 @@ var NetworkN = class {
     const act = [inputs];
     for (const layer of this.layers) act.push(layer.predict(act[act.length - 1]));
     const pred = act[act.length - 1];
-    let deltas = pred.map((p, i) => (targets[i] - p) * p * (1 - p));
+    const outAct = this.layers[this.layers.length - 1].neurons[0].activation;
+    let deltas = pred.map((p, i) => (targets[i] - p) * outAct.dfn(p));
     for (let l = this.layers.length - 1; l >= 0; l--) {
       const layer = this.layers[l];
       const layerIn = act[l];
+      const prevAct = l > 0 ? this.layers[l - 1].neurons[0].activation : null;
       const prevDeltas = layerIn.map((out, j) => {
         const errProp = layer.neurons.reduce((s, n, k) => s + deltas[k] * n.weights[j], 0);
-        return errProp * out * (1 - out);
+        return prevAct ? errProp * prevAct.dfn(out) : errProp;
       });
       layer.neurons.forEach((n, k) => {
-        n.weights = n.weights.map((w, j) => w + lr * deltas[k] * layerIn[j]);
-        n.bias += lr * deltas[k];
+        n._update(layerIn.map((inp) => deltas[k] * inp), deltas[k], lr);
       });
       deltas = prevDeltas;
     }
@@ -125,13 +209,13 @@ var NetworkN = class {
     for (let l = this.layers.length - 1; l >= 0; l--) {
       const layer = this.layers[l];
       const layerIn = act[l];
+      const prevAct = l > 0 ? this.layers[l - 1].neurons[0].activation : null;
       const prevDeltas = layerIn.map((out, j) => {
         const errProp = layer.neurons.reduce((s, n, k) => s + deltas[k] * n.weights[j], 0);
-        return errProp * out * (1 - out);
+        return prevAct ? errProp * prevAct.dfn(out) : errProp;
       });
       layer.neurons.forEach((n, k) => {
-        n.weights = n.weights.map((w, j) => w + lr * deltas[k] * layerIn[j]);
-        n.bias += lr * deltas[k];
+        n._update(layerIn.map((inp) => deltas[k] * inp), deltas[k], lr);
       });
       deltas = prevDeltas;
     }
@@ -142,7 +226,7 @@ var NetworkN = class {
 function sigmoid3(x) {
   return 1 / (1 + Math.exp(-x));
 }
-function tanh(x) {
+function tanh2(x) {
   const e = Math.exp(2 * x);
   return (e - 1) / (e + 1);
 }
@@ -191,10 +275,10 @@ var LSTMLayer = class {
     const zo = this.outputGate.linear(combined);
     const zf_a = zf.map(sigmoid3);
     const zi_a = zi.map(sigmoid3);
-    const zg_a = zg.map(tanh);
+    const zg_a = zg.map(tanh2);
     const zo_a = zo.map(sigmoid3);
     const c = c_prev.map((cv, k) => zf_a[k] * cv + zi_a[k] * zg_a[k]);
-    const h = zo_a.map((o, k) => o * tanh(c[k]));
+    const h = zo_a.map((o, k) => o * tanh2(c[k]));
     this._traj.push({ combined, c_prev, zf, zf_a, zi, zi_a, zg, zg_a, zo, zo_a, c, h });
     this.h = h;
     this.c = c;
@@ -222,7 +306,7 @@ var LSTMLayer = class {
     for (let t = T - 1; t >= 0; t--) {
       const s = this._traj[t];
       const dh = dh_seq[t].map((d, k) => d + dh_next[k]);
-      const tanh_c = s.c.map(tanh);
+      const tanh_c = s.c.map(tanh2);
       const do_a = dh.map((d, k) => d * tanh_c[k]);
       const dc = dh.map(
         (d, k) => d * s.zo_a[k] * (1 - tanh_c[k] ** 2) + dc_next[k]
@@ -291,16 +375,19 @@ var LSTMLayer = class {
 };
 
 // src/NetworkLSTM.ts
+var defaultOptimizer4 = () => new SGD();
 var NetworkLSTM = class {
   // [T][layer+1][neuron]
-  constructor(inputSize, hiddenSize, denseStructure) {
+  constructor(inputSize, hiddenSize, denseStructure, options = {}) {
     this.inputSize = inputSize;
     this.hiddenSize = hiddenSize;
     this.lstm = new LSTMLayer(inputSize, hiddenSize);
+    const activation = options.denseActivation ?? sigmoid2;
+    const optimizer = options.optimizer ?? defaultOptimizer4;
     this.denseLayers = [];
     const sizes = [hiddenSize, ...denseStructure];
     for (let i = 1; i < sizes.length; i++) {
-      this.denseLayers.push(new Layer(sizes[i], sizes[i - 1]));
+      this.denseLayers.push(new Layer(sizes[i], sizes[i - 1], activation, optimizer));
     }
     this._acts = [];
   }
@@ -333,14 +420,16 @@ var NetworkLSTM = class {
     for (let t = 0; t < T; t++) {
       const acts = this._acts[t];
       const pred = acts[acts.length - 1];
-      let deltas = pred.map((p, i) => (targets[t][i] - p) * p * (1 - p));
+      const outAct = this.denseLayers[this.denseLayers.length - 1].neurons[0].activation;
+      let deltas = pred.map((p, i) => (targets[t][i] - p) * outAct.dfn(p));
       for (let l = this.denseLayers.length - 1; l >= 0; l--) {
         const layer = this.denseLayers[l];
         const layerIn = acts[l];
         const grad = denseGrads[l];
+        const prevAct = l > 0 ? this.denseLayers[l - 1].neurons[0].activation : null;
         const prevDeltas = layerIn.map((out, j) => {
           const errProp = layer.neurons.reduce((s, n, k) => s + deltas[k] * n.weights[j], 0);
-          return l === 0 ? errProp : errProp * out * (1 - out);
+          return prevAct ? errProp * prevAct.dfn(out) : errProp;
         });
         layer.neurons.forEach((n, k) => {
           n.weights.forEach((_, j) => {
@@ -356,8 +445,11 @@ var NetworkLSTM = class {
       const layer = this.denseLayers[l];
       const grad = denseGrads[l];
       layer.neurons.forEach((n, k) => {
-        n.weights = n.weights.map((w, j) => w + lr / T * grad.dW[k][j]);
-        n.bias += lr / T * grad.db[k];
+        n._update(
+          grad.dW[k].map((g) => g / T),
+          grad.db[k] / T,
+          lr
+        );
       });
     }
     this.lstm.backprop(dh_seq, lr);
@@ -382,12 +474,51 @@ var NetworkLSTM = class {
     });
   }
 };
+
+// src/losses.ts
+function mse(predicted, actual) {
+  return predicted.reduce((sum, p, i) => sum + (actual[i] - p) ** 2, 0) / predicted.length;
+}
+function crossEntropy(predicted, actual) {
+  const eps = 1e-15;
+  return -predicted.reduce((sum, p, i) => {
+    const clipped = Math.max(eps, Math.min(1 - eps, p));
+    return sum + actual[i] * Math.log(clipped) + (1 - actual[i]) * Math.log(1 - clipped);
+  }, 0) / predicted.length;
+}
+function mseDelta(predicted, actual) {
+  return actual - predicted;
+}
+function crossEntropyDelta(predicted, actual) {
+  return actual - predicted;
+}
+function crossEntropyDeltaRaw(predicted, actual) {
+  const eps = 1e-15;
+  const p = Math.max(eps, Math.min(1 - eps, predicted));
+  return actual / p - (1 - actual) / (1 - p);
+}
 export {
+  Adam,
   LSTMLayer,
   Layer,
+  Momentum,
   Network,
   NetworkLSTM,
   NetworkN,
   Neuron,
-  NeuronN
+  NeuronN,
+  SGD,
+  crossEntropy,
+  crossEntropyDelta,
+  crossEntropyDeltaRaw,
+  elu,
+  leakyRelu,
+  linear,
+  makeElu,
+  makeLeakyRelu,
+  mse,
+  mseDelta,
+  relu,
+  sigmoid2 as sigmoid,
+  tanh
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dniskav/neuron",
-  "version": "0.1.4",
+  "version": "0.1.6",
   "description": "Minimal neural network from scratch — neuron, layer, network, backpropagation. No dependencies.",
   "main": "dist/index.js",
   "module": "dist/index.mjs",