adjointly 0.0.0-stage → 0.1.0

package/docs/pitfalls.md

@@ -0,0 +1,133 @@
+# Pitfalls
+
+Each of these gives a wrong derivative without an error. Most are invisible to the type checker.
+When a gradient looks wrong, run once with `new Tape({ debug: true })` (or `new Forward({ debug:
+true })`): it checks every operand and throws at the first misuse it can see.
+
+## Telling variables from numbers with `typeof`
+
+**Never tell a variable from a plain number by `typeof`, or by its value.** A `Var` (or `Dual`) is a
+number at run time, so `typeof x === 'number'` is true for every variable. Code that uses such a test
+to turn "numbers" into constants turns every variable into a constant too, and every gradient comes
+out exactly 0, silently:
+
+```ts
+import { type Arithmetic, gradient, Tape, type Var } from 'adjointly';
+
+// Wrong: meant to lift plain numbers into constants and pass variables through.
+function lift<T>(A: Arithmetic<T>, v: T | number): T {
+  return typeof v === 'number' ? A.constant(v) : v;
+}
+gradient((A, [x]) => A.mul(lift(A, x), lift(A, 2)), [3]).gradient; //=> [0]
+
+// Debug mode refuses a variable where a number belongs.
+const t = new Tape({ debug: true });
+const x: Var = t.input(3);
+let message = '';
+try {
+  lift(t, x);
+} catch (e) {
+  message = (e as Error).message.split(',')[0]!;
+}
+message; //=> 'Tape.constant: argument 1 is a variable'
+```
+
+Keep variables and numbers in separate places (separate parameters, separate arrays, a field that
+says which), and pass numbers to the plain-number operations, which take them as such:
+
+```ts
+import { gradient } from 'adjointly';
+
+gradient((A, [x]) => A.mulc(x, 2), [3]).gradient; //=> [2]
+```
+
+## Arithmetic operators on a `Var`
+
+`Var` and `Dual` are numbers at run time, and TypeScript allows `+`, `*`, `Math.exp` on number
+subtypes. The result is arithmetic on _indices_: meaningless, and unchecked.
+
+```ts
+import { Tape } from 'adjointly';
+
+const t = new Tape();
+const x = t.input(10);
+const y = t.input(20);
+// Wrong: `x + y` adds the nodes' indices (1 and 2: slot 0 is reserved). Right:
+t.value(t.add(x, y)); //=> 30
+```
+
+Route every operation through the arithmetic object, `Math.*` included.
+
+## Reading a value out and feeding it back
+
+`A.value(x)` is a plain number. Anything computed from it and fed back through `A.constant` is a
+constant: the derivative through it is lost.
+
+```ts
+import { gradient } from 'adjointly';
+
+// Intends x^2, differentiates as if the second x were a constant.
+gradient((A, [x]) => A.mul(x, A.constant(A.value(x))), [3]).gradient; //=> [3]
+gradient((A, [x]) => A.square(x), [3]).gradient; //=> [6]
+```
+
+Use `A.value` for branch conditions, logging, step sizes and results; never for a quantity a
+derivative should flow through.
+
+## Branches at their boundary, and kinks written by hand
+
+The derivative is that of the branch taken. A hand-written `abs` is `+1` at zero; `A.abs` is `0`
+by convention. Prefer the primitives for kinks.
+
+```ts
+import { gradient } from 'adjointly';
+
+gradient((A, [x]) => (A.value(x) >= 0 ? x : A.neg(x)), [0]).gradient; //=> [1]
+gradient((A, [x]) => A.abs(x), [0]).gradient; //=> [0]
+```
+
+A piecewise function that is discontinuous at a boundary has no derivative there, and the one you
+get describes only the side taken. Finite differences across the boundary are meaningless too, so
+test gradients away from boundaries.
+
+## Unstable compositions
+
+```ts
+import { plain } from 'adjointly';
+
+// Built by hand from exp and log, softplus overflows; the primitive does not.
+plain.log(plain.add(plain.constant(1), plain.exp(800))); //=> Infinity
+plain.softplus(800); //=> 800
+```
+
+Use `sigmoid`, `softplus`, `log1p`, `expm1`, `logsumexp` and `softmax` rather than assembling them.
+
+## Leaving a function's domain
+
+`log`, `sqrt`, `div` by zero and `pow` with a non-positive base produce `NaN` or `Infinity` in value
+and derivative. A parameter that must stay positive should be reparametrised (`softplus(raw)`)
+rather than clamped after the fact.
+
+## Using a `Var` after `clear()` or `rewind()`
+
+A `Var` is an index. After `clear()`, or `rewind(m)` to before it was made, it names nothing, or
+whichever newer node took its slot. Nothing checks. Keep only `Var`s made before a `mark()` you
+rewind to.
+
+## A `Var` from another tape
+
+The brand cannot tell two tapes apart. Passing a `Var` from tape A to tape B reads B's node with the
+same index. Keep one tape per computation, and do not mix them.
+
+## `noUncheckedIndexedAccess` and generic accumulators
+
+Under that compiler option `xs[i]` is `T | undefined`. For a fixed number of inputs, type them as a
+tuple (`[x, y]: readonly [T, T]`, or `ScalarFunction<[number, number]>`) and they are plain `T`s;
+`tape.inputs([1, 2])` likewise returns a tuple of `Var`s. For a list of any length, index with `!`;
+and note that `let s = xs[0]!` infers `NonNullable<T>`, which a later `s = A.add(s, ...)` (returning
+`T`) cannot be assigned to. Annotate it: `let s: T = xs[0]!`.
+
+## Adjoints after the output
+
+`backward(output)` only zeroes and sweeps nodes up to `output`. Adjoints of nodes recorded after it
+are left over from earlier sweeps and mean nothing.

package/docs/recipes.md

@@ -0,0 +1,151 @@
+# Recipes
+
+## Gradient descent in an inner loop
+
+Keep one tape for the whole loop and clear it per step; pass a buffer for the gradient. After the
+first step the tape has grown to the size of the computation and nothing more is allocated.
+
+```ts
+import { type Arithmetic, Tape } from 'adjointly';
+
+// Least squares: fit y = a x + b to three points.
+const xs = [0, 1, 2];
+const ys = [1, 3, 5];
+function loss<T>(A: Arithmetic<T>, a: T, b: T): T {
+  let s = A.constant(0);
+  for (let i = 0; i < xs.length; i++) {
+    const r = A.subc(A.add(A.mulc(a, xs[i]!), b), ys[i]!);
+    s = A.add(s, A.square(r));
+  }
+  return s;
+}
+
+const tape = new Tape();
+const grad = new Float64Array(2);
+let a = 0;
+let b = 0;
+for (let step = 0; step < 2000; step++) {
+  tape.clear();
+  const av = tape.input(a);
+  const bv = tape.input(b);
+  tape.gradient(loss(tape, av, bv), [av, bv], grad);
+  a -= 0.05 * grad[0]!;
+  b -= 0.05 * grad[1]!;
+}
+[a, b]; //=> ~[2, 1]
+```
+
+## A shared prefix with `mark` and `rewind`
+
+When many evaluations share their first part (parameters, features computed from them), record it
+once, mark the tape, and rewind to the mark for each evaluation. Vars recorded before the mark stay
+valid.
+
+```ts
+import { Tape } from 'adjointly';
+
+const t = new Tape();
+const w = t.input(2);
+const w2 = t.square(w); // shared: recorded once
+const m = t.mark();
+let total = 0;
+for (const x of [1, 2, 3]) {
+  t.rewind(m);
+  // d/dw of w^2 * x = 2 w x
+  t.backward(t.mulc(w2, x));
+  total += t.adjoint(w);
+}
+total; //=> 2 * 2 * (1 + 2 + 3)
+```
+
+## Several outputs: one sweep each, or one weighted sweep
+
+```ts
+import { Tape } from 'adjointly';
+
+const t = new Tape();
+const [x, y] = t.inputs([1, 2]);
+const f = t.mul(x, y);
+const g = t.add(x, t.square(y));
+
+t.backward(f);
+t.adjoints([x, y]); //=> [2, 1]
+t.backward(g);
+t.adjoints([x, y]); //=> [1, 4]
+t.vjp([f, g], [10, 1]); // gradient of 10 f + g
+t.adjoints([x, y]); //=> [21, 14]
+```
+
+## A full Jacobian
+
+Reverse mode gives a row per sweep; forward mode a column per pass. Use whichever needs fewer.
+
+```ts
+import { type Arithmetic, Forward, Tape } from 'adjointly';
+
+// F(x, y) = (x y, x + y, sin x)
+function F<T>(A: Arithmetic<T>, x: T, y: T): T[] {
+  return [A.mul(x, y), A.add(x, y), A.sin(x)];
+}
+
+// Rows: one recording, one sweep per output.
+const t = new Tape();
+const x = t.input(2);
+const y = t.input(3);
+const outs = F(t, x, y);
+const rows = outs.map((o) => Array.from(t.gradient(o, [x, y])));
+rows; //=> [[3, 2], [1, 1], [cos(2), 0]]
+
+// Columns: one forward pass per input.
+const cols = [0, 1].map((j) => {
+  const fw = new Forward();
+  const ys = F(fw, fw.input(2, j === 0 ? 1 : 0), fw.input(3, j === 1 ? 1 : 0));
+  return ys.map((v) => fw.tangent(v));
+});
+cols; //=> [[3, 1, cos(2)], [2, 1, 0]]
+```
+
+## Constrained parameters
+
+Optimise an unconstrained number and map it into the domain with a smooth primitive: `softplus`
+for positive, `sigmoid` for a probability, `exp` for a scale.
+
+```ts
+import { type Arithmetic, gradient } from 'adjointly';
+
+// A rate that must stay positive, parametrised by an unconstrained raw value.
+function negLogLikelihood<T>(A: Arithmetic<T>, [raw]: readonly [T]): T {
+  const rate = A.softplus(raw);
+  const waits = [0.5, 1.5, 1.0];
+  // exponential distribution: -sum(log(rate) - rate * w)
+  let s = A.constant(0);
+  for (const w of waits) s = A.sub(s, A.sub(A.log(rate), A.mulc(rate, w)));
+  return s;
+}
+// At the optimum rate = 1 / mean(waits) = 1, i.e. raw = log(e - 1), the gradient vanishes.
+gradient(negLogLikelihood, [Math.log(Math.E - 1)]).gradient; //=> ~[0]
+```
+
+## Test your own gradient
+
+```ts
+import { checkGradient, logsumexp, type ScalarFunction } from 'adjointly';
+
+const f: ScalarFunction = (A, xs) =>
+  A.add(logsumexp(A, xs), A.mul(A.tanh(xs[0]!), A.sigmoid(xs[1]!)));
+
+for (const at of [
+  [0.1, -0.7, 2],
+  [-3, 0.4, 0.5],
+]) {
+  const r = checkGradient(f, at);
+  if (!r.ok) throw new Error(`component ${String(r.worst.index)} is off`);
+}
+checkGradient(f, [1, 2, 3]).ok; //=> true
+```
+
+## The worked example
+
+[examples/production-line/](https://github.com/feldgendler/adjointly/tree/main/examples/production-line) (in the repository, not the npm package) puts these together: a
+minute-by-minute simulation written once, its gradient with respect to 1 440 decisions from one
+tape pass, first-order estimates ranking discrete moves, and forward mode for a parameter.

package/llms.txt

@@ -0,0 +1,36 @@
+# adjointly
+
+> Dependency-free automatic differentiation for scalar float64 programs in TypeScript. Write a function once against `Arithmetic<T>`; evaluate it with `plain`, get its gradient by reverse mode with `Tape`, or a directional derivative by forward mode with `Forward`. ESM only, Node 22+.
+
+Rules for code that this library differentiates:
+
+- Write the function generic in `T`: `function f<T>(A: Arithmetic<T>, xs: readonly T[]): T`.
+- Never tell a variable from a number with `typeof x === 'number'` or by value: a `Var`/`Dual` is a number at run time and passes, so every variable would become a constant and every gradient 0. Keep them in separate places. To find misuse, run once with `new Tape({ debug: true })`, which throws at any operand that is not a live variable of that tape, and at a variable where a number belongs.
+- Every operation on a differentiable value goes through `A`: `A.add`, `A.mul`, `A.exp`, never `+`, `*` or `Math.*`. `Var`/`Dual` are numbers at run time, so the mistake type-checks and is silent.
+- Numbers enter through `A.constant(c)`, or better as the number operand of `addc`/`subc`/`csub`/`mulc`/`divc`/`cdiv`/`powc` (one tape node instead of two), and leave through `A.value(x)`. A number taken out with `A.value` and fed back with `A.constant` carries no derivative.
+- Branching on `A.value(x)` is fine; the derivative is that of the branch taken.
+- Use the stable primitives `sigmoid`, `softplus`, `log1p`, `expm1`, `square`, and the composites `logsumexp`, `softmax`.
+- In loops, reuse one `Tape`: `clear()` (or `rewind(mark)`) per evaluation, and pass an output buffer to `tape.gradient`.
+- Verify a new function with `checkGradient(f, at)` at a few points away from kinks.
+
+```ts
+import { type Arithmetic, checkGradient, gradient, jvp } from 'adjointly';
+
+function f<T>(A: Arithmetic<T>, [x, y]: readonly [T, T]): T {
+  return A.add(A.mul(x, y), A.softplus(x));
+}
+gradient(f, [1, 2]); //=> { value: 2 + log(1 + E), gradient: [2 + 1 / (1 + exp(-1)), 1] }
+jvp(f, [1, 2], [0, 1]); //=> { value: 2 + log(1 + E), derivative: 1 }
+checkGradient(f, [1, 2]).ok; //=> true
+```
+
+Primitives (all on `Arithmetic`): neg, add, sub, mul, div, recip, square, sqrt, exp, expm1, log, log1p, pow, abs, min, max, sin, cos, tanh, sigmoid, softplus. With a plain-number operand c, in one tape node instead of constant(c) plus the operation: addc(a, c), subc(a, c), csub(c, a), mulc(a, c), divc(a, c), cdiv(c, a), powc(a, c). Subgradients: abs'(0) = 0; max/min ties go to the first argument; pow's exponent derivative is 0 for a base <= 0, its base derivative 0 for exponent 0; sqrt'(0) = sqrt'(-0) = Infinity; max/min with a NaN argument have NaN partials. Zero rule: a chain-rule term with an exactly zero partial, adjoint or tangent is 0, even against Infinity or NaN.
+
+## Docs
+
+- [README](README.md): overview, quick start, API overview, conventions, design notes, rules for agents
+- [docs/concepts.md](docs/concepts.md): the tape, adjoints, forward vs reverse mode, control flow
+- [docs/api.md](docs/api.md): every export by topic, with its contract
+- [docs/recipes.md](docs/recipes.md): inner loops, mark/rewind, several outputs, Jacobians, constrained parameters, testing gradients
+- [docs/pitfalls.md](docs/pitfalls.md): mistakes that silently give wrong derivatives
+- [examples/production-line](https://github.com/feldgendler/adjointly/tree/main/examples/production-line): a worked optimisation example (in the repository, not the package)

package/package.json

@@ -1,6 +1,93 @@
 {
   "name": "adjointly",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
-}
+  "version": "0.1.0",
+  "description": "Dependency-free automatic differentiation for scalar float64 programs in TypeScript: reverse mode on a typed-array tape and forward mode, from one function written once.",
+  "keywords": [
+    "autodiff",
+    "automatic-differentiation",
+    "gradient",
+    "reverse-mode",
+    "forward-mode",
+    "backpropagation",
+    "optimization",
+    "typescript"
+  ],
+  "license": "MIT",
+  "author": "Alexey Feldgendler",
+  "homepage": "https://github.com/feldgendler/adjointly#readme",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/feldgendler/adjointly.git"
+  },
+  "bugs": {
+    "url": "https://github.com/feldgendler/adjointly/issues"
+  },
+  "type": "module",
+  "sideEffects": false,
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
+    },
+    "./package.json": "./package.json"
+  },
+  "types": "./dist/index.d.ts",
+  "files": [
+    "dist",
+    "docs",
+    "llms.txt",
+    "README.md",
+    "CHANGELOG.md"
+  ],
+  "engines": {
+    "node": ">=22"
+  },
+  "packageManager": "npm@11.19.1",
+  "publishConfig": {
+    "access": "public",
+    "provenance": true
+  },
+  "scripts": {
+    "build": "tsdown",
+    "typecheck": "tsc -p tsconfig.json",
+    "lint": "eslint .",
+    "format": "prettier --write .",
+    "format:check": "prettier --check .",
+    "test": "vitest run",
+    "test:coverage": "vitest run --coverage",
+    "api:check": "api-extractor run",
+    "api:update": "npm run build && api-extractor run --local",
+    "publint": "publint --strict",
+    "attw": "attw --pack . --profile esm-only",
+    "check": "npm run format:check && npm run lint && npm run typecheck && npm run test:coverage && npm run build && npm run publint && npm run attw && npm run api:check",
+    "mutate": "stryker run",
+    "bench": "node bench/measure.ts",
+    "changeset": "changeset",
+    "release": "changeset publish",
+    "prepublishOnly": "npm run check",
+    "example": "node examples/production-line/run.ts",
+    "version-packages": "changeset version",
+    "bench:ab": "node bench/ab.ts"
+  },
+  "devDependencies": {
+    "@arethetypeswrong/cli": "^0.18.5",
+    "@changesets/cli": "^3.0.3",
+    "@eslint/js": "^10.0.1",
+    "@microsoft/api-extractor": "^7.60.0",
+    "@stryker-mutator/core": "^10.1.1",
+    "@stryker-mutator/typescript-checker": "^10.1.1",
+    "@stryker-mutator/vitest-runner": "^10.1.1",
+    "@types/node": "^26.6.5",
+    "@vitest/coverage-v8": "^5.0.5",
+    "eslint": "^10.13.0",
+    "eslint-plugin-tsdoc": "^0.5.4",
+    "fast-check": "^4.10.2",
+    "prettier": "^3.9.10",
+    "publint": "^0.3.25",
+    "tinybench": "^6.2.2",
+    "tsdown": "^0.23.3",
+    "typescript": "~6.0.3",
+    "typescript-eslint": "^8.72.1",
+    "vitest": "^5.0.5"
+  }
+}