@danielsimonjr/mathts-autograd 0.3.13 → 0.3.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -21
- package/dist/dual-tensor.d.ts +132 -0
- package/dist/dual-tensor.d.ts.map +1 -0
- package/dist/forward-grad.d.ts +45 -0
- package/dist/forward-grad.d.ts.map +1 -0
- package/dist/grad.d.ts +58 -0
- package/dist/grad.d.ts.map +1 -0
- package/dist/index.d.ts +14 -653
- package/dist/index.d.ts.map +1 -0
- package/dist/reverse-grad.d.ts +25 -0
- package/dist/reverse-grad.d.ts.map +1 -0
- package/dist/tape.d.ts +444 -0
- package/dist/tape.d.ts.map +1 -0
- package/package.json +11 -11
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Reverse-mode AD via tape. Returns the value and the vector-Jacobian product
|
|
3
|
+
* (gradient = ∂(cotangent · value) / ∂x). gradient.shape = x.shape.
|
|
4
|
+
*
|
|
5
|
+
* Default cotangent: ones-like(value) — required for scalar outputs, useful
|
|
6
|
+
* for VJP defaults. For non-scalar value, cotangent's shape must match value.shape.
|
|
7
|
+
*/
|
|
8
|
+
import { Tensor } from '@danielsimonjr/mathts-tensor';
|
|
9
|
+
/**
|
|
10
|
+
* Computes the value of `fn` at `x` and the vector-Jacobian product with reverse-mode AD.
|
|
11
|
+
*
|
|
12
|
+
* `fn` receives a TapedTensor in place of `x`. It must return a result of TapedTensor operations on that argument.
|
|
13
|
+
*
|
|
14
|
+
* @param fn - Function to differentiate.
|
|
15
|
+
* @param x - Point at which to evaluate `fn`.
|
|
16
|
+
* @param cotangent - Seed for the backward pass. Its shape must match the value shape. The default is all ones.
|
|
17
|
+
* @returns The value of `fn(x)`, and the gradient of `cotangent · value` with respect to `x`, with the shape of `x`.
|
|
18
|
+
* @throws Error if `fn` does not return a TapedTensor.
|
|
19
|
+
* @throws Error if the `cotangent` shape does not match the value shape.
|
|
20
|
+
*/
|
|
21
|
+
export declare function reverseGrad(fn: (x: Tensor) => Tensor, x: Tensor, cotangent?: Tensor): {
|
|
22
|
+
value: Tensor;
|
|
23
|
+
gradient: Tensor;
|
|
24
|
+
};
|
|
25
|
+
//# sourceMappingURL=reverse-grad.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"reverse-grad.d.ts","sourceRoot":"","sources":["../src/reverse-grad.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AACH,OAAO,EAAE,MAAM,EAAE,MAAM,8BAA8B,CAAC;AAGtD;;;;;;;;;;;GAWG;AACH,wBAAgB,WAAW,CACzB,EAAE,EAAE,CAAC,CAAC,EAAE,MAAM,KAAK,MAAM,EACzB,CAAC,EAAE,MAAM,EACT,SAAS,CAAC,EAAE,MAAM,GACjB;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,CAyCrC"}
|
package/dist/tape.d.ts
ADDED
|
@@ -0,0 +1,444 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Tape — records the sequence of ops during a forward pass so we can
|
|
3
|
+
* replay them in reverse to compute the vector-Jacobian product.
|
|
4
|
+
*
|
|
5
|
+
* Each TapedTensor wraps a primal Tensor + a tape-node id. Ops on
|
|
6
|
+
* TapedTensors record their backward closures into a shared Tape.
|
|
7
|
+
* After the forward pass, calling Tape.backward(outputGrad) walks the
|
|
8
|
+
* tape in reverse, accumulating gradients into each input slot.
|
|
9
|
+
*
|
|
10
|
+
* v0.1.0 supports the same ops as DualTensor: add, sub, mul, scale.
|
|
11
|
+
* v0.2.0 adds: contract (named-index), matmul (batched rank-N).
|
|
12
|
+
*/
|
|
13
|
+
import { Tensor, Index } from '@danielsimonjr/mathts-tensor';
|
|
14
|
+
type BackwardFn = (outputGrad: Float64Array) => void;
|
|
15
|
+
/**
|
|
16
|
+
* Records the operations of a forward pass so that `backward` can replay them in reverse.
|
|
17
|
+
*
|
|
18
|
+
* `allocate` gives inputs negative ids. `record` gives operations non-negative ids.
|
|
19
|
+
* Each id owns a gradient slot. `backward` writes `outputGrad` into the output slot: it replaces the
|
|
20
|
+
* value in that slot. `backward` adds gradients into the input and intermediate slots.
|
|
21
|
+
* `backward` does not clear the input and intermediate slots. A second call adds to the gradients of the
|
|
22
|
+
* first call in those slots.
|
|
23
|
+
*/
|
|
24
|
+
export declare class Tape {
|
|
25
|
+
private nodes;
|
|
26
|
+
private inputGradSlots;
|
|
27
|
+
private nextOpId;
|
|
28
|
+
private nextInputId;
|
|
29
|
+
/** Allocate a fresh id for an input or intermediate. */
|
|
30
|
+
allocate(size: number): {
|
|
31
|
+
id: number;
|
|
32
|
+
gradSlot: Float64Array;
|
|
33
|
+
};
|
|
34
|
+
record(inputIds: ReadonlyArray<number>, outputSize: number, backward: BackwardFn): {
|
|
35
|
+
id: number;
|
|
36
|
+
gradSlot: Float64Array;
|
|
37
|
+
};
|
|
38
|
+
/** Seed the final output's gradient slot, then replay in reverse. */
|
|
39
|
+
backward(outputId: number, outputGrad: Float64Array): void;
|
|
40
|
+
getInputGrad(id: number): Float64Array | undefined;
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* Wraps tensor values with a tape id, so that each operation records its backward step on the tape.
|
|
44
|
+
*
|
|
45
|
+
* The `data` getter returns the primal values, so tensor ops that read `.data` accept a TapedTensor.
|
|
46
|
+
* The `id` names the gradient slot of this tensor on the tape.
|
|
47
|
+
* The optional per-axis Index labels enable `contract`.
|
|
48
|
+
*/
|
|
49
|
+
export declare class TapedTensor {
|
|
50
|
+
readonly shape: ReadonlyArray<number>;
|
|
51
|
+
readonly primal: Float64Array;
|
|
52
|
+
readonly tape: Tape;
|
|
53
|
+
readonly id: number;
|
|
54
|
+
/**
|
|
55
|
+
* Optional per-axis Index labels. When set, enables `contract`.
|
|
56
|
+
* Must have the same length as `shape` when present.
|
|
57
|
+
*/
|
|
58
|
+
readonly axisLabels?: ReadonlyArray<Index>;
|
|
59
|
+
constructor(shape: ReadonlyArray<number>, primal: Float64Array, tape: Tape, id: number, axisLabels?: ReadonlyArray<Index>);
|
|
60
|
+
/**
|
|
61
|
+
* S5 fix: existing engine ops (e.g. lower, pderiv, contract) reach into
|
|
62
|
+
* `.data`. The getter returns the primal so those ops still work when a
|
|
63
|
+
* TapedTensor flows through them as a structurally-compatible Tensor.
|
|
64
|
+
* (AD-aware ops branch on `'tape' in arg` before reaching here.)
|
|
65
|
+
*/
|
|
66
|
+
get data(): Float64Array;
|
|
67
|
+
static fromTensorAsInput(t: Tensor, tape: Tape): TapedTensor;
|
|
68
|
+
toPrimalTensor(): Tensor;
|
|
69
|
+
/**
|
|
70
|
+
* Reconstruct the Tensor primal with the given axisLabels (used internally
|
|
71
|
+
* when the primal was computed from an op that produces labelled output).
|
|
72
|
+
*/
|
|
73
|
+
private toPrimalTensorWith;
|
|
74
|
+
add(other: TapedTensor): TapedTensor;
|
|
75
|
+
sub(other: TapedTensor): TapedTensor;
|
|
76
|
+
/**
|
|
77
|
+
* Elementwise division: this / other.
|
|
78
|
+
*
|
|
79
|
+
* Adjoints (quotient rule):
|
|
80
|
+
* dA = dY / b
|
|
81
|
+
* dB = −dY · a / b²
|
|
82
|
+
*
|
|
83
|
+
* The alias case (a.divide(a)) is handled explicitly: the combined gradient
|
|
84
|
+
* is dA + dB = dY/a − dY·a/a² = dY/a − dY/a = 0. This is correct since
|
|
85
|
+
* a/a = 1 everywhere and d(1)/da = 0.
|
|
86
|
+
*/
|
|
87
|
+
divide(other: TapedTensor): TapedTensor;
|
|
88
|
+
mul(other: TapedTensor): TapedTensor;
|
|
89
|
+
scale(k: number): TapedTensor;
|
|
90
|
+
/**
|
|
91
|
+
* Reverse-mode AD over `Tensor.contract`.
|
|
92
|
+
*
|
|
93
|
+
* Both operands must carry `axisLabels`; the resulting TapedTensor inherits
|
|
94
|
+
* the contracted-output axisLabels (non-shared axis concatenation).
|
|
95
|
+
*
|
|
96
|
+
* Adjoint derivation (T-notation):
|
|
97
|
+
* Let Y = A.contract(B) (contraction over shared indices S).
|
|
98
|
+
* dA = dY.contract(B') where B' = B with its free axes re-labelled to
|
|
99
|
+
* match the shared positions in A. Equivalently: for each element of A,
|
|
100
|
+
* dA[...a_free, ...s] = Σ_{...b_free} dY[...a_free, ...b_free] · B[...s, ...b_free]
|
|
101
|
+
* dB = A'.contract(dY) symmetrically.
|
|
102
|
+
*
|
|
103
|
+
* Implementation: because `Tensor.contract` matches by Index identity, we
|
|
104
|
+
* build the backward contraction by re-labelling the free axes of dY with
|
|
105
|
+
* the Index objects from the other operand — so `Tensor.contract` automatically
|
|
106
|
+
* finds the right shared axes.
|
|
107
|
+
*/
|
|
108
|
+
contract(other: TapedTensor): TapedTensor;
|
|
109
|
+
/**
|
|
110
|
+
* Reverse-mode AD over batched matmul.
|
|
111
|
+
*
|
|
112
|
+
* Accepts operands of rank ≥ 2. Convention: trailing 2 axes are the matrix
|
|
113
|
+
* dims (rows × cols); all leading axes are batch dims (must be broadcast-
|
|
114
|
+
* compatible). For rank-2 inputs this is classical matrix multiplication.
|
|
115
|
+
*
|
|
116
|
+
* Adjoint derivation (classical matrix calc, extended to batched):
|
|
117
|
+
* Y = A @ B (A: ...×m×k, B: ...×k×n → Y: ...×m×n)
|
|
118
|
+
* dA = dY @ Bᵀ (dA: ...×m×k)
|
|
119
|
+
* dB = Aᵀ @ dY (dB: ...×k×n)
|
|
120
|
+
*
|
|
121
|
+
* Implemented via `Tensor.einsum` with a dynamically-built spec:
|
|
122
|
+
* forward: '...ik,...kj->...ij'
|
|
123
|
+
* dA: '...ij,...kj->...ik' (contract dY with B on j)
|
|
124
|
+
* dB: '...ki,...kj->...ij' (contract A on k with dY on k)
|
|
125
|
+
*/
|
|
126
|
+
matmul(other: TapedTensor): TapedTensor;
|
|
127
|
+
private checkSameShape;
|
|
128
|
+
/**
|
|
129
|
+
* Sum elements along the given axis/axes (or all axes if omitted).
|
|
130
|
+
*
|
|
131
|
+
* Adjoint: dX[...] = dY[reduced(idx)] broadcast back to input shape.
|
|
132
|
+
* Each input element receives the output-gradient entry from its reduced
|
|
133
|
+
* counterpart (the non-reduced coordinates select the dY element; the
|
|
134
|
+
* reduced coordinates are collapsed to 0 in the keepDims=false case).
|
|
135
|
+
*/
|
|
136
|
+
sum(axis?: number | ReadonlyArray<number>, opts?: {
|
|
137
|
+
keepDims?: boolean;
|
|
138
|
+
}): TapedTensor;
|
|
139
|
+
/**
|
|
140
|
+
* Arithmetic mean along the given axis/axes (or all axes if omitted).
|
|
141
|
+
*
|
|
142
|
+
* Adjoint: dX[...] = dY[reduced(idx)] / N, broadcast back to input shape.
|
|
143
|
+
* N = product of reduced-axis dimensions.
|
|
144
|
+
*/
|
|
145
|
+
mean(axis?: number | ReadonlyArray<number>, opts?: {
|
|
146
|
+
keepDims?: boolean;
|
|
147
|
+
}): TapedTensor;
|
|
148
|
+
/**
|
|
149
|
+
* Product of elements along the given axis/axes (or all axes if omitted).
|
|
150
|
+
*
|
|
151
|
+
* Adjoint: dX_i = dY * (prod_over_axes(x) / x_i) per element.
|
|
152
|
+
*
|
|
153
|
+
* Zero-element corners:
|
|
154
|
+
* - Exactly one x_i = 0 in the reduced group: d/dx_i = product of all others
|
|
155
|
+
* (which is the full product / x_i evaluated via alternate product),
|
|
156
|
+
* and d/dx_j = 0 for all j ≠ i where x_j ≠ 0.
|
|
157
|
+
* - Two or more zeros in the reduced group: gradient is 0 everywhere for that
|
|
158
|
+
* group (because changing any single zero cannot change a product that is
|
|
159
|
+
* zero due to another zero).
|
|
160
|
+
*
|
|
161
|
+
* Implementation: uses prefix/suffix products to handle zeros robustly.
|
|
162
|
+
*/
|
|
163
|
+
prod(axis?: number | ReadonlyArray<number>, opts?: {
|
|
164
|
+
keepDims?: boolean;
|
|
165
|
+
}): TapedTensor;
|
|
166
|
+
/**
|
|
167
|
+
* Maximum value along the given axis/axes (or all axes if omitted).
|
|
168
|
+
*
|
|
169
|
+
* Adjoint: dY is scattered to the argmax position(s).
|
|
170
|
+
* Tie-breaking: "first-wins" — the gradient flows to the first (smallest
|
|
171
|
+
* flat-index) element among those that attain the maximum.
|
|
172
|
+
*/
|
|
173
|
+
max(axis?: number | ReadonlyArray<number>, opts?: {
|
|
174
|
+
keepDims?: boolean;
|
|
175
|
+
}): TapedTensor;
|
|
176
|
+
/**
|
|
177
|
+
* Minimum value along the given axis/axes (or all axes if omitted).
|
|
178
|
+
*
|
|
179
|
+
* Adjoint: dY is scattered to the argmin position(s).
|
|
180
|
+
* Tie-breaking: "first-wins" — gradient flows to the first (smallest
|
|
181
|
+
* flat-index) element that attains the minimum.
|
|
182
|
+
*/
|
|
183
|
+
min(axis?: number | ReadonlyArray<number>, opts?: {
|
|
184
|
+
keepDims?: boolean;
|
|
185
|
+
}): TapedTensor;
|
|
186
|
+
/**
|
|
187
|
+
* p-norm of the tensor.
|
|
188
|
+
*
|
|
189
|
+
* Supported p values: 1, 2, 'fro', 'inf'. Default p = 2.
|
|
190
|
+
* When `opts.axis` is given, reduces along that axis; otherwise reduces all axes.
|
|
191
|
+
*
|
|
192
|
+
* Adjoints:
|
|
193
|
+
* - p=2 / p='fro': dX = dY · x / ‖x‖₂ (Frobenius is the 2-norm of the flattened tensor)
|
|
194
|
+
* - p=1: dX = dY · sign(x) (subgradient = 0 at exact zero)
|
|
195
|
+
* - p='inf': dX scattered to the element(s) of max absolute value;
|
|
196
|
+
* tie-breaking: first-wins. Sign of the scattered gradient
|
|
197
|
+
* matches sign(x_max).
|
|
198
|
+
*/
|
|
199
|
+
norm(opts?: {
|
|
200
|
+
p?: 1 | 2 | 'fro' | 'inf';
|
|
201
|
+
axis?: number;
|
|
202
|
+
keepDims?: boolean;
|
|
203
|
+
}): TapedTensor;
|
|
204
|
+
/**
|
|
205
|
+
* Elementwise natural logarithm.
|
|
206
|
+
*
|
|
207
|
+
* Adjoint: dX = dY / x
|
|
208
|
+
*/
|
|
209
|
+
log(): TapedTensor;
|
|
210
|
+
/**
|
|
211
|
+
* Elementwise exponential.
|
|
212
|
+
*
|
|
213
|
+
* Adjoint: dX = dY · y where y = exp(x). Primal output is cached.
|
|
214
|
+
*/
|
|
215
|
+
exp(): TapedTensor;
|
|
216
|
+
/**
|
|
217
|
+
* Elementwise sine.
|
|
218
|
+
*
|
|
219
|
+
* Adjoint: dX = dY · cos(x)
|
|
220
|
+
*/
|
|
221
|
+
sin(): TapedTensor;
|
|
222
|
+
/**
|
|
223
|
+
* Elementwise cosine.
|
|
224
|
+
*
|
|
225
|
+
* Adjoint: dX = −dY · sin(x)
|
|
226
|
+
*/
|
|
227
|
+
cos(): TapedTensor;
|
|
228
|
+
/**
|
|
229
|
+
* Elementwise tangent.
|
|
230
|
+
*
|
|
231
|
+
* Adjoint: dX = dY / cos²(x) (= dY · sec²(x))
|
|
232
|
+
*/
|
|
233
|
+
tan(): TapedTensor;
|
|
234
|
+
/**
|
|
235
|
+
* Elementwise square root.
|
|
236
|
+
*
|
|
237
|
+
* Adjoint: dX = dY / (2·y) where y = sqrt(x). Primal output is cached.
|
|
238
|
+
*/
|
|
239
|
+
sqrt(): TapedTensor;
|
|
240
|
+
/**
|
|
241
|
+
* Elementwise square (x²).
|
|
242
|
+
*
|
|
243
|
+
* Adjoint: dX = dY · 2x
|
|
244
|
+
*/
|
|
245
|
+
square(): TapedTensor;
|
|
246
|
+
/**
|
|
247
|
+
* Elementwise power: x^k.
|
|
248
|
+
*
|
|
249
|
+
* `k` may be a fixed `number` (constant exponent) or a `TapedTensor`
|
|
250
|
+
* (variable exponent, both inputs on the tape).
|
|
251
|
+
*
|
|
252
|
+
* Adjoints, for y = a^b:
|
|
253
|
+
* - fixed exponent k: dA = dY · k · a^(k−1)
|
|
254
|
+
* - variable exponent: dA = dY · b · a^(b−1), dB = dY · a^b · ln(a)
|
|
255
|
+
* - aliased a^a: dA = dY · a^a · (ln a + 1)
|
|
256
|
+
*
|
|
257
|
+
* The dB term needs ln(a); for a ≤ 0 it is mathematically undefined and
|
|
258
|
+
* yields NaN/−Inf, honestly signalling non-differentiability w.r.t. the
|
|
259
|
+
* exponent rather than masking it.
|
|
260
|
+
*/
|
|
261
|
+
pow(k: number | TapedTensor): TapedTensor;
|
|
262
|
+
/**
|
|
263
|
+
* Elementwise reciprocal: 1 / x.
|
|
264
|
+
*
|
|
265
|
+
* Adjoint: dX = −dY / x²
|
|
266
|
+
*/
|
|
267
|
+
reciprocal(): TapedTensor;
|
|
268
|
+
/**
|
|
269
|
+
* Elementwise absolute value: |x|.
|
|
270
|
+
*
|
|
271
|
+
* Adjoint: dX = dY · sign(x)
|
|
272
|
+
* Subgradient at exact zero is defined as 0 (rather than undefined).
|
|
273
|
+
*/
|
|
274
|
+
abs(): TapedTensor;
|
|
275
|
+
/**
|
|
276
|
+
* Build a reverse-mode node for an elementwise unary op.
|
|
277
|
+
* @param primalFn forward value y = f(x)
|
|
278
|
+
* @param localGrad local derivative dy/dx, given x and the computed y
|
|
279
|
+
*/
|
|
280
|
+
private _unaryElementwise;
|
|
281
|
+
/** Hyperbolic sine. Adjoint: dX = dY · cosh(x). */
|
|
282
|
+
sinh(): TapedTensor;
|
|
283
|
+
/** Hyperbolic cosine. Adjoint: dX = dY · sinh(x). */
|
|
284
|
+
cosh(): TapedTensor;
|
|
285
|
+
/** Hyperbolic tangent. Adjoint: dX = dY · (1 − tanh²(x)) = dY · (1 − y²). */
|
|
286
|
+
tanh(): TapedTensor;
|
|
287
|
+
/** Arcsine. Adjoint: dX = dY / √(1 − x²). */
|
|
288
|
+
asin(): TapedTensor;
|
|
289
|
+
/** Arccosine. Adjoint: dX = −dY / √(1 − x²). */
|
|
290
|
+
acos(): TapedTensor;
|
|
291
|
+
/** Arctangent. Adjoint: dX = dY / (1 + x²). */
|
|
292
|
+
atan(): TapedTensor;
|
|
293
|
+
/** Inverse hyperbolic sine. Adjoint: dX = dY / √(x² + 1). */
|
|
294
|
+
asinh(): TapedTensor;
|
|
295
|
+
/** Inverse hyperbolic cosine (x ≥ 1). Adjoint: dX = dY / √(x² − 1). */
|
|
296
|
+
acosh(): TapedTensor;
|
|
297
|
+
/** Inverse hyperbolic tangent (|x| < 1). Adjoint: dX = dY / (1 − x²). */
|
|
298
|
+
atanh(): TapedTensor;
|
|
299
|
+
/** Base-2 logarithm. Adjoint: dX = dY / (x · ln2). */
|
|
300
|
+
log2(): TapedTensor;
|
|
301
|
+
/** Base-10 logarithm. Adjoint: dX = dY / (x · ln10). */
|
|
302
|
+
log10(): TapedTensor;
|
|
303
|
+
/** log(1 + x) (accurate near 0). Adjoint: dX = dY / (1 + x). */
|
|
304
|
+
log1p(): TapedTensor;
|
|
305
|
+
/** exp(x) − 1 (accurate near 0). Adjoint: dX = dY · eˣ = dY · (y + 1). */
|
|
306
|
+
expm1(): TapedTensor;
|
|
307
|
+
/** Cube root. Adjoint: dX = dY / (3 · y²) where y = x^(1/3) (→∞ at x = 0). */
|
|
308
|
+
cbrt(): TapedTensor;
|
|
309
|
+
/** Sign of x (−1/0/+1). Adjoint: 0 almost everywhere. */
|
|
310
|
+
sign(): TapedTensor;
|
|
311
|
+
/**
|
|
312
|
+
* Two-argument arctangent: atan2(this, other), with this = y, other = x.
|
|
313
|
+
*
|
|
314
|
+
* Adjoints (a = this, b = other, d = a² + b²):
|
|
315
|
+
* dA = dY · b / d
|
|
316
|
+
* dB = −dY · a / d
|
|
317
|
+
*
|
|
318
|
+
* The aliased case atan2(a, a) is locally constant (±π/4 or ±3π/4), so its
|
|
319
|
+
* gradient is 0 — produced naturally by contributing nothing when aliased
|
|
320
|
+
* (dA + dB = dY · (b − a)/d = 0 at a = b).
|
|
321
|
+
*/
|
|
322
|
+
atan2(other: TapedTensor): TapedTensor;
|
|
323
|
+
/**
|
|
324
|
+
* Reverse-mode AD over `Tensor.tensordot`.
|
|
325
|
+
*
|
|
326
|
+
* `axes[i] = [a, b]` contracts axis `a` of `this` with axis `b` of `other`.
|
|
327
|
+
* The result shape is `this`'s non-contracted axes (in original order)
|
|
328
|
+
* followed by `other`'s non-contracted axes (in original order).
|
|
329
|
+
*
|
|
330
|
+
* Adjoint derivation (NumPy/PyTorch tensordot backward, see Townsend 2016 §6,
|
|
331
|
+
* and the canonical PyTorch implementation `TensorDotBackward0` in
|
|
332
|
+
* `torch/csrc/autograd/generated/Functions.cpp`):
|
|
333
|
+
*
|
|
334
|
+
* Z = tensordot(A, B, axes)
|
|
335
|
+
* dA = tensordot(dZ, B, [axes_of_dZ_corresponding_to_B's_free, B's_free])
|
|
336
|
+
* then permute back into A's original axis order.
|
|
337
|
+
* dB = tensordot(A, dZ, [A's_free, axes_of_dZ_corresponding_to_A's_free])
|
|
338
|
+
* then permute back into B's original axis order.
|
|
339
|
+
*
|
|
340
|
+
* The axis-permutation bookkeeping is the trickiest part: Tensor.tensordot
|
|
341
|
+
* produces output axes in the order [A's free axes (original A order),
|
|
342
|
+
* B's free axes (original B order)] — and after the backward contractions
|
|
343
|
+
* the survivors come out in B's original (resp. A's original) axis order
|
|
344
|
+
* for the contracted side, which then needs to be permuted into pair order
|
|
345
|
+
* (so axis k of the contracted block matches the kth pair) and finally
|
|
346
|
+
* scattered back into A's (resp. B's) full original axis order.
|
|
347
|
+
*
|
|
348
|
+
* For the rank-2 × rank-2 single-axis case (i.e. ordinary matmul A·B with
|
|
349
|
+
* axes = [[1, 0]]), this reduces to dA = dZ · Bᵀ, dB = Aᵀ · dZ — the same
|
|
350
|
+
* adjoint as `TapedTensor.matmul`.
|
|
351
|
+
*/
|
|
352
|
+
tensordot(other: TapedTensor, axes: ReadonlyArray<readonly [number, number]>): TapedTensor;
|
|
353
|
+
/**
|
|
354
|
+
* Reverse-mode AD over the full SVD of a rank-2 matrix.
|
|
355
|
+
*
|
|
356
|
+
* Forward: `A = U · diag(S) · Vt`, where for input shape [m, n], k = min(m, n):
|
|
357
|
+
* - U has shape [m, k]
|
|
358
|
+
* - S has shape [k]
|
|
359
|
+
* - Vt has shape [k, n] (Vt is V^T in the standard A = U Σ V^T convention,
|
|
360
|
+
* i.e. its rows are right-singular-vector components)
|
|
361
|
+
*
|
|
362
|
+
* Returned TapedTensors share a single backward closure. When backward()
|
|
363
|
+
* runs, it pulls dU, dS, dV from each output's gradient slot, assembles
|
|
364
|
+
* dA, and writes to the input's gradient slot.
|
|
365
|
+
*
|
|
366
|
+
* Adjoint (real, distinct nonzero singular values, m = n square case;
|
|
367
|
+
* extended to rectangular below). Derived directly from the forward
|
|
368
|
+
* Jacobian; equivalent to PyTorch's `svd_backward`
|
|
369
|
+
* (`aten/src/ATen/native/BatchLinearAlgebra.cpp`) and Townsend (2016)
|
|
370
|
+
* "Differentiating the Singular Value Decomposition" §3:
|
|
371
|
+
*
|
|
372
|
+
* Let α = skew(U^T · dU), β = skew(V^T · dV) (k×k, antisymmetric)
|
|
373
|
+
* Build C (k×k):
|
|
374
|
+
* C[i,i] = dS[i]
|
|
375
|
+
* C[i,j] = (α[i,j] + β[i,j]) / (s_j − s_i)
|
|
376
|
+
* + (α[i,j] − β[i,j]) / (s_j + s_i) for i ≠ j
|
|
377
|
+
* dA_in = U · C · V^T (m×n in-subspace part)
|
|
378
|
+
*
|
|
379
|
+
* Rectangular correction (when m > k, i.e. m > n):
|
|
380
|
+
* dA += (I − U U^T) · dU · diag(1/s) · V^T
|
|
381
|
+
* Rectangular correction (when n > k, i.e. n > m):
|
|
382
|
+
* dA += U · diag(1/s) · dV^T · (I − V V^T)
|
|
383
|
+
*
|
|
384
|
+
* Regularisation at repeated/near-zero singular values (PyTorch-equivalent
|
|
385
|
+
* subgradient choice): the (i,j) entry of C is masked to 0 whenever
|
|
386
|
+
* `|s_j − s_i| < REL_TOL · max(|s|)` (the "difference" denominator) or
|
|
387
|
+
* `|s_j + s_i| < REL_TOL · max(|s|)` (the "sum" denominator, only relevant
|
|
388
|
+
* when both are ~0). REL_TOL = 1e-10. This makes the gradient a subgradient
|
|
389
|
+
* at exact degeneracy — finite, but not the unique true derivative (which
|
|
390
|
+
* does not exist at degeneracies). The rectangular correction also masks
|
|
391
|
+
* 1/s_i when |s_i| < REL_TOL · max(|s|).
|
|
392
|
+
*
|
|
393
|
+
* Throws if input is not rank-2. For rank > 2 inputs the user should
|
|
394
|
+
* reshape first.
|
|
395
|
+
*/
|
|
396
|
+
svd(): {
|
|
397
|
+
U: TapedTensor;
|
|
398
|
+
S: TapedTensor;
|
|
399
|
+
V: TapedTensor;
|
|
400
|
+
};
|
|
401
|
+
/**
|
|
402
|
+
* Reverse-mode AD over the eigendecomposition of a rank-2 matrix.
|
|
403
|
+
*
|
|
404
|
+
* Symmetric path (`symmetric: true`):
|
|
405
|
+
* Forward: `A = U · diag(Λ) · U^T` for symmetric A (n×n).
|
|
406
|
+
* Adjoint (Magnus & Neudecker 1999 §10.6.6; PyTorch `linalg_eigh_backward`):
|
|
407
|
+
* F[i,j] = 1 / (λ_j − λ_i) for i ≠ j, 0 otherwise (with degeneracy mask)
|
|
408
|
+
* dA_raw = U · (diag(dΛ) + F ∘ (U^T · dU)) · U^T
|
|
409
|
+
* dA = (dA_raw + dA_raw^T) / 2 (symmetrise)
|
|
410
|
+
*
|
|
411
|
+
* Non-symmetric path (`symmetric: false`):
|
|
412
|
+
* Forward: `A = V · diag(λ) · V^{-1}` (V columns are right eigenvectors).
|
|
413
|
+
* Adjoint (Magnus & Neudecker 1999 §10.6 / Giles 2008 §3.2 / Townsend 2016 §4;
|
|
414
|
+
* cross-check: PyTorch `linalg_eig_backward`):
|
|
415
|
+
* E[i,j] = 1 / (λ_j − λ_i) for i ≠ j, 0 otherwise (with degeneracy mask)
|
|
416
|
+
* dA = V^{-T} · ( E ∘ (V^T · dV) + diag(dλ) ) · V^T
|
|
417
|
+
*
|
|
418
|
+
* Restrictions (all enforced — throw a clear error otherwise):
|
|
419
|
+
* 1. Eigenvalues must be real. The underlying matrix-eig primitive returns
|
|
420
|
+
* placeholder eigenvectors (not actual complex vectors) when complex
|
|
421
|
+
* eigenvalues arise, so the adjoint formula cannot be evaluated. Real-
|
|
422
|
+
* Schur differentiation would require complex arithmetic infrastructure
|
|
423
|
+
* throughout the Tape/TapedTensor stack, which is out of scope.
|
|
424
|
+
* 2. A must be diagonalisable (non-defective). The adjoint assumes V is
|
|
425
|
+
* invertible; defective inputs have algebraic > geometric multiplicity
|
|
426
|
+
* so V is rank-deficient. Detected by cond_∞(V) > 1e14.
|
|
427
|
+
*
|
|
428
|
+
* Regularisation at repeated eigenvalues (subgradient choice): mask
|
|
429
|
+
* F/E[i,j] = 0 when `|λ_i − λ_j| < REL_TOL · max(|λ|)`. REL_TOL = 1e-10.
|
|
430
|
+
*
|
|
431
|
+
* Throws if input is not rank-2 or square, if `symmetric` is missing, or
|
|
432
|
+
* (non-symmetric path) on complex eigenvalues / defective input.
|
|
433
|
+
*/
|
|
434
|
+
eig(opts: {
|
|
435
|
+
symmetric: boolean;
|
|
436
|
+
}): {
|
|
437
|
+
eigvals: TapedTensor;
|
|
438
|
+
eigvecs: TapedTensor;
|
|
439
|
+
};
|
|
440
|
+
private _eigSymmetric;
|
|
441
|
+
private _eigGeneral;
|
|
442
|
+
}
|
|
443
|
+
export {};
|
|
444
|
+
//# sourceMappingURL=tape.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"tape.d.ts","sourceRoot":"","sources":["../src/tape.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AACH,OAAO,EAAE,MAAM,EAAE,KAAK,EAAwB,MAAM,8BAA8B,CAAC;AAEnF,KAAK,UAAU,GAAG,CAAC,UAAU,EAAE,YAAY,KAAK,IAAI,CAAC;AAQrD;;;;;;;;GAQG;AACH,qBAAa,IAAI;IACf,OAAO,CAAC,KAAK,CAAkB;IAC/B,OAAO,CAAC,cAAc,CAAmC;IAKzD,OAAO,CAAC,QAAQ,CAAK;IACrB,OAAO,CAAC,WAAW,CAAM;IAEzB,wDAAwD;IACxD,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,YAAY,CAAA;KAAE,CAK7D;IAED,MAAM,CACJ,QAAQ,EAAE,aAAa,CAAC,MAAM,CAAC,EAC/B,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,UAAU,GACnB;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,YAAY,CAAA;KAAE,CAMxC;IAED,qEAAqE;IACrE,QAAQ,CAAC,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,YAAY,GAAG,IAAI,CASzD;IAED,YAAY,CAAC,EAAE,EAAE,MAAM,GAAG,YAAY,GAAG,SAAS,CAEjD;CACF;AAED;;;;;;GAMG;AACH,qBAAa,WAAW;IAQpB,QAAQ,CAAC,KAAK,EAAE,aAAa,CAAC,MAAM,CAAC;IACrC,QAAQ,CAAC,MAAM,EAAE,YAAY;IAC7B,QAAQ,CAAC,IAAI,EAAE,IAAI;IACnB,QAAQ,CAAC,EAAE,EAAE,MAAM;IAVrB;;;OAGG;IACH,QAAQ,CAAC,UAAU,CAAC,EAAE,aAAa,CAAC,KAAK,CAAC,CAAC;IAE3C,YACW,KAAK,EAAE,aAAa,CAAC,MAAM,CAAC,EAC5B,MAAM,EAAE,YAAY,EACpB,IAAI,EAAE,IAAI,EACV,EAAE,EAAE,MAAM,EACnB,UAAU,CAAC,EAAE,aAAa,CAAC,KAAK,CAAC,EAKlC;IAED;;;;;OAKG;IACH,IAAI,IAAI,IAAI,YAAY,CAEvB;IAED,MAAM,CAAC,iBAAiB,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,GAAG,WAAW,CAG3D;IAED,cAAc,IAAI,MAAM,CAEvB;IAED;;;OAGG;IACH,OAAO,CAAC,kBAAkB;IAI1B,GAAG,CAAC,KAAK,EAAE,WAAW,GAAG,WAAW,CAanC;IAED,GAAG,CAAC,KAAK,EAAE,WAAW,GAAG,WAAW,CAanC;IAED;;;;;;;;;;OAUG;IACH,MAAM,CAAC,KAAK,EAAE,WAAW,GAAG,WAAW,CA0BtC;IAED,GAAG,CAAC,KAAK,EAAE,WAAW,GAAG,WAAW,CA0BnC;IAED,KAAK,CAAC,CAAC,EAAE,MAAM,GAAG,WAAW,CAU5B;IAED;;;;;;;;;;;;;;;;;OAiBG;IACH,QAAQ,CAAC,KAAK,EAAE,WAAW,GAAG,WAAW,CA6GxC;IAED;;;;;;;;;;;;;;;;OAgBG;IACH,MAAM,CAAC,KAAK,EAAE,WAAW,GAAG,WAAW,CAqEtC;IAED,OAAO,CAAC,cAAc;IAatB;;;;;;;OAOG;IACH,GAAG,CAAC,IAAI,CAAC,EAAE,MAAM,GAAG,aAAa,CAAC,MAAM,CAAC,EAAE,IAAI,CAAC,EAAE;QAAE,QAAQ,CAAC,EAAE,OAAO,CAAA;KAAE,GAAG,WAAW,CA+CrF;IAED;;;;;OAKG;IACH,IAAI,CAAC,IAAI,CAAC,EAAE,MAAM,GAAG,aAAa,CAAC,MAAM,CAAC,EAAE,IAAI,CAAC,EAAE;QAAE,QAAQ,CAAC,EAAE,OAAO,CAAA;KAAE,GAAG,WAAW,CAwCtF;IAED;;;;;;;;;;;;;;OAcG;IACH,IAAI,CAAC,IAAI,CAAC,EAAE,MAAM,GAAG,aAAa,CAAC,MAAM,CAAC,EAAE,IAAI,CAAC,EAAE;QAAE,QAAQ,CAAC,EAAE,OAAO,CAAA;KAAE,GAAG,WAAW,CA6EtF;IAED;;;;;;OAMG;IACH,GAAG,CAAC,IAAI,CAAC,EAAE,MAAM,GAAG,aAAa,CAAC,MAAM,CAAC,EAAE,IAAI,CAAC,EAAE;QAAE,QAAQ,CAAC,EAAE,OAAO,CAAA;KAAE,GAAG,WAAW,CAoDrF;IAED;;;;;;OAMG;IACH,GAAG,CAAC,IAAI,CAAC,EAAE,MAAM,GAAG,aAAa,CAAC,MAAM,CAAC,EAAE,IAAI,CAAC,EAAE;QAAE,QAAQ,CAAC,EAAE,OAAO,CAAA;KAAE,GAAG,WAAW,CAiDrF;IAED;;;;;;;;;;;;OAYG;IACH,IAAI,CAAC,IAAI,CAAC,EAAE;QAAE,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,GAAG,KAAK,GAAG,KAAK,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,OAAO,CAAA;KAAE,GAAG,WAAW,CA0HzF;IAMD;;;;OAIG;IACH,GAAG,IAAI,WAAW,CAYjB;IAED;;;;OAIG;IACH,GAAG,IAAI,WAAW,CAYjB;IAED;;;;OAIG;IACH,GAAG,IAAI,WAAW,CAejB;IAED;;;;OAIG;IACH,GAAG,IAAI,WAAW,CAejB;IAED;;;;OAIG;IACH,GAAG,IAAI,WAAW,CAgBjB;IAED;;;;OAIG;IACH,IAAI,IAAI,WAAW,CAYlB;IAED;;;;OAIG;IACH,MAAM,IAAI,WAAW,CAYpB;IAED;;;;;;;;;;;;;;OAcG;IACH,GAAG,CAAC,CAAC,EAAE,MAAM,GAAG,WAAW,GAAG,WAAW,CA2CxC;IAED;;;;OAIG;IACH,UAAU,IAAI,WAAW,CAaxB;IAED;;;;;OAKG;IACH,GAAG,IAAI,WAAW,CAcjB;IAUD;;;;OAIG;IACH,OAAO,CAAC,iBAAiB;IAiBzB,mDAAmD;IACnD,IAAI,IAAI,WAAW,CAElB;IAED,qDAAqD;IACrD,IAAI,IAAI,WAAW,CAElB;IAED,6EAA6E;IAC7E,IAAI,IAAI,WAAW,CAElB;IAED,6CAA6C;IAC7C,IAAI,IAAI,WAAW,CAElB;IAED,gDAAgD;IAChD,IAAI,IAAI,WAAW,CAElB;IAED,+CAA+C;IAC/C,IAAI,IAAI,WAAW,CAElB;IAED,6DAA6D;IAC7D,KAAK,IAAI,WAAW,CAEnB;IAED,uEAAuE;IACvE,KAAK,IAAI,WAAW,CAEnB;IAED,yEAAyE;IACzE,KAAK,IAAI,WAAW,CAEnB;IAED,sDAAsD;IACtD,IAAI,IAAI,WAAW,CAElB;IAED,wDAAwD;IACxD,KAAK,IAAI,WAAW,CAEnB;IAED,gEAAgE;IAChE,KAAK,IAAI,WAAW,CAEnB;IAED,0EAA0E;IAC1E,KAAK,IAAI,WAAW,CAEnB;IAED,8EAA8E;IAC9E,IAAI,IAAI,WAAW,CAElB;IAED,yDAAyD;IACzD,IAAI,IAAI,WAAW,CAElB;IAED;;;;;;;;;;OAUG;IACH,KAAK,CAAC,KAAK,EAAE,WAAW,GAAG,WAAW,CAkBrC;IAMD;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA4BG;IACH,SAAS,CAAC,KAAK,EAAE,WAAW,EAAE,IAAI,EAAE,aAAa,CAAC,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,GAAG,WAAW,CA4KzF;IAMD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA0CG;IACH,GAAG,IAAI;QAAE,CAAC,EAAE,WAAW,CAAC;QAAC,CAAC,EAAE,WAAW,CAAC;QAAC,CAAC,EAAE,WAAW,CAAA;KAAE,CAkGxD;IAMD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAgCG;IACH,GAAG,CAAC,IAAI,EAAE;QAAE,SAAS,EAAE,OAAO,CAAA;KAAE,GAAG;QAAE,OAAO,EAAE,WAAW,CAAC;QAAC,OAAO,EAAE,WAAW,CAAA;KAAE,CAgBhF;IAED,OAAO,CAAC,aAAa;IAkCrB,OAAO,CAAC,WAAW;CA+DpB"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@danielsimonjr/mathts-autograd",
|
|
3
|
-
"version": "0.3.
|
|
3
|
+
"version": "0.3.15",
|
|
4
4
|
"description": "Forward + reverse-mode automatic differentiation for MathTS rank-N Tensor",
|
|
5
5
|
"author": "Daniel Simon Jr.",
|
|
6
6
|
"license": "MIT",
|
|
@@ -19,26 +19,26 @@
|
|
|
19
19
|
"README.md"
|
|
20
20
|
],
|
|
21
21
|
"scripts": {
|
|
22
|
-
"build": "tsup src/index.ts --format esm --dts
|
|
22
|
+
"build": "tsup src/index.ts --format esm --clean && tsc -p tsconfig.dts.json",
|
|
23
23
|
"dev": "tsup src/index.ts --format esm --dts --watch",
|
|
24
|
-
"test": "
|
|
24
|
+
"test": "bun test --isolate --timeout 30000",
|
|
25
25
|
"test:watch": "vitest",
|
|
26
|
-
"test:coverage": "
|
|
26
|
+
"test:coverage": "bun test --isolate --timeout 30000 --coverage",
|
|
27
27
|
"typecheck": "tsc --noEmit",
|
|
28
|
-
"lint": "
|
|
29
|
-
"lint:fix": "
|
|
28
|
+
"lint": "oxlint src",
|
|
29
|
+
"lint:fix": "oxlint src --fix",
|
|
30
30
|
"clean": "rm -rf dist",
|
|
31
31
|
"build:prod": "tsup src/index.ts --format esm --dts --clean --minify --treeshake"
|
|
32
32
|
},
|
|
33
33
|
"dependencies": {
|
|
34
|
-
"@danielsimonjr/mathts-core": "^0.
|
|
35
|
-
"@danielsimonjr/mathts-tensor": "^0.2.
|
|
34
|
+
"@danielsimonjr/mathts-core": "^0.15.4",
|
|
35
|
+
"@danielsimonjr/mathts-tensor": "^0.2.21"
|
|
36
36
|
},
|
|
37
37
|
"devDependencies": {
|
|
38
|
-
"@types/node": "^
|
|
38
|
+
"@types/node": "^26.4.0",
|
|
39
39
|
"tsup": "^8.0.0",
|
|
40
|
-
"typescript": "^
|
|
41
|
-
"vitest": "^
|
|
40
|
+
"typescript": "^7.0.2",
|
|
41
|
+
"vitest": "^5.0.0"
|
|
42
42
|
},
|
|
43
43
|
"publishConfig": {
|
|
44
44
|
"access": "public"
|