@danielsimonjr/mathts-autograd 0.3.18 → 0.3.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +1409 -2124
- package/package.json +18 -7
package/dist/index.js
CHANGED
|
@@ -1,2163 +1,1448 @@
|
|
|
1
|
-
|
|
2
|
-
import { Tensor } from "@danielsimonjr/mathts-tensor";
|
|
1
|
+
import { Tensor, tensorEig, tensorSvd } from "@danielsimonjr/mathts-tensor";
|
|
3
2
|
import { DUAL_UNARY_RULES } from "@danielsimonjr/mathts-core";
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
/** Hyperbolic sine. f'(x) = cosh(x). */
|
|
196
|
-
sinh() {
|
|
197
|
-
return this._rule(DUAL_UNARY_RULES.sinh);
|
|
198
|
-
}
|
|
199
|
-
/** Hyperbolic cosine. f'(x) = sinh(x). */
|
|
200
|
-
cosh() {
|
|
201
|
-
return this._rule(DUAL_UNARY_RULES.cosh);
|
|
202
|
-
}
|
|
203
|
-
/** Hyperbolic tangent. f'(x) = 1 − tanh²(x) = 1 − y². */
|
|
204
|
-
tanh() {
|
|
205
|
-
return this._rule(DUAL_UNARY_RULES.tanh);
|
|
206
|
-
}
|
|
207
|
-
/** Arcsine. f'(x) = 1/√(1 − x²). */
|
|
208
|
-
asin() {
|
|
209
|
-
return this._rule(DUAL_UNARY_RULES.asin);
|
|
210
|
-
}
|
|
211
|
-
/** Arccosine. f'(x) = −1/√(1 − x²). */
|
|
212
|
-
acos() {
|
|
213
|
-
return this._rule(DUAL_UNARY_RULES.acos);
|
|
214
|
-
}
|
|
215
|
-
/** Arctangent. f'(x) = 1/(1 + x²). */
|
|
216
|
-
atan() {
|
|
217
|
-
return this._rule(DUAL_UNARY_RULES.atan);
|
|
218
|
-
}
|
|
219
|
-
/** Inverse hyperbolic sine. f'(x) = 1/√(x² + 1). */
|
|
220
|
-
asinh() {
|
|
221
|
-
return this._rule(DUAL_UNARY_RULES.asinh);
|
|
222
|
-
}
|
|
223
|
-
/** Inverse hyperbolic cosine (x ≥ 1). f'(x) = 1/√(x² − 1). */
|
|
224
|
-
acosh() {
|
|
225
|
-
return this._rule(DUAL_UNARY_RULES.acosh);
|
|
226
|
-
}
|
|
227
|
-
/** Inverse hyperbolic tangent (|x| < 1). f'(x) = 1/(1 − x²). */
|
|
228
|
-
atanh() {
|
|
229
|
-
return this._rule(DUAL_UNARY_RULES.atanh);
|
|
230
|
-
}
|
|
231
|
-
/** Base-2 logarithm. f'(x) = 1/(x·ln2). */
|
|
232
|
-
log2() {
|
|
233
|
-
return this._rule(DUAL_UNARY_RULES.log2);
|
|
234
|
-
}
|
|
235
|
-
/** Base-10 logarithm. f'(x) = 1/(x·ln10). */
|
|
236
|
-
log10() {
|
|
237
|
-
return this._rule(DUAL_UNARY_RULES.log10);
|
|
238
|
-
}
|
|
239
|
-
/** log(1 + x). f'(x) = 1/(1 + x). */
|
|
240
|
-
log1p() {
|
|
241
|
-
return this._rule(DUAL_UNARY_RULES.log1p);
|
|
242
|
-
}
|
|
243
|
-
/** exp(x) − 1. f'(x) = eˣ = y + 1. */
|
|
244
|
-
expm1() {
|
|
245
|
-
return this._rule(DUAL_UNARY_RULES.expm1);
|
|
246
|
-
}
|
|
247
|
-
/** Cube root. f'(x) = 1/(3·y²) where y = x^(1/3) (→∞ at x = 0). */
|
|
248
|
-
cbrt() {
|
|
249
|
-
return this._rule(DUAL_UNARY_RULES.cbrt);
|
|
250
|
-
}
|
|
251
|
-
/** Sign (−1/0/+1). f'(x) = 0 almost everywhere. */
|
|
252
|
-
sign() {
|
|
253
|
-
return this._rule(DUAL_UNARY_RULES.sign);
|
|
254
|
-
}
|
|
255
|
-
/**
|
|
256
|
-
* Two-argument arctangent: atan2(this, other), this = y, other = x.
|
|
257
|
-
* Tangent: (b·a' − a·b') / (a² + b²), with a = this, b = other.
|
|
258
|
-
*/
|
|
259
|
-
atan2(other) {
|
|
260
|
-
this.checkSameShape(other, "atan2");
|
|
261
|
-
const p = new Float64Array(this.primal.length);
|
|
262
|
-
const t = new Float64Array(this.tangent.length);
|
|
263
|
-
for (let i = 0; i < p.length; i++) {
|
|
264
|
-
const a = this.primal[i];
|
|
265
|
-
const b = other.primal[i];
|
|
266
|
-
p[i] = Math.atan2(a, b);
|
|
267
|
-
const d = a * a + b * b;
|
|
268
|
-
t[i] = (b * this.tangent[i] - a * other.tangent[i]) / d;
|
|
269
|
-
}
|
|
270
|
-
return new _DualTensor(this.shape, p, t);
|
|
271
|
-
}
|
|
272
|
-
checkSameShape(other, op) {
|
|
273
|
-
if (this.shape.length !== other.shape.length || !this.shape.every((v, i) => v === other.shape[i])) {
|
|
274
|
-
throw new Error(`DualTensor.${op}: shape mismatch [${this.shape}] vs [${other.shape}]`);
|
|
275
|
-
}
|
|
276
|
-
}
|
|
3
|
+
//#region src/dual-tensor.ts
|
|
4
|
+
var DualTensor = class DualTensor {
|
|
5
|
+
shape;
|
|
6
|
+
primal;
|
|
7
|
+
tangent;
|
|
8
|
+
constructor(shape, primal, tangent) {
|
|
9
|
+
if (primal.length !== tangent.length) throw new Error(`DualTensor: primal length ${primal.length} != tangent length ${tangent.length}`);
|
|
10
|
+
this.shape = shape;
|
|
11
|
+
this.primal = primal;
|
|
12
|
+
this.tangent = tangent;
|
|
13
|
+
}
|
|
14
|
+
get data() {
|
|
15
|
+
return this.primal;
|
|
16
|
+
}
|
|
17
|
+
static fromTensor(t) {
|
|
18
|
+
return new DualTensor(t.shape, new Float64Array(t.data), new Float64Array(t.data.length));
|
|
19
|
+
}
|
|
20
|
+
static unitAt(t, i) {
|
|
21
|
+
const tan = new Float64Array(t.data.length);
|
|
22
|
+
tan[i] = 1;
|
|
23
|
+
return new DualTensor(t.shape, new Float64Array(t.data), tan);
|
|
24
|
+
}
|
|
25
|
+
toPrimalTensor() {
|
|
26
|
+
return new Tensor(this.shape, new Float64Array(this.primal));
|
|
27
|
+
}
|
|
28
|
+
toTangentTensor() {
|
|
29
|
+
return new Tensor(this.shape, new Float64Array(this.tangent));
|
|
30
|
+
}
|
|
31
|
+
add(other) {
|
|
32
|
+
this.checkSameShape(other, "add");
|
|
33
|
+
const p = new Float64Array(this.primal.length);
|
|
34
|
+
const t = new Float64Array(this.tangent.length);
|
|
35
|
+
for (let i = 0; i < p.length; i++) {
|
|
36
|
+
p[i] = this.primal[i] + other.primal[i];
|
|
37
|
+
t[i] = this.tangent[i] + other.tangent[i];
|
|
38
|
+
}
|
|
39
|
+
return new DualTensor(this.shape, p, t);
|
|
40
|
+
}
|
|
41
|
+
sub(other) {
|
|
42
|
+
this.checkSameShape(other, "sub");
|
|
43
|
+
const p = new Float64Array(this.primal.length);
|
|
44
|
+
const t = new Float64Array(this.tangent.length);
|
|
45
|
+
for (let i = 0; i < p.length; i++) {
|
|
46
|
+
p[i] = this.primal[i] - other.primal[i];
|
|
47
|
+
t[i] = this.tangent[i] - other.tangent[i];
|
|
48
|
+
}
|
|
49
|
+
return new DualTensor(this.shape, p, t);
|
|
50
|
+
}
|
|
51
|
+
mul(other) {
|
|
52
|
+
this.checkSameShape(other, "mul");
|
|
53
|
+
const p = new Float64Array(this.primal.length);
|
|
54
|
+
const t = new Float64Array(this.tangent.length);
|
|
55
|
+
if (this === other) for (let i = 0; i < p.length; i++) {
|
|
56
|
+
p[i] = this.primal[i] * this.primal[i];
|
|
57
|
+
t[i] = 2 * this.primal[i] * this.tangent[i];
|
|
58
|
+
}
|
|
59
|
+
else for (let i = 0; i < p.length; i++) {
|
|
60
|
+
p[i] = this.primal[i] * other.primal[i];
|
|
61
|
+
t[i] = this.tangent[i] * other.primal[i] + this.primal[i] * other.tangent[i];
|
|
62
|
+
}
|
|
63
|
+
return new DualTensor(this.shape, p, t);
|
|
64
|
+
}
|
|
65
|
+
scale(k) {
|
|
66
|
+
const p = new Float64Array(this.primal.length);
|
|
67
|
+
const t = new Float64Array(this.tangent.length);
|
|
68
|
+
for (let i = 0; i < p.length; i++) {
|
|
69
|
+
p[i] = this.primal[i] * k;
|
|
70
|
+
t[i] = this.tangent[i] * k;
|
|
71
|
+
}
|
|
72
|
+
return new DualTensor(this.shape, p, t);
|
|
73
|
+
}
|
|
74
|
+
divide(other) {
|
|
75
|
+
this.checkSameShape(other, "divide");
|
|
76
|
+
const p = new Float64Array(this.primal.length);
|
|
77
|
+
const t = new Float64Array(this.tangent.length);
|
|
78
|
+
if (this === other) for (let i = 0; i < p.length; i++) {
|
|
79
|
+
p[i] = 1;
|
|
80
|
+
t[i] = 0;
|
|
81
|
+
}
|
|
82
|
+
else for (let i = 0; i < p.length; i++) {
|
|
83
|
+
const a = this.primal[i];
|
|
84
|
+
const b = other.primal[i];
|
|
85
|
+
p[i] = a / b;
|
|
86
|
+
t[i] = (this.tangent[i] * b - a * other.tangent[i]) / (b * b);
|
|
87
|
+
}
|
|
88
|
+
return new DualTensor(this.shape, p, t);
|
|
89
|
+
}
|
|
90
|
+
_unaryElementwise(primalFn, deriv) {
|
|
91
|
+
const p = new Float64Array(this.primal.length);
|
|
92
|
+
const t = new Float64Array(this.tangent.length);
|
|
93
|
+
for (let i = 0; i < p.length; i++) {
|
|
94
|
+
const y = primalFn(this.primal[i]);
|
|
95
|
+
p[i] = y;
|
|
96
|
+
t[i] = deriv(this.primal[i], y) * this.tangent[i];
|
|
97
|
+
}
|
|
98
|
+
return new DualTensor(this.shape, p, t);
|
|
99
|
+
}
|
|
100
|
+
_rule(rule) {
|
|
101
|
+
return this._unaryElementwise(rule.primal, rule.deriv);
|
|
102
|
+
}
|
|
103
|
+
exp() {
|
|
104
|
+
return this._rule(DUAL_UNARY_RULES.exp);
|
|
105
|
+
}
|
|
106
|
+
log() {
|
|
107
|
+
return this._rule(DUAL_UNARY_RULES.log);
|
|
108
|
+
}
|
|
109
|
+
sin() {
|
|
110
|
+
return this._rule(DUAL_UNARY_RULES.sin);
|
|
111
|
+
}
|
|
112
|
+
cos() {
|
|
113
|
+
return this._rule(DUAL_UNARY_RULES.cos);
|
|
114
|
+
}
|
|
115
|
+
tan() {
|
|
116
|
+
return this._rule(DUAL_UNARY_RULES.tan);
|
|
117
|
+
}
|
|
118
|
+
sqrt() {
|
|
119
|
+
return this._rule(DUAL_UNARY_RULES.sqrt);
|
|
120
|
+
}
|
|
121
|
+
square() {
|
|
122
|
+
return this._rule(DUAL_UNARY_RULES.square);
|
|
123
|
+
}
|
|
124
|
+
reciprocal() {
|
|
125
|
+
return this._rule(DUAL_UNARY_RULES.reciprocal);
|
|
126
|
+
}
|
|
127
|
+
abs() {
|
|
128
|
+
return this._rule(DUAL_UNARY_RULES.abs);
|
|
129
|
+
}
|
|
130
|
+
pow(k) {
|
|
131
|
+
return this._unaryElementwise((x) => Math.pow(x, k), (x) => k * Math.pow(x, k - 1));
|
|
132
|
+
}
|
|
133
|
+
sinh() {
|
|
134
|
+
return this._rule(DUAL_UNARY_RULES.sinh);
|
|
135
|
+
}
|
|
136
|
+
cosh() {
|
|
137
|
+
return this._rule(DUAL_UNARY_RULES.cosh);
|
|
138
|
+
}
|
|
139
|
+
tanh() {
|
|
140
|
+
return this._rule(DUAL_UNARY_RULES.tanh);
|
|
141
|
+
}
|
|
142
|
+
asin() {
|
|
143
|
+
return this._rule(DUAL_UNARY_RULES.asin);
|
|
144
|
+
}
|
|
145
|
+
acos() {
|
|
146
|
+
return this._rule(DUAL_UNARY_RULES.acos);
|
|
147
|
+
}
|
|
148
|
+
atan() {
|
|
149
|
+
return this._rule(DUAL_UNARY_RULES.atan);
|
|
150
|
+
}
|
|
151
|
+
asinh() {
|
|
152
|
+
return this._rule(DUAL_UNARY_RULES.asinh);
|
|
153
|
+
}
|
|
154
|
+
acosh() {
|
|
155
|
+
return this._rule(DUAL_UNARY_RULES.acosh);
|
|
156
|
+
}
|
|
157
|
+
atanh() {
|
|
158
|
+
return this._rule(DUAL_UNARY_RULES.atanh);
|
|
159
|
+
}
|
|
160
|
+
log2() {
|
|
161
|
+
return this._rule(DUAL_UNARY_RULES.log2);
|
|
162
|
+
}
|
|
163
|
+
log10() {
|
|
164
|
+
return this._rule(DUAL_UNARY_RULES.log10);
|
|
165
|
+
}
|
|
166
|
+
log1p() {
|
|
167
|
+
return this._rule(DUAL_UNARY_RULES.log1p);
|
|
168
|
+
}
|
|
169
|
+
expm1() {
|
|
170
|
+
return this._rule(DUAL_UNARY_RULES.expm1);
|
|
171
|
+
}
|
|
172
|
+
cbrt() {
|
|
173
|
+
return this._rule(DUAL_UNARY_RULES.cbrt);
|
|
174
|
+
}
|
|
175
|
+
sign() {
|
|
176
|
+
return this._rule(DUAL_UNARY_RULES.sign);
|
|
177
|
+
}
|
|
178
|
+
atan2(other) {
|
|
179
|
+
this.checkSameShape(other, "atan2");
|
|
180
|
+
const p = new Float64Array(this.primal.length);
|
|
181
|
+
const t = new Float64Array(this.tangent.length);
|
|
182
|
+
for (let i = 0; i < p.length; i++) {
|
|
183
|
+
const a = this.primal[i];
|
|
184
|
+
const b = other.primal[i];
|
|
185
|
+
p[i] = Math.atan2(a, b);
|
|
186
|
+
const d = a * a + b * b;
|
|
187
|
+
t[i] = (b * this.tangent[i] - a * other.tangent[i]) / d;
|
|
188
|
+
}
|
|
189
|
+
return new DualTensor(this.shape, p, t);
|
|
190
|
+
}
|
|
191
|
+
checkSameShape(other, op) {
|
|
192
|
+
if (this.shape.length !== other.shape.length || !this.shape.every((v, i) => v === other.shape[i])) throw new Error(`DualTensor.${op}: shape mismatch [${this.shape}] vs [${other.shape}]`);
|
|
193
|
+
}
|
|
277
194
|
};
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
import { Tensor as Tensor2 } from "@danielsimonjr/mathts-tensor";
|
|
195
|
+
//#endregion
|
|
196
|
+
//#region src/forward-grad.ts
|
|
281
197
|
function forwardGrad(fn, x) {
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
}
|
|
301
|
-
const yDual = yDualRaw;
|
|
302
|
-
for (let kY = 0; kY < ySize; kY++) {
|
|
303
|
-
jacobianData[kY * xSize + kX] = yDual.tangent[kY];
|
|
304
|
-
}
|
|
305
|
-
}
|
|
306
|
-
return {
|
|
307
|
-
value: yPrimal,
|
|
308
|
-
jacobian: new Tensor2(jacobianShape, jacobianData)
|
|
309
|
-
};
|
|
198
|
+
const yProbeRaw = fn(DualTensor.fromTensor(x));
|
|
199
|
+
if (!(yProbeRaw instanceof DualTensor)) throw new Error("forwardGrad: fn must be AD-traceable — its return must propagate through DualTensor arithmetic (use add/sub/mul/scale on the argument). A fresh Tensor return loses the tangent and silently corrupts the Jacobian.");
|
|
200
|
+
const yPrimal = yProbeRaw.toPrimalTensor();
|
|
201
|
+
const jacobianShape = [...yPrimal.shape, ...x.shape];
|
|
202
|
+
const jacobianSize = jacobianShape.reduce((a, b) => a * b, 1);
|
|
203
|
+
const jacobianData = new Float64Array(jacobianSize);
|
|
204
|
+
const xSize = x.data.length;
|
|
205
|
+
const ySize = yPrimal.data.length;
|
|
206
|
+
for (let kX = 0; kX < xSize; kX++) {
|
|
207
|
+
const yDualRaw = fn(DualTensor.unitAt(x, kX));
|
|
208
|
+
if (!(yDualRaw instanceof DualTensor)) throw new Error("forwardGrad: fn lost AD trace mid-sweep (returned non-DualTensor)");
|
|
209
|
+
const yDual = yDualRaw;
|
|
210
|
+
for (let kY = 0; kY < ySize; kY++) jacobianData[kY * xSize + kX] = yDual.tangent[kY];
|
|
211
|
+
}
|
|
212
|
+
return {
|
|
213
|
+
value: yPrimal,
|
|
214
|
+
jacobian: new Tensor(jacobianShape, jacobianData)
|
|
215
|
+
};
|
|
310
216
|
}
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
import { Tensor as Tensor3, tensorSvd, tensorEig } from "@danielsimonjr/mathts-tensor";
|
|
217
|
+
//#endregion
|
|
218
|
+
//#region src/tape.ts
|
|
314
219
|
var Tape = class {
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
220
|
+
nodes = [];
|
|
221
|
+
inputGradSlots = /* @__PURE__ */ new Map();
|
|
222
|
+
nextOpId = 0;
|
|
223
|
+
nextInputId = -1;
|
|
224
|
+
allocate(size) {
|
|
225
|
+
const id = this.nextInputId--;
|
|
226
|
+
const gradSlot = new Float64Array(size);
|
|
227
|
+
this.inputGradSlots.set(id, gradSlot);
|
|
228
|
+
return {
|
|
229
|
+
id,
|
|
230
|
+
gradSlot
|
|
231
|
+
};
|
|
232
|
+
}
|
|
233
|
+
record(inputIds, outputSize, backward) {
|
|
234
|
+
const outputGradSlot = new Float64Array(outputSize);
|
|
235
|
+
const id = this.nextOpId++;
|
|
236
|
+
this.nodes.push({
|
|
237
|
+
inputIds,
|
|
238
|
+
backward,
|
|
239
|
+
outputGradSlot
|
|
240
|
+
});
|
|
241
|
+
this.inputGradSlots.set(id, outputGradSlot);
|
|
242
|
+
return {
|
|
243
|
+
id,
|
|
244
|
+
gradSlot: outputGradSlot
|
|
245
|
+
};
|
|
246
|
+
}
|
|
247
|
+
backward(outputId, outputGrad) {
|
|
248
|
+
const slot = this.inputGradSlots.get(outputId);
|
|
249
|
+
if (!slot) throw new Error(`Tape.backward: unknown outputId ${outputId}`);
|
|
250
|
+
slot.set(outputGrad);
|
|
251
|
+
for (let n = this.nodes.length - 1; n >= 0; n--) this.nodes[n].backward(this.nodes[n].outputGradSlot);
|
|
252
|
+
}
|
|
253
|
+
getInputGrad(id) {
|
|
254
|
+
return this.inputGradSlots.get(id);
|
|
255
|
+
}
|
|
349
256
|
};
|
|
350
|
-
var TapedTensor = class
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
|
|
1082
|
-
|
|
1083
|
-
|
|
1084
|
-
|
|
1085
|
-
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
|
|
1093
|
-
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
|
|
1097
|
-
|
|
1098
|
-
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
|
|
1111
|
-
|
|
1112
|
-
|
|
1113
|
-
|
|
1114
|
-
|
|
1115
|
-
|
|
1116
|
-
|
|
1117
|
-
|
|
1118
|
-
|
|
1119
|
-
|
|
1120
|
-
|
|
1121
|
-
|
|
1122
|
-
|
|
1123
|
-
|
|
1124
|
-
|
|
1125
|
-
|
|
1126
|
-
|
|
1127
|
-
|
|
1128
|
-
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
|
|
1132
|
-
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
|
|
1137
|
-
|
|
1138
|
-
|
|
1139
|
-
|
|
1140
|
-
|
|
1141
|
-
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
|
|
1145
|
-
|
|
1146
|
-
|
|
1147
|
-
|
|
1148
|
-
|
|
1149
|
-
|
|
1150
|
-
|
|
1151
|
-
|
|
1152
|
-
|
|
1153
|
-
|
|
1154
|
-
|
|
1155
|
-
|
|
1156
|
-
|
|
1157
|
-
|
|
1158
|
-
|
|
1159
|
-
|
|
1160
|
-
|
|
1161
|
-
|
|
1162
|
-
|
|
1163
|
-
|
|
1164
|
-
|
|
1165
|
-
|
|
1166
|
-
|
|
1167
|
-
|
|
1168
|
-
|
|
1169
|
-
|
|
1170
|
-
|
|
1171
|
-
|
|
1172
|
-
|
|
1173
|
-
|
|
1174
|
-
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
|
|
1179
|
-
|
|
1180
|
-
|
|
1181
|
-
|
|
1182
|
-
|
|
1183
|
-
|
|
1184
|
-
|
|
1185
|
-
|
|
1186
|
-
|
|
1187
|
-
|
|
1188
|
-
|
|
1189
|
-
|
|
1190
|
-
|
|
1191
|
-
|
|
1192
|
-
|
|
1193
|
-
|
|
1194
|
-
|
|
1195
|
-
|
|
1196
|
-
|
|
1197
|
-
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
|
|
1201
|
-
|
|
1202
|
-
|
|
1203
|
-
|
|
1204
|
-
|
|
1205
|
-
|
|
1206
|
-
|
|
1207
|
-
|
|
1208
|
-
|
|
1209
|
-
|
|
1210
|
-
|
|
1211
|
-
|
|
1212
|
-
|
|
1213
|
-
for (let i = 0; i < out.length; i++) out[i] = 1 / this.primal[i];
|
|
1214
|
-
const primalData = new Float64Array(this.primal);
|
|
1215
|
-
const thisGradSlot = this.tape.getInputGrad(this.id);
|
|
1216
|
-
const { id } = this.tape.record([this.id], out.length, (outputGrad) => {
|
|
1217
|
-
for (let i = 0; i < outputGrad.length; i++) {
|
|
1218
|
-
const xi = primalData[i];
|
|
1219
|
-
thisGradSlot[i] += -outputGrad[i] / (xi * xi);
|
|
1220
|
-
}
|
|
1221
|
-
});
|
|
1222
|
-
return new _TapedTensor(this.shape, out, this.tape, id);
|
|
1223
|
-
}
|
|
1224
|
-
/**
|
|
1225
|
-
* Elementwise absolute value: |x|.
|
|
1226
|
-
*
|
|
1227
|
-
* Adjoint: dX = dY · sign(x)
|
|
1228
|
-
* Subgradient at exact zero is defined as 0 (rather than undefined).
|
|
1229
|
-
*/
|
|
1230
|
-
abs() {
|
|
1231
|
-
const out = new Float64Array(this.primal.length);
|
|
1232
|
-
for (let i = 0; i < out.length; i++) out[i] = Math.abs(this.primal[i]);
|
|
1233
|
-
const primalData = new Float64Array(this.primal);
|
|
1234
|
-
const thisGradSlot = this.tape.getInputGrad(this.id);
|
|
1235
|
-
const { id } = this.tape.record([this.id], out.length, (outputGrad) => {
|
|
1236
|
-
for (let i = 0; i < outputGrad.length; i++) {
|
|
1237
|
-
const xi = primalData[i];
|
|
1238
|
-
const sign = xi > 0 ? 1 : xi < 0 ? -1 : 0;
|
|
1239
|
-
thisGradSlot[i] += outputGrad[i] * sign;
|
|
1240
|
-
}
|
|
1241
|
-
});
|
|
1242
|
-
return new _TapedTensor(this.shape, out, this.tape, id);
|
|
1243
|
-
}
|
|
1244
|
-
// ---------------------------------------------------------------------------
|
|
1245
|
-
// Extended elementwise transcendentals (hyperbolic, inverse-trig, log/exp
|
|
1246
|
-
// variants, sign, cbrt, atan2). Each builds a reverse-mode node via the
|
|
1247
|
-
// `_unaryElementwise` helper; `localGrad(x, y)` is dy/dx given input x and
|
|
1248
|
-
// output y. The helper caches both x and y so adjoints that need either
|
|
1249
|
-
// (e.g. tanh needs y, log needs x) are uniform.
|
|
1250
|
-
// ---------------------------------------------------------------------------
|
|
1251
|
-
/**
|
|
1252
|
-
* Build a reverse-mode node for an elementwise unary op.
|
|
1253
|
-
* @param primalFn forward value y = f(x)
|
|
1254
|
-
* @param localGrad local derivative dy/dx, given x and the computed y
|
|
1255
|
-
*/
|
|
1256
|
-
_unaryElementwise(primalFn, localGrad) {
|
|
1257
|
-
const xData = new Float64Array(this.primal);
|
|
1258
|
-
const out = new Float64Array(xData.length);
|
|
1259
|
-
for (let i = 0; i < out.length; i++) out[i] = primalFn(xData[i]);
|
|
1260
|
-
const yData = new Float64Array(out);
|
|
1261
|
-
const thisGradSlot = this.tape.getInputGrad(this.id);
|
|
1262
|
-
const { id } = this.tape.record([this.id], out.length, (outputGrad) => {
|
|
1263
|
-
for (let i = 0; i < outputGrad.length; i++) {
|
|
1264
|
-
thisGradSlot[i] += outputGrad[i] * localGrad(xData[i], yData[i]);
|
|
1265
|
-
}
|
|
1266
|
-
});
|
|
1267
|
-
return new _TapedTensor(this.shape, out, this.tape, id);
|
|
1268
|
-
}
|
|
1269
|
-
/** Hyperbolic sine. Adjoint: dX = dY · cosh(x). */
|
|
1270
|
-
sinh() {
|
|
1271
|
-
return this._unaryElementwise(Math.sinh, (x) => Math.cosh(x));
|
|
1272
|
-
}
|
|
1273
|
-
/** Hyperbolic cosine. Adjoint: dX = dY · sinh(x). */
|
|
1274
|
-
cosh() {
|
|
1275
|
-
return this._unaryElementwise(Math.cosh, (x) => Math.sinh(x));
|
|
1276
|
-
}
|
|
1277
|
-
/** Hyperbolic tangent. Adjoint: dX = dY · (1 − tanh²(x)) = dY · (1 − y²). */
|
|
1278
|
-
tanh() {
|
|
1279
|
-
return this._unaryElementwise(Math.tanh, (_x, y) => 1 - y * y);
|
|
1280
|
-
}
|
|
1281
|
-
/** Arcsine. Adjoint: dX = dY / √(1 − x²). */
|
|
1282
|
-
asin() {
|
|
1283
|
-
return this._unaryElementwise(Math.asin, (x) => 1 / Math.sqrt(1 - x * x));
|
|
1284
|
-
}
|
|
1285
|
-
/** Arccosine. Adjoint: dX = −dY / √(1 − x²). */
|
|
1286
|
-
acos() {
|
|
1287
|
-
return this._unaryElementwise(Math.acos, (x) => -1 / Math.sqrt(1 - x * x));
|
|
1288
|
-
}
|
|
1289
|
-
/** Arctangent. Adjoint: dX = dY / (1 + x²). */
|
|
1290
|
-
atan() {
|
|
1291
|
-
return this._unaryElementwise(Math.atan, (x) => 1 / (1 + x * x));
|
|
1292
|
-
}
|
|
1293
|
-
/** Inverse hyperbolic sine. Adjoint: dX = dY / √(x² + 1). */
|
|
1294
|
-
asinh() {
|
|
1295
|
-
return this._unaryElementwise(Math.asinh, (x) => 1 / Math.sqrt(x * x + 1));
|
|
1296
|
-
}
|
|
1297
|
-
/** Inverse hyperbolic cosine (x ≥ 1). Adjoint: dX = dY / √(x² − 1). */
|
|
1298
|
-
acosh() {
|
|
1299
|
-
return this._unaryElementwise(Math.acosh, (x) => 1 / Math.sqrt(x * x - 1));
|
|
1300
|
-
}
|
|
1301
|
-
/** Inverse hyperbolic tangent (|x| < 1). Adjoint: dX = dY / (1 − x²). */
|
|
1302
|
-
atanh() {
|
|
1303
|
-
return this._unaryElementwise(Math.atanh, (x) => 1 / (1 - x * x));
|
|
1304
|
-
}
|
|
1305
|
-
/** Base-2 logarithm. Adjoint: dX = dY / (x · ln2). */
|
|
1306
|
-
log2() {
|
|
1307
|
-
return this._unaryElementwise(Math.log2, (x) => 1 / (x * Math.LN2));
|
|
1308
|
-
}
|
|
1309
|
-
/** Base-10 logarithm. Adjoint: dX = dY / (x · ln10). */
|
|
1310
|
-
log10() {
|
|
1311
|
-
return this._unaryElementwise(Math.log10, (x) => 1 / (x * Math.LN10));
|
|
1312
|
-
}
|
|
1313
|
-
/** log(1 + x) (accurate near 0). Adjoint: dX = dY / (1 + x). */
|
|
1314
|
-
log1p() {
|
|
1315
|
-
return this._unaryElementwise(Math.log1p, (x) => 1 / (1 + x));
|
|
1316
|
-
}
|
|
1317
|
-
/** exp(x) − 1 (accurate near 0). Adjoint: dX = dY · eˣ = dY · (y + 1). */
|
|
1318
|
-
expm1() {
|
|
1319
|
-
return this._unaryElementwise(Math.expm1, (_x, y) => y + 1);
|
|
1320
|
-
}
|
|
1321
|
-
/** Cube root. Adjoint: dX = dY / (3 · y²) where y = x^(1/3) (→∞ at x = 0). */
|
|
1322
|
-
cbrt() {
|
|
1323
|
-
return this._unaryElementwise(Math.cbrt, (_x, y) => 1 / (3 * y * y));
|
|
1324
|
-
}
|
|
1325
|
-
/** Sign of x (−1/0/+1). Adjoint: 0 almost everywhere. */
|
|
1326
|
-
sign() {
|
|
1327
|
-
return this._unaryElementwise(Math.sign, () => 0);
|
|
1328
|
-
}
|
|
1329
|
-
/**
|
|
1330
|
-
* Two-argument arctangent: atan2(this, other), with this = y, other = x.
|
|
1331
|
-
*
|
|
1332
|
-
* Adjoints (a = this, b = other, d = a² + b²):
|
|
1333
|
-
* dA = dY · b / d
|
|
1334
|
-
* dB = −dY · a / d
|
|
1335
|
-
*
|
|
1336
|
-
* The aliased case atan2(a, a) is locally constant (±π/4 or ±3π/4), so its
|
|
1337
|
-
* gradient is 0 — produced naturally by contributing nothing when aliased
|
|
1338
|
-
* (dA + dB = dY · (b − a)/d = 0 at a = b).
|
|
1339
|
-
*/
|
|
1340
|
-
atan2(other) {
|
|
1341
|
-
this.checkSameShape(other, "atan2");
|
|
1342
|
-
const a = new Float64Array(this.primal);
|
|
1343
|
-
const b = new Float64Array(other.primal);
|
|
1344
|
-
const out = new Float64Array(a.length);
|
|
1345
|
-
for (let i = 0; i < out.length; i++) out[i] = Math.atan2(a[i], b[i]);
|
|
1346
|
-
const thisGradSlot = this.tape.getInputGrad(this.id);
|
|
1347
|
-
const otherGradSlot = this.tape.getInputGrad(other.id);
|
|
1348
|
-
const isAliased = this === other;
|
|
1349
|
-
const { id } = this.tape.record([this.id, other.id], out.length, (outputGrad) => {
|
|
1350
|
-
if (isAliased) return;
|
|
1351
|
-
for (let i = 0; i < outputGrad.length; i++) {
|
|
1352
|
-
const d = a[i] * a[i] + b[i] * b[i];
|
|
1353
|
-
thisGradSlot[i] += outputGrad[i] * b[i] / d;
|
|
1354
|
-
otherGradSlot[i] += -outputGrad[i] * a[i] / d;
|
|
1355
|
-
}
|
|
1356
|
-
});
|
|
1357
|
-
return new _TapedTensor(this.shape, out, this.tape, id);
|
|
1358
|
-
}
|
|
1359
|
-
// ---------------------------------------------------------------------------
|
|
1360
|
-
// tensordot — generalised dot product over explicit axis pairs.
|
|
1361
|
-
// ---------------------------------------------------------------------------
|
|
1362
|
-
/**
|
|
1363
|
-
* Reverse-mode AD over `Tensor.tensordot`.
|
|
1364
|
-
*
|
|
1365
|
-
* `axes[i] = [a, b]` contracts axis `a` of `this` with axis `b` of `other`.
|
|
1366
|
-
* The result shape is `this`'s non-contracted axes (in original order)
|
|
1367
|
-
* followed by `other`'s non-contracted axes (in original order).
|
|
1368
|
-
*
|
|
1369
|
-
* Adjoint derivation (NumPy/PyTorch tensordot backward, see Townsend 2016 §6,
|
|
1370
|
-
* and the canonical PyTorch implementation `TensorDotBackward0` in
|
|
1371
|
-
* `torch/csrc/autograd/generated/Functions.cpp`):
|
|
1372
|
-
*
|
|
1373
|
-
* Z = tensordot(A, B, axes)
|
|
1374
|
-
* dA = tensordot(dZ, B, [axes_of_dZ_corresponding_to_B's_free, B's_free])
|
|
1375
|
-
* then permute back into A's original axis order.
|
|
1376
|
-
* dB = tensordot(A, dZ, [A's_free, axes_of_dZ_corresponding_to_A's_free])
|
|
1377
|
-
* then permute back into B's original axis order.
|
|
1378
|
-
*
|
|
1379
|
-
* The axis-permutation bookkeeping is the trickiest part: Tensor.tensordot
|
|
1380
|
-
* produces output axes in the order [A's free axes (original A order),
|
|
1381
|
-
* B's free axes (original B order)] — and after the backward contractions
|
|
1382
|
-
* the survivors come out in B's original (resp. A's original) axis order
|
|
1383
|
-
* for the contracted side, which then needs to be permuted into pair order
|
|
1384
|
-
* (so axis k of the contracted block matches the kth pair) and finally
|
|
1385
|
-
* scattered back into A's (resp. B's) full original axis order.
|
|
1386
|
-
*
|
|
1387
|
-
* For the rank-2 × rank-2 single-axis case (i.e. ordinary matmul A·B with
|
|
1388
|
-
* axes = [[1, 0]]), this reduces to dA = dZ · Bᵀ, dB = Aᵀ · dZ — the same
|
|
1389
|
-
* adjoint as `TapedTensor.matmul`.
|
|
1390
|
-
*/
|
|
1391
|
-
tensordot(other, axes) {
|
|
1392
|
-
const aShape = [...this.shape];
|
|
1393
|
-
const bShape = [...other.shape];
|
|
1394
|
-
const aRank = aShape.length;
|
|
1395
|
-
const bRank = bShape.length;
|
|
1396
|
-
const aContractedInPairOrder = [];
|
|
1397
|
-
const bContractedInPairOrder = [];
|
|
1398
|
-
for (const [a, b] of axes) {
|
|
1399
|
-
aContractedInPairOrder.push(a);
|
|
1400
|
-
bContractedInPairOrder.push(b);
|
|
1401
|
-
}
|
|
1402
|
-
const aContractedSet = new Set(aContractedInPairOrder);
|
|
1403
|
-
const bContractedSet = new Set(bContractedInPairOrder);
|
|
1404
|
-
const aFree = [];
|
|
1405
|
-
for (let i = 0; i < aRank; i++) if (!aContractedSet.has(i)) aFree.push(i);
|
|
1406
|
-
const bFree = [];
|
|
1407
|
-
for (let i = 0; i < bRank; i++) if (!bContractedSet.has(i)) bFree.push(i);
|
|
1408
|
-
const aPrimalT = new Tensor3(aShape, new Float64Array(this.primal));
|
|
1409
|
-
const bPrimalT = new Tensor3(bShape, new Float64Array(other.primal));
|
|
1410
|
-
const zPrimal = aPrimalT.tensordot(bPrimalT, axes);
|
|
1411
|
-
const zShape = [...zPrimal.shape];
|
|
1412
|
-
const naFree = aFree.length;
|
|
1413
|
-
const nbFree = bFree.length;
|
|
1414
|
-
const aPrimalData = new Float64Array(this.primal);
|
|
1415
|
-
const bPrimalData = new Float64Array(other.primal);
|
|
1416
|
-
const aGradSlot = this.tape.getInputGrad(this.id);
|
|
1417
|
-
const bGradSlot = this.tape.getInputGrad(other.id);
|
|
1418
|
-
const { id } = this.tape.record(
|
|
1419
|
-
[this.id, other.id],
|
|
1420
|
-
zPrimal.data.length,
|
|
1421
|
-
(outputGrad) => {
|
|
1422
|
-
const dZ = new Tensor3(zShape, new Float64Array(outputGrad));
|
|
1423
|
-
const aPrimalTbk = new Tensor3(aShape, aPrimalData);
|
|
1424
|
-
const bPrimalTbk = new Tensor3(bShape, bPrimalData);
|
|
1425
|
-
const dAaxes = [];
|
|
1426
|
-
for (let i = 0; i < nbFree; i++) {
|
|
1427
|
-
dAaxes.push([naFree + i, bFree[i]]);
|
|
1428
|
-
}
|
|
1429
|
-
const dAPartial = dZ.tensordot(bPrimalTbk, dAaxes);
|
|
1430
|
-
const bContractedSortedByOriginal = [...bContractedInPairOrder].sort((x, y) => x - y);
|
|
1431
|
-
const nPairs = aContractedInPairOrder.length;
|
|
1432
|
-
const tailPerm = [];
|
|
1433
|
-
for (let k = 0; k < nPairs; k++) {
|
|
1434
|
-
const bOrigAxis = bContractedInPairOrder[k];
|
|
1435
|
-
const idx = bContractedSortedByOriginal.indexOf(bOrigAxis);
|
|
1436
|
-
tailPerm.push(idx);
|
|
1437
|
-
}
|
|
1438
|
-
const dAFirstPerm = [];
|
|
1439
|
-
for (let i = 0; i < naFree; i++) dAFirstPerm.push(i);
|
|
1440
|
-
for (let k = 0; k < nPairs; k++) dAFirstPerm.push(naFree + tailPerm[k]);
|
|
1441
|
-
const isFirstPermIdentity = dAFirstPerm.every((v, idx) => v === idx);
|
|
1442
|
-
const dAReordered = isFirstPermIdentity ? dAPartial : dAPartial.transpose(dAFirstPerm);
|
|
1443
|
-
const scatterA = new Array(aRank);
|
|
1444
|
-
for (let a = 0; a < aRank; a++) {
|
|
1445
|
-
const freeIdx = aFree.indexOf(a);
|
|
1446
|
-
if (freeIdx !== -1) {
|
|
1447
|
-
scatterA[a] = freeIdx;
|
|
1448
|
-
} else {
|
|
1449
|
-
const contIdx = aContractedInPairOrder.indexOf(a);
|
|
1450
|
-
scatterA[a] = naFree + contIdx;
|
|
1451
|
-
}
|
|
1452
|
-
}
|
|
1453
|
-
const isScatterAIdentity = scatterA.every((v, idx) => v === idx);
|
|
1454
|
-
const dAFinal = isScatterAIdentity ? dAReordered : dAReordered.transpose(scatterA);
|
|
1455
|
-
for (let i = 0; i < aGradSlot.length; i++) {
|
|
1456
|
-
aGradSlot[i] += dAFinal.data[i];
|
|
1457
|
-
}
|
|
1458
|
-
const dBaxes = [];
|
|
1459
|
-
for (let i = 0; i < naFree; i++) {
|
|
1460
|
-
dBaxes.push([aFree[i], i]);
|
|
1461
|
-
}
|
|
1462
|
-
const dBPartial = aPrimalTbk.tensordot(dZ, dBaxes);
|
|
1463
|
-
const aContractedSortedByOriginal = [...aContractedInPairOrder].sort((x, y) => x - y);
|
|
1464
|
-
const dBHeadPerm = [];
|
|
1465
|
-
for (let k = 0; k < nPairs; k++) {
|
|
1466
|
-
const aOrigAxis = aContractedInPairOrder[k];
|
|
1467
|
-
const idx = aContractedSortedByOriginal.indexOf(aOrigAxis);
|
|
1468
|
-
dBHeadPerm.push(idx);
|
|
1469
|
-
}
|
|
1470
|
-
const dBFirstPerm = [];
|
|
1471
|
-
for (let k = 0; k < nPairs; k++) dBFirstPerm.push(dBHeadPerm[k]);
|
|
1472
|
-
for (let i = 0; i < nbFree; i++) dBFirstPerm.push(nPairs + i);
|
|
1473
|
-
const isDBFirstPermIdentity = dBFirstPerm.every((v, idx) => v === idx);
|
|
1474
|
-
const dBReordered = isDBFirstPermIdentity ? dBPartial : dBPartial.transpose(dBFirstPerm);
|
|
1475
|
-
const scatterB = new Array(bRank);
|
|
1476
|
-
for (let b = 0; b < bRank; b++) {
|
|
1477
|
-
const freeIdx = bFree.indexOf(b);
|
|
1478
|
-
if (freeIdx !== -1) {
|
|
1479
|
-
scatterB[b] = nPairs + freeIdx;
|
|
1480
|
-
} else {
|
|
1481
|
-
const contIdx = bContractedInPairOrder.indexOf(b);
|
|
1482
|
-
scatterB[b] = contIdx;
|
|
1483
|
-
}
|
|
1484
|
-
}
|
|
1485
|
-
const isScatterBIdentity = scatterB.every((v, idx) => v === idx);
|
|
1486
|
-
const dBFinal = isScatterBIdentity ? dBReordered : dBReordered.transpose(scatterB);
|
|
1487
|
-
for (let i = 0; i < bGradSlot.length; i++) {
|
|
1488
|
-
bGradSlot[i] += dBFinal.data[i];
|
|
1489
|
-
}
|
|
1490
|
-
}
|
|
1491
|
-
);
|
|
1492
|
-
return new _TapedTensor(zShape, new Float64Array(zPrimal.data), this.tape, id);
|
|
1493
|
-
}
|
|
1494
|
-
// ---------------------------------------------------------------------------
|
|
1495
|
-
// SVD — full singular value decomposition on rank-2 input.
|
|
1496
|
-
// ---------------------------------------------------------------------------
|
|
1497
|
-
/**
|
|
1498
|
-
* Reverse-mode AD over the full SVD of a rank-2 matrix.
|
|
1499
|
-
*
|
|
1500
|
-
* Forward: `A = U · diag(S) · Vt`, where for input shape [m, n], k = min(m, n):
|
|
1501
|
-
* - U has shape [m, k]
|
|
1502
|
-
* - S has shape [k]
|
|
1503
|
-
* - Vt has shape [k, n] (Vt is V^T in the standard A = U Σ V^T convention,
|
|
1504
|
-
* i.e. its rows are right-singular-vector components)
|
|
1505
|
-
*
|
|
1506
|
-
* Returned TapedTensors share a single backward closure. When backward()
|
|
1507
|
-
* runs, it pulls dU, dS, dV from each output's gradient slot, assembles
|
|
1508
|
-
* dA, and writes to the input's gradient slot.
|
|
1509
|
-
*
|
|
1510
|
-
* Adjoint (real, distinct nonzero singular values, m = n square case;
|
|
1511
|
-
* extended to rectangular below). Derived directly from the forward
|
|
1512
|
-
* Jacobian; equivalent to PyTorch's `svd_backward`
|
|
1513
|
-
* (`aten/src/ATen/native/BatchLinearAlgebra.cpp`) and Townsend (2016)
|
|
1514
|
-
* "Differentiating the Singular Value Decomposition" §3:
|
|
1515
|
-
*
|
|
1516
|
-
* Let α = skew(U^T · dU), β = skew(V^T · dV) (k×k, antisymmetric)
|
|
1517
|
-
* Build C (k×k):
|
|
1518
|
-
* C[i,i] = dS[i]
|
|
1519
|
-
* C[i,j] = (α[i,j] + β[i,j]) / (s_j − s_i)
|
|
1520
|
-
* + (α[i,j] − β[i,j]) / (s_j + s_i) for i ≠ j
|
|
1521
|
-
* dA_in = U · C · V^T (m×n in-subspace part)
|
|
1522
|
-
*
|
|
1523
|
-
* Rectangular correction (when m > k, i.e. m > n):
|
|
1524
|
-
* dA += (I − U U^T) · dU · diag(1/s) · V^T
|
|
1525
|
-
* Rectangular correction (when n > k, i.e. n > m):
|
|
1526
|
-
* dA += U · diag(1/s) · dV^T · (I − V V^T)
|
|
1527
|
-
*
|
|
1528
|
-
* Regularisation at repeated/near-zero singular values (PyTorch-equivalent
|
|
1529
|
-
* subgradient choice): the (i,j) entry of C is masked to 0 whenever
|
|
1530
|
-
* `|s_j − s_i| < REL_TOL · max(|s|)` (the "difference" denominator) or
|
|
1531
|
-
* `|s_j + s_i| < REL_TOL · max(|s|)` (the "sum" denominator, only relevant
|
|
1532
|
-
* when both are ~0). REL_TOL = 1e-10. This makes the gradient a subgradient
|
|
1533
|
-
* at exact degeneracy — finite, but not the unique true derivative (which
|
|
1534
|
-
* does not exist at degeneracies). The rectangular correction also masks
|
|
1535
|
-
* 1/s_i when |s_i| < REL_TOL · max(|s|).
|
|
1536
|
-
*
|
|
1537
|
-
* Throws if input is not rank-2. For rank > 2 inputs the user should
|
|
1538
|
-
* reshape first.
|
|
1539
|
-
*/
|
|
1540
|
-
svd() {
|
|
1541
|
-
if (this.shape.length !== 2) {
|
|
1542
|
-
throw new Error(
|
|
1543
|
-
`TapedTensor.svd: expected rank-2 input, got rank ${this.shape.length}. Reshape the tensor to 2-D first.`
|
|
1544
|
-
);
|
|
1545
|
-
}
|
|
1546
|
-
const m = this.shape[0];
|
|
1547
|
-
const n = this.shape[1];
|
|
1548
|
-
const k = Math.min(m, n);
|
|
1549
|
-
const inputT = new Tensor3([m, n], new Float64Array(this.primal));
|
|
1550
|
-
const svdResult = tensorSvd(inputT, [0]);
|
|
1551
|
-
if (svdResult.truncatedDim !== k) {
|
|
1552
|
-
throw new Error(
|
|
1553
|
-
`TapedTensor.svd: internal \u2014 tensorSvd returned truncatedDim=${svdResult.truncatedDim}, expected ${k}. (A zero singular value was dropped by the default cutoff?)`
|
|
1554
|
-
);
|
|
1555
|
-
}
|
|
1556
|
-
const uPrimal = new Float64Array(svdResult.U.data);
|
|
1557
|
-
const sPrimal = new Float64Array(svdResult.S.data);
|
|
1558
|
-
const vtPrimal = new Float64Array(svdResult.V.data);
|
|
1559
|
-
const dUBuf = new Float64Array(m * k);
|
|
1560
|
-
const dSBuf = new Float64Array(k);
|
|
1561
|
-
const dVtBuf = new Float64Array(k * n);
|
|
1562
|
-
const thisGradSlot = this.tape.getInputGrad(this.id);
|
|
1563
|
-
const sharedBackward = () => {
|
|
1564
|
-
const dA = _svdBackward(uPrimal, sPrimal, vtPrimal, dUBuf, dSBuf, dVtBuf, m, n, k);
|
|
1565
|
-
for (let i = 0; i < thisGradSlot.length; i++) {
|
|
1566
|
-
thisGradSlot[i] += dA[i];
|
|
1567
|
-
}
|
|
1568
|
-
};
|
|
1569
|
-
const { id: uId } = this.tape.record([this.id], m * k, (outputGrad) => {
|
|
1570
|
-
for (let i = 0; i < dUBuf.length; i++) dUBuf[i] += outputGrad[i];
|
|
1571
|
-
sharedBackward();
|
|
1572
|
-
});
|
|
1573
|
-
const { id: sId } = this.tape.record([this.id], k, (outputGrad) => {
|
|
1574
|
-
for (let i = 0; i < dSBuf.length; i++) dSBuf[i] += outputGrad[i];
|
|
1575
|
-
});
|
|
1576
|
-
const { id: vId } = this.tape.record([this.id], k * n, (outputGrad) => {
|
|
1577
|
-
for (let i = 0; i < dVtBuf.length; i++) dVtBuf[i] += outputGrad[i];
|
|
1578
|
-
});
|
|
1579
|
-
const uTape = new _TapedTensor([m, k], uPrimal, this.tape, uId);
|
|
1580
|
-
const sTape = new _TapedTensor([k], sPrimal, this.tape, sId);
|
|
1581
|
-
const vTape = new _TapedTensor([k, n], vtPrimal, this.tape, vId);
|
|
1582
|
-
return { U: uTape, S: sTape, V: vTape };
|
|
1583
|
-
}
|
|
1584
|
-
// ---------------------------------------------------------------------------
|
|
1585
|
-
// Symmetric eigendecomposition.
|
|
1586
|
-
// ---------------------------------------------------------------------------
|
|
1587
|
-
/**
|
|
1588
|
-
* Reverse-mode AD over the eigendecomposition of a rank-2 matrix.
|
|
1589
|
-
*
|
|
1590
|
-
* Symmetric path (`symmetric: true`):
|
|
1591
|
-
* Forward: `A = U · diag(Λ) · U^T` for symmetric A (n×n).
|
|
1592
|
-
* Adjoint (Magnus & Neudecker 1999 §10.6.6; PyTorch `linalg_eigh_backward`):
|
|
1593
|
-
* F[i,j] = 1 / (λ_j − λ_i) for i ≠ j, 0 otherwise (with degeneracy mask)
|
|
1594
|
-
* dA_raw = U · (diag(dΛ) + F ∘ (U^T · dU)) · U^T
|
|
1595
|
-
* dA = (dA_raw + dA_raw^T) / 2 (symmetrise)
|
|
1596
|
-
*
|
|
1597
|
-
* Non-symmetric path (`symmetric: false`):
|
|
1598
|
-
* Forward: `A = V · diag(λ) · V^{-1}` (V columns are right eigenvectors).
|
|
1599
|
-
* Adjoint (Magnus & Neudecker 1999 §10.6 / Giles 2008 §3.2 / Townsend 2016 §4;
|
|
1600
|
-
* cross-check: PyTorch `linalg_eig_backward`):
|
|
1601
|
-
* E[i,j] = 1 / (λ_j − λ_i) for i ≠ j, 0 otherwise (with degeneracy mask)
|
|
1602
|
-
* dA = V^{-T} · ( E ∘ (V^T · dV) + diag(dλ) ) · V^T
|
|
1603
|
-
*
|
|
1604
|
-
* Restrictions (all enforced — throw a clear error otherwise):
|
|
1605
|
-
* 1. Eigenvalues must be real. The underlying matrix-eig primitive returns
|
|
1606
|
-
* placeholder eigenvectors (not actual complex vectors) when complex
|
|
1607
|
-
* eigenvalues arise, so the adjoint formula cannot be evaluated. Real-
|
|
1608
|
-
* Schur differentiation would require complex arithmetic infrastructure
|
|
1609
|
-
* throughout the Tape/TapedTensor stack, which is out of scope.
|
|
1610
|
-
* 2. A must be diagonalisable (non-defective). The adjoint assumes V is
|
|
1611
|
-
* invertible; defective inputs have algebraic > geometric multiplicity
|
|
1612
|
-
* so V is rank-deficient. Detected by cond_∞(V) > 1e14.
|
|
1613
|
-
*
|
|
1614
|
-
* Regularisation at repeated eigenvalues (subgradient choice): mask
|
|
1615
|
-
* F/E[i,j] = 0 when `|λ_i − λ_j| < REL_TOL · max(|λ|)`. REL_TOL = 1e-10.
|
|
1616
|
-
*
|
|
1617
|
-
* Throws if input is not rank-2 or square, if `symmetric` is missing, or
|
|
1618
|
-
* (non-symmetric path) on complex eigenvalues / defective input.
|
|
1619
|
-
*/
|
|
1620
|
-
eig(opts) {
|
|
1621
|
-
if (!opts || typeof opts.symmetric !== "boolean") {
|
|
1622
|
-
throw new Error(
|
|
1623
|
-
"TapedTensor.eig: opts.symmetric must be set to true or false. Pass { symmetric: true } for symmetric A, { symmetric: false } for general A."
|
|
1624
|
-
);
|
|
1625
|
-
}
|
|
1626
|
-
if (this.shape.length !== 2 || this.shape[0] !== this.shape[1]) {
|
|
1627
|
-
throw new Error(`TapedTensor.eig: expected rank-2 square input, got shape [${this.shape}].`);
|
|
1628
|
-
}
|
|
1629
|
-
const n = this.shape[0];
|
|
1630
|
-
if (opts.symmetric === true) {
|
|
1631
|
-
return this._eigSymmetric(n);
|
|
1632
|
-
}
|
|
1633
|
-
return this._eigGeneral(n);
|
|
1634
|
-
}
|
|
1635
|
-
_eigSymmetric(n) {
|
|
1636
|
-
const inputT = new Tensor3([n, n], new Float64Array(this.primal));
|
|
1637
|
-
const eigResult = tensorEig(inputT, [0], { symmetric: true, computeVectors: true });
|
|
1638
|
-
const lambdaPrimal = new Float64Array(eigResult.eigenvalues.data);
|
|
1639
|
-
if (!eigResult.eigenvectors) {
|
|
1640
|
-
throw new Error("TapedTensor.eig: tensorEig did not return eigenvectors");
|
|
1641
|
-
}
|
|
1642
|
-
const uPrimal = new Float64Array(eigResult.eigenvectors.data);
|
|
1643
|
-
const dLambdaBuf = new Float64Array(n);
|
|
1644
|
-
const dUBuf = new Float64Array(n * n);
|
|
1645
|
-
const thisGradSlot = this.tape.getInputGrad(this.id);
|
|
1646
|
-
const sharedBackward = () => {
|
|
1647
|
-
const dA = _eigSymBackward(uPrimal, lambdaPrimal, dUBuf, dLambdaBuf, n);
|
|
1648
|
-
for (let i = 0; i < thisGradSlot.length; i++) {
|
|
1649
|
-
thisGradSlot[i] += dA[i];
|
|
1650
|
-
}
|
|
1651
|
-
};
|
|
1652
|
-
const { id: lId } = this.tape.record([this.id], n, (outputGrad) => {
|
|
1653
|
-
for (let i = 0; i < dLambdaBuf.length; i++) dLambdaBuf[i] += outputGrad[i];
|
|
1654
|
-
sharedBackward();
|
|
1655
|
-
});
|
|
1656
|
-
const { id: uId } = this.tape.record([this.id], n * n, (outputGrad) => {
|
|
1657
|
-
for (let i = 0; i < dUBuf.length; i++) dUBuf[i] += outputGrad[i];
|
|
1658
|
-
});
|
|
1659
|
-
const lTape = new _TapedTensor([n], lambdaPrimal, this.tape, lId);
|
|
1660
|
-
const uTape = new _TapedTensor([n, n], uPrimal, this.tape, uId);
|
|
1661
|
-
return { eigvals: lTape, eigvecs: uTape };
|
|
1662
|
-
}
|
|
1663
|
-
_eigGeneral(n) {
|
|
1664
|
-
const inputT = new Tensor3([n, n], new Float64Array(this.primal));
|
|
1665
|
-
const eigResult = tensorEig(inputT, [0], { symmetric: false, computeVectors: true });
|
|
1666
|
-
if (eigResult.eigenvaluesImaginary) {
|
|
1667
|
-
const imData = eigResult.eigenvaluesImaginary.data;
|
|
1668
|
-
for (let i = 0; i < n; i++) {
|
|
1669
|
-
if (imData[i] !== 0) {
|
|
1670
|
-
throw new Error(
|
|
1671
|
-
"TapedTensor.eig({ symmetric: false }): input has complex eigenvalues. Reverse-mode AD over the general non-symmetric eig is restricted to real eigenvalues (the underlying matrix primitive does not return complex eigenvectors). Either symmetrise the input, or restrict to inputs with a real spectrum."
|
|
1672
|
-
);
|
|
1673
|
-
}
|
|
1674
|
-
}
|
|
1675
|
-
}
|
|
1676
|
-
if (!eigResult.eigenvectors) {
|
|
1677
|
-
throw new Error("TapedTensor.eig: tensorEig did not return eigenvectors");
|
|
1678
|
-
}
|
|
1679
|
-
const lambdaPrimal = new Float64Array(eigResult.eigenvalues.data);
|
|
1680
|
-
const vPrimal = new Float64Array(eigResult.eigenvectors.data);
|
|
1681
|
-
const vInvOrNull = _invertWithCondCheck(vPrimal, n, 1e14);
|
|
1682
|
-
if (vInvOrNull === null) {
|
|
1683
|
-
throw new Error(
|
|
1684
|
-
"TapedTensor.eig({ symmetric: false }): input appears defective (eigenvector matrix V has cond_\u221E(V) > 1e14). The general-case adjoint requires diagonalisability. Consider perturbing the input or using the symmetric path if A is symmetric."
|
|
1685
|
-
);
|
|
1686
|
-
}
|
|
1687
|
-
const vInv = vInvOrNull;
|
|
1688
|
-
const dLambdaBuf = new Float64Array(n);
|
|
1689
|
-
const dVBuf = new Float64Array(n * n);
|
|
1690
|
-
const thisGradSlot = this.tape.getInputGrad(this.id);
|
|
1691
|
-
const sharedBackward = () => {
|
|
1692
|
-
const dA = _eigGeneralBackward(vPrimal, vInv, lambdaPrimal, dVBuf, dLambdaBuf, n);
|
|
1693
|
-
for (let i = 0; i < thisGradSlot.length; i++) {
|
|
1694
|
-
thisGradSlot[i] += dA[i];
|
|
1695
|
-
}
|
|
1696
|
-
};
|
|
1697
|
-
const { id: lId } = this.tape.record([this.id], n, (outputGrad) => {
|
|
1698
|
-
for (let i = 0; i < dLambdaBuf.length; i++) dLambdaBuf[i] += outputGrad[i];
|
|
1699
|
-
sharedBackward();
|
|
1700
|
-
});
|
|
1701
|
-
const { id: vId } = this.tape.record([this.id], n * n, (outputGrad) => {
|
|
1702
|
-
for (let i = 0; i < dVBuf.length; i++) dVBuf[i] += outputGrad[i];
|
|
1703
|
-
});
|
|
1704
|
-
const lTape = new _TapedTensor([n], lambdaPrimal, this.tape, lId);
|
|
1705
|
-
const vTape = new _TapedTensor([n, n], vPrimal, this.tape, vId);
|
|
1706
|
-
return { eigvals: lTape, eigvecs: vTape };
|
|
1707
|
-
}
|
|
257
|
+
var TapedTensor = class TapedTensor {
|
|
258
|
+
shape;
|
|
259
|
+
primal;
|
|
260
|
+
tape;
|
|
261
|
+
id;
|
|
262
|
+
axisLabels;
|
|
263
|
+
constructor(shape, primal, tape, id, axisLabels) {
|
|
264
|
+
this.shape = shape;
|
|
265
|
+
this.primal = primal;
|
|
266
|
+
this.tape = tape;
|
|
267
|
+
this.id = id;
|
|
268
|
+
if (axisLabels !== void 0) this.axisLabels = [...axisLabels];
|
|
269
|
+
}
|
|
270
|
+
get data() {
|
|
271
|
+
return this.primal;
|
|
272
|
+
}
|
|
273
|
+
static fromTensorAsInput(t, tape) {
|
|
274
|
+
const { id } = tape.allocate(t.data.length);
|
|
275
|
+
return new TapedTensor(t.shape, new Float64Array(t.data), tape, id, t.axisLabels);
|
|
276
|
+
}
|
|
277
|
+
toPrimalTensor() {
|
|
278
|
+
return new Tensor(this.shape, new Float64Array(this.primal), this.axisLabels);
|
|
279
|
+
}
|
|
280
|
+
toPrimalTensorWith(axisLabels) {
|
|
281
|
+
return new Tensor(this.shape, new Float64Array(this.primal), axisLabels);
|
|
282
|
+
}
|
|
283
|
+
add(other) {
|
|
284
|
+
this.checkSameShape(other, "add");
|
|
285
|
+
const out = new Float64Array(this.primal.length);
|
|
286
|
+
for (let i = 0; i < out.length; i++) out[i] = this.primal[i] + other.primal[i];
|
|
287
|
+
const thisGradSlot = this.tape.getInputGrad(this.id);
|
|
288
|
+
const otherGradSlot = this.tape.getInputGrad(other.id);
|
|
289
|
+
const { id } = this.tape.record([this.id, other.id], out.length, (outputGrad) => {
|
|
290
|
+
for (let i = 0; i < outputGrad.length; i++) {
|
|
291
|
+
thisGradSlot[i] += outputGrad[i];
|
|
292
|
+
otherGradSlot[i] += outputGrad[i];
|
|
293
|
+
}
|
|
294
|
+
});
|
|
295
|
+
return new TapedTensor(this.shape, out, this.tape, id);
|
|
296
|
+
}
|
|
297
|
+
sub(other) {
|
|
298
|
+
this.checkSameShape(other, "sub");
|
|
299
|
+
const out = new Float64Array(this.primal.length);
|
|
300
|
+
for (let i = 0; i < out.length; i++) out[i] = this.primal[i] - other.primal[i];
|
|
301
|
+
const thisGradSlot = this.tape.getInputGrad(this.id);
|
|
302
|
+
const otherGradSlot = this.tape.getInputGrad(other.id);
|
|
303
|
+
const { id } = this.tape.record([this.id, other.id], out.length, (outputGrad) => {
|
|
304
|
+
for (let i = 0; i < outputGrad.length; i++) {
|
|
305
|
+
thisGradSlot[i] += outputGrad[i];
|
|
306
|
+
otherGradSlot[i] -= outputGrad[i];
|
|
307
|
+
}
|
|
308
|
+
});
|
|
309
|
+
return new TapedTensor(this.shape, out, this.tape, id);
|
|
310
|
+
}
|
|
311
|
+
divide(other) {
|
|
312
|
+
this.checkSameShape(other, "divide");
|
|
313
|
+
const out = new Float64Array(this.primal.length);
|
|
314
|
+
for (let i = 0; i < out.length; i++) out[i] = this.primal[i] / other.primal[i];
|
|
315
|
+
const thisPrimal = new Float64Array(this.primal);
|
|
316
|
+
const otherPrimal = new Float64Array(other.primal);
|
|
317
|
+
const thisGradSlot = this.tape.getInputGrad(this.id);
|
|
318
|
+
const otherGradSlot = this.tape.getInputGrad(other.id);
|
|
319
|
+
const isAliased = this === other;
|
|
320
|
+
const { id } = this.tape.record([this.id, other.id], out.length, (outputGrad) => {
|
|
321
|
+
for (let i = 0; i < outputGrad.length; i++) if (isAliased) {} else {
|
|
322
|
+
const b = otherPrimal[i];
|
|
323
|
+
const a = thisPrimal[i];
|
|
324
|
+
thisGradSlot[i] += outputGrad[i] / b;
|
|
325
|
+
otherGradSlot[i] += -outputGrad[i] * a / (b * b);
|
|
326
|
+
}
|
|
327
|
+
});
|
|
328
|
+
return new TapedTensor(this.shape, out, this.tape, id);
|
|
329
|
+
}
|
|
330
|
+
mul(other) {
|
|
331
|
+
this.checkSameShape(other, "mul");
|
|
332
|
+
const out = new Float64Array(this.primal.length);
|
|
333
|
+
for (let i = 0; i < out.length; i++) out[i] = this.primal[i] * other.primal[i];
|
|
334
|
+
const thisPrimal = this.primal;
|
|
335
|
+
const otherPrimal = other.primal;
|
|
336
|
+
const thisGradSlot = this.tape.getInputGrad(this.id);
|
|
337
|
+
const otherGradSlot = this.tape.getInputGrad(other.id);
|
|
338
|
+
const isAliased = this === other;
|
|
339
|
+
const { id } = this.tape.record([this.id, other.id], out.length, (outputGrad) => {
|
|
340
|
+
for (let i = 0; i < outputGrad.length; i++) if (isAliased) thisGradSlot[i] += 2 * outputGrad[i] * thisPrimal[i];
|
|
341
|
+
else {
|
|
342
|
+
thisGradSlot[i] += outputGrad[i] * otherPrimal[i];
|
|
343
|
+
otherGradSlot[i] += outputGrad[i] * thisPrimal[i];
|
|
344
|
+
}
|
|
345
|
+
});
|
|
346
|
+
return new TapedTensor(this.shape, out, this.tape, id);
|
|
347
|
+
}
|
|
348
|
+
scale(k) {
|
|
349
|
+
const out = new Float64Array(this.primal.length);
|
|
350
|
+
for (let i = 0; i < out.length; i++) out[i] = this.primal[i] * k;
|
|
351
|
+
const thisGradSlot = this.tape.getInputGrad(this.id);
|
|
352
|
+
const { id } = this.tape.record([this.id], out.length, (outputGrad) => {
|
|
353
|
+
for (let i = 0; i < outputGrad.length; i++) thisGradSlot[i] += outputGrad[i] * k;
|
|
354
|
+
});
|
|
355
|
+
return new TapedTensor(this.shape, out, this.tape, id);
|
|
356
|
+
}
|
|
357
|
+
contract(other) {
|
|
358
|
+
const aPrimal = this.toPrimalTensorWith(this.axisLabels);
|
|
359
|
+
const bPrimal = other.toPrimalTensorWith(other.axisLabels);
|
|
360
|
+
const yPrimal = aPrimal.contract(bPrimal);
|
|
361
|
+
const aLabels = this.axisLabels;
|
|
362
|
+
const bLabels = other.axisLabels;
|
|
363
|
+
const yLabels = yPrimal.axisLabels;
|
|
364
|
+
const aFreeAxes = [];
|
|
365
|
+
const aSharedAxes = [];
|
|
366
|
+
for (let i = 0; i < aLabels.length; i++) if (yLabels.some((yl) => yl.matches(aLabels[i]))) aFreeAxes.push(i);
|
|
367
|
+
else aSharedAxes.push(i);
|
|
368
|
+
const bFreeAxes = [];
|
|
369
|
+
const bSharedAxes = [];
|
|
370
|
+
for (let i = 0; i < bLabels.length; i++) if (yLabels.some((yl) => yl.matches(bLabels[i]))) bFreeAxes.push(i);
|
|
371
|
+
else bSharedAxes.push(i);
|
|
372
|
+
const aGradSlot = this.tape.getInputGrad(this.id);
|
|
373
|
+
const bGradSlot = this.tape.getInputGrad(other.id);
|
|
374
|
+
const aPrimalData = new Float64Array(this.primal);
|
|
375
|
+
const bPrimalData = new Float64Array(other.primal);
|
|
376
|
+
const aShape = [...this.shape];
|
|
377
|
+
const bShape = [...other.shape];
|
|
378
|
+
const { id } = this.tape.record([this.id, other.id], yPrimal.data.length, (outputGrad) => {
|
|
379
|
+
const dY = new Tensor([...yPrimal.shape], new Float64Array(outputGrad), yLabels);
|
|
380
|
+
let bRelabelled = new Tensor(bShape, bPrimalData, bLabels);
|
|
381
|
+
for (let k = 0; k < bSharedAxes.length; k++) {
|
|
382
|
+
const bAxis = bSharedAxes[k];
|
|
383
|
+
const aAxis = aSharedAxes[k];
|
|
384
|
+
bRelabelled = bRelabelled.replaceIndex(bLabels[bAxis], aLabels[aAxis]);
|
|
385
|
+
}
|
|
386
|
+
const dAData = _scatterToOriginalAxes(dY.contract(bRelabelled), aLabels);
|
|
387
|
+
for (let i = 0; i < aGradSlot.length; i++) aGradSlot[i] += dAData[i];
|
|
388
|
+
let aRelabelled = new Tensor(aShape, aPrimalData, aLabels);
|
|
389
|
+
for (let k = 0; k < aSharedAxes.length; k++) {
|
|
390
|
+
const aAxis = aSharedAxes[k];
|
|
391
|
+
const bAxis = bSharedAxes[k];
|
|
392
|
+
aRelabelled = aRelabelled.replaceIndex(aLabels[aAxis], bLabels[bAxis]);
|
|
393
|
+
}
|
|
394
|
+
const dBData = _scatterToOriginalAxes(aRelabelled.contract(dY), bLabels);
|
|
395
|
+
for (let i = 0; i < bGradSlot.length; i++) bGradSlot[i] += dBData[i];
|
|
396
|
+
});
|
|
397
|
+
return new TapedTensor([...yPrimal.shape], new Float64Array(yPrimal.data), this.tape, id, yLabels);
|
|
398
|
+
}
|
|
399
|
+
matmul(other) {
|
|
400
|
+
const rankA = this.shape.length;
|
|
401
|
+
const rankB = other.shape.length;
|
|
402
|
+
if (rankA < 2) throw new Error(`TapedTensor.matmul: 'this' operand must have rank ≥ 2, got rank ${rankA}`);
|
|
403
|
+
if (rankB < 2) throw new Error(`TapedTensor.matmul: 'other' operand must have rank ≥ 2, got rank ${rankB}`);
|
|
404
|
+
const batchRankA = rankA - 2;
|
|
405
|
+
const batchRankB = rankB - 2;
|
|
406
|
+
if (batchRankA !== batchRankB) throw new Error(`TapedTensor.matmul: batch rank mismatch (${batchRankA} vs ${batchRankB}); both operands must have the same rank`);
|
|
407
|
+
for (let b = 0; b < batchRankA; b++) if (this.shape[b] !== other.shape[b]) throw new Error(`TapedTensor.matmul: batch dimension ${b} mismatch (${this.shape[b]} vs ${other.shape[b]})`);
|
|
408
|
+
const m = this.shape[rankA - 2];
|
|
409
|
+
const k = this.shape[rankA - 1];
|
|
410
|
+
const k2 = other.shape[rankB - 2];
|
|
411
|
+
const n = other.shape[rankB - 1];
|
|
412
|
+
if (k !== k2) throw new Error(`TapedTensor.matmul: inner dimension mismatch (${k} vs ${k2})`);
|
|
413
|
+
const aShape = [...this.shape];
|
|
414
|
+
const bShape = [...other.shape];
|
|
415
|
+
const yShape = [
|
|
416
|
+
...aShape.slice(0, batchRankA),
|
|
417
|
+
m,
|
|
418
|
+
n
|
|
419
|
+
];
|
|
420
|
+
const aPrimalData = new Float64Array(this.primal);
|
|
421
|
+
const bPrimalData = new Float64Array(other.primal);
|
|
422
|
+
const yData = _batchedMatmul(aPrimalData, bPrimalData, aShape, bShape);
|
|
423
|
+
const aGradSlot = this.tape.getInputGrad(this.id);
|
|
424
|
+
const bGradSlot = this.tape.getInputGrad(other.id);
|
|
425
|
+
const { id } = this.tape.record([this.id, other.id], yData.length, (outputGrad) => {
|
|
426
|
+
const dA = _batchedMatmulGradA(outputGrad, bPrimalData, yShape, bShape);
|
|
427
|
+
for (let i = 0; i < aGradSlot.length; i++) aGradSlot[i] += dA[i];
|
|
428
|
+
const dB = _batchedMatmulGradB(aPrimalData, outputGrad, aShape, yShape);
|
|
429
|
+
for (let i = 0; i < bGradSlot.length; i++) bGradSlot[i] += dB[i];
|
|
430
|
+
});
|
|
431
|
+
return new TapedTensor(yShape, new Float64Array(yData), this.tape, id);
|
|
432
|
+
}
|
|
433
|
+
checkSameShape(other, op) {
|
|
434
|
+
if (this.shape.length !== other.shape.length || !this.shape.every((v, i) => v === other.shape[i])) throw new Error(`TapedTensor.${op}: shape mismatch [${this.shape}] vs [${other.shape}]`);
|
|
435
|
+
}
|
|
436
|
+
sum(axis, opts) {
|
|
437
|
+
const outT = this.toPrimalTensor().sum(axis, opts);
|
|
438
|
+
const inputShape = [...this.shape];
|
|
439
|
+
const inputSize = this.primal.length;
|
|
440
|
+
const keepDims = opts?.keepDims ?? false;
|
|
441
|
+
const axes = _resolveAxes(axis, this.shape.length);
|
|
442
|
+
const axisSet = new Set(axes);
|
|
443
|
+
const thisGradSlot = this.tape.getInputGrad(this.id);
|
|
444
|
+
const { id } = this.tape.record([this.id], outT.data.length, (outputGrad) => {
|
|
445
|
+
const outShape = [...outT.shape];
|
|
446
|
+
const outStrides = Tensor.rowMajorStrides(outShape);
|
|
447
|
+
Tensor.rowMajorStrides(inputShape);
|
|
448
|
+
const inCoords = new Array(inputShape.length).fill(0);
|
|
449
|
+
for (let n = 0; n < inputSize; n++) {
|
|
450
|
+
let outFlat = 0;
|
|
451
|
+
let outDim = 0;
|
|
452
|
+
for (let k = 0; k < inputShape.length; k++) if (axisSet.has(k)) {
|
|
453
|
+
if (keepDims) outDim++;
|
|
454
|
+
} else {
|
|
455
|
+
outFlat += inCoords[k] * outStrides[outDim];
|
|
456
|
+
outDim++;
|
|
457
|
+
}
|
|
458
|
+
thisGradSlot[n] += outputGrad[outFlat];
|
|
459
|
+
for (let k = inputShape.length - 1; k >= 0; k--) {
|
|
460
|
+
if (++inCoords[k] < inputShape[k]) break;
|
|
461
|
+
inCoords[k] = 0;
|
|
462
|
+
}
|
|
463
|
+
}
|
|
464
|
+
});
|
|
465
|
+
return new TapedTensor([...outT.shape], new Float64Array(outT.data), this.tape, id);
|
|
466
|
+
}
|
|
467
|
+
mean(axis, opts) {
|
|
468
|
+
const outT = this.toPrimalTensor().mean(axis, opts);
|
|
469
|
+
const inputShape = [...this.shape];
|
|
470
|
+
const inputSize = this.primal.length;
|
|
471
|
+
const keepDims = opts?.keepDims ?? false;
|
|
472
|
+
const axes = _resolveAxes(axis, this.shape.length);
|
|
473
|
+
const axisSet = new Set(axes);
|
|
474
|
+
const N = axes.reduce((prod, ax) => prod * this.shape[ax], 1);
|
|
475
|
+
const thisGradSlot = this.tape.getInputGrad(this.id);
|
|
476
|
+
const { id } = this.tape.record([this.id], outT.data.length, (outputGrad) => {
|
|
477
|
+
const outShape = [...outT.shape];
|
|
478
|
+
const outStrides = Tensor.rowMajorStrides(outShape);
|
|
479
|
+
const inCoords = new Array(inputShape.length).fill(0);
|
|
480
|
+
for (let n = 0; n < inputSize; n++) {
|
|
481
|
+
let outFlat = 0;
|
|
482
|
+
let outDim = 0;
|
|
483
|
+
for (let k = 0; k < inputShape.length; k++) if (axisSet.has(k)) {
|
|
484
|
+
if (keepDims) outDim++;
|
|
485
|
+
} else {
|
|
486
|
+
outFlat += inCoords[k] * outStrides[outDim];
|
|
487
|
+
outDim++;
|
|
488
|
+
}
|
|
489
|
+
thisGradSlot[n] += outputGrad[outFlat] / N;
|
|
490
|
+
for (let k = inputShape.length - 1; k >= 0; k--) {
|
|
491
|
+
if (++inCoords[k] < inputShape[k]) break;
|
|
492
|
+
inCoords[k] = 0;
|
|
493
|
+
}
|
|
494
|
+
}
|
|
495
|
+
});
|
|
496
|
+
return new TapedTensor([...outT.shape], new Float64Array(outT.data), this.tape, id);
|
|
497
|
+
}
|
|
498
|
+
prod(axis, opts) {
|
|
499
|
+
const outT = this.toPrimalTensor().prod(axis, opts);
|
|
500
|
+
const inputShape = [...this.shape];
|
|
501
|
+
const keepDims = opts?.keepDims ?? false;
|
|
502
|
+
const axes = _resolveAxes(axis, this.shape.length);
|
|
503
|
+
const axisSet = new Set(axes);
|
|
504
|
+
const primalData = new Float64Array(this.primal);
|
|
505
|
+
const thisGradSlot = this.tape.getInputGrad(this.id);
|
|
506
|
+
const { id } = this.tape.record([this.id], outT.data.length, (outputGrad) => {
|
|
507
|
+
const outShape = [...outT.shape];
|
|
508
|
+
const outStrides = Tensor.rowMajorStrides(outShape);
|
|
509
|
+
Tensor.rowMajorStrides(inputShape);
|
|
510
|
+
const groups = /* @__PURE__ */ new Map();
|
|
511
|
+
const inCoords = new Array(inputShape.length).fill(0);
|
|
512
|
+
for (let n = 0; n < primalData.length; n++) {
|
|
513
|
+
let outFlat = 0;
|
|
514
|
+
let outDim = 0;
|
|
515
|
+
for (let k = 0; k < inputShape.length; k++) if (axisSet.has(k)) {
|
|
516
|
+
if (keepDims) outDim++;
|
|
517
|
+
} else {
|
|
518
|
+
outFlat += inCoords[k] * outStrides[outDim];
|
|
519
|
+
outDim++;
|
|
520
|
+
}
|
|
521
|
+
const group = groups.get(outFlat);
|
|
522
|
+
if (group === void 0) groups.set(outFlat, [n]);
|
|
523
|
+
else group.push(n);
|
|
524
|
+
for (let k = inputShape.length - 1; k >= 0; k--) {
|
|
525
|
+
if (++inCoords[k] < inputShape[k]) break;
|
|
526
|
+
inCoords[k] = 0;
|
|
527
|
+
}
|
|
528
|
+
}
|
|
529
|
+
for (const [outFlat, group] of groups) {
|
|
530
|
+
const dY = outputGrad[outFlat];
|
|
531
|
+
const m = group.length;
|
|
532
|
+
const prefix = new Float64Array(m);
|
|
533
|
+
prefix[0] = 1;
|
|
534
|
+
for (let i = 1; i < m; i++) prefix[i] = prefix[i - 1] * primalData[group[i - 1]];
|
|
535
|
+
const suffix = new Float64Array(m);
|
|
536
|
+
suffix[m - 1] = 1;
|
|
537
|
+
for (let i = m - 2; i >= 0; i--) suffix[i] = suffix[i + 1] * primalData[group[i + 1]];
|
|
538
|
+
for (let i = 0; i < m; i++) thisGradSlot[group[i]] += dY * prefix[i] * suffix[i];
|
|
539
|
+
}
|
|
540
|
+
});
|
|
541
|
+
return new TapedTensor([...outT.shape], new Float64Array(outT.data), this.tape, id);
|
|
542
|
+
}
|
|
543
|
+
max(axis, opts) {
|
|
544
|
+
const outT = this.toPrimalTensor().max(axis, opts);
|
|
545
|
+
const inputShape = [...this.shape];
|
|
546
|
+
const keepDims = opts?.keepDims ?? false;
|
|
547
|
+
const axes = _resolveAxes(axis, this.shape.length);
|
|
548
|
+
const axisSet = new Set(axes);
|
|
549
|
+
const primalData = new Float64Array(this.primal);
|
|
550
|
+
const thisGradSlot = this.tape.getInputGrad(this.id);
|
|
551
|
+
const { id } = this.tape.record([this.id], outT.data.length, (outputGrad) => {
|
|
552
|
+
const outShape = [...outT.shape];
|
|
553
|
+
const outStrides = Tensor.rowMajorStrides(outShape);
|
|
554
|
+
const argmaxSlot = new Int32Array(outT.data.length).fill(-1);
|
|
555
|
+
const inCoords = new Array(inputShape.length).fill(0);
|
|
556
|
+
for (let n = 0; n < primalData.length; n++) {
|
|
557
|
+
let outFlat = 0;
|
|
558
|
+
let outDim = 0;
|
|
559
|
+
for (let k = 0; k < inputShape.length; k++) if (axisSet.has(k)) {
|
|
560
|
+
if (keepDims) outDim++;
|
|
561
|
+
} else {
|
|
562
|
+
outFlat += inCoords[k] * outStrides[outDim];
|
|
563
|
+
outDim++;
|
|
564
|
+
}
|
|
565
|
+
if (argmaxSlot[outFlat] === -1 || primalData[n] > primalData[argmaxSlot[outFlat]]) argmaxSlot[outFlat] = n;
|
|
566
|
+
for (let k = inputShape.length - 1; k >= 0; k--) {
|
|
567
|
+
if (++inCoords[k] < inputShape[k]) break;
|
|
568
|
+
inCoords[k] = 0;
|
|
569
|
+
}
|
|
570
|
+
}
|
|
571
|
+
for (let outFlat = 0; outFlat < outT.data.length; outFlat++) {
|
|
572
|
+
const winner = argmaxSlot[outFlat];
|
|
573
|
+
if (winner !== -1) thisGradSlot[winner] += outputGrad[outFlat];
|
|
574
|
+
}
|
|
575
|
+
});
|
|
576
|
+
return new TapedTensor([...outT.shape], new Float64Array(outT.data), this.tape, id);
|
|
577
|
+
}
|
|
578
|
+
min(axis, opts) {
|
|
579
|
+
const outT = this.toPrimalTensor().min(axis, opts);
|
|
580
|
+
const inputShape = [...this.shape];
|
|
581
|
+
const keepDims = opts?.keepDims ?? false;
|
|
582
|
+
const axes = _resolveAxes(axis, this.shape.length);
|
|
583
|
+
const axisSet = new Set(axes);
|
|
584
|
+
const primalData = new Float64Array(this.primal);
|
|
585
|
+
const thisGradSlot = this.tape.getInputGrad(this.id);
|
|
586
|
+
const { id } = this.tape.record([this.id], outT.data.length, (outputGrad) => {
|
|
587
|
+
const outShape = [...outT.shape];
|
|
588
|
+
const outStrides = Tensor.rowMajorStrides(outShape);
|
|
589
|
+
const argminSlot = new Int32Array(outT.data.length).fill(-1);
|
|
590
|
+
const inCoords = new Array(inputShape.length).fill(0);
|
|
591
|
+
for (let n = 0; n < primalData.length; n++) {
|
|
592
|
+
let outFlat = 0;
|
|
593
|
+
let outDim = 0;
|
|
594
|
+
for (let k = 0; k < inputShape.length; k++) if (axisSet.has(k)) {
|
|
595
|
+
if (keepDims) outDim++;
|
|
596
|
+
} else {
|
|
597
|
+
outFlat += inCoords[k] * outStrides[outDim];
|
|
598
|
+
outDim++;
|
|
599
|
+
}
|
|
600
|
+
if (argminSlot[outFlat] === -1 || primalData[n] < primalData[argminSlot[outFlat]]) argminSlot[outFlat] = n;
|
|
601
|
+
for (let k = inputShape.length - 1; k >= 0; k--) {
|
|
602
|
+
if (++inCoords[k] < inputShape[k]) break;
|
|
603
|
+
inCoords[k] = 0;
|
|
604
|
+
}
|
|
605
|
+
}
|
|
606
|
+
for (let outFlat = 0; outFlat < outT.data.length; outFlat++) {
|
|
607
|
+
const winner = argminSlot[outFlat];
|
|
608
|
+
if (winner !== -1) thisGradSlot[winner] += outputGrad[outFlat];
|
|
609
|
+
}
|
|
610
|
+
});
|
|
611
|
+
return new TapedTensor([...outT.shape], new Float64Array(outT.data), this.tape, id);
|
|
612
|
+
}
|
|
613
|
+
norm(opts) {
|
|
614
|
+
const p = opts?.p ?? 2;
|
|
615
|
+
const keepDims = opts?.keepDims ?? false;
|
|
616
|
+
const outT = this.toPrimalTensor().norm({
|
|
617
|
+
p,
|
|
618
|
+
axis: opts?.axis,
|
|
619
|
+
keepDims
|
|
620
|
+
});
|
|
621
|
+
const inputShape = [...this.shape];
|
|
622
|
+
const primalData = new Float64Array(this.primal);
|
|
623
|
+
const thisGradSlot = this.tape.getInputGrad(this.id);
|
|
624
|
+
const axes = opts?.axis !== void 0 ? [opts.axis] : _resolveAxes(void 0, this.shape.length);
|
|
625
|
+
const axisSet = new Set(axes);
|
|
626
|
+
if (p === 2 || p === "fro") {
|
|
627
|
+
const normData = new Float64Array(outT.data);
|
|
628
|
+
const { id } = this.tape.record([this.id], outT.data.length, (outputGrad) => {
|
|
629
|
+
const outShape = [...outT.shape];
|
|
630
|
+
const outStrides = Tensor.rowMajorStrides(outShape);
|
|
631
|
+
const inCoords = new Array(inputShape.length).fill(0);
|
|
632
|
+
for (let n = 0; n < primalData.length; n++) {
|
|
633
|
+
let outFlat = 0;
|
|
634
|
+
let outDim = 0;
|
|
635
|
+
for (let k = 0; k < inputShape.length; k++) if (axisSet.has(k)) {
|
|
636
|
+
if (keepDims) outDim++;
|
|
637
|
+
} else {
|
|
638
|
+
outFlat += inCoords[k] * outStrides[outDim];
|
|
639
|
+
outDim++;
|
|
640
|
+
}
|
|
641
|
+
const normVal = normData[outFlat];
|
|
642
|
+
if (normVal !== 0) thisGradSlot[n] += outputGrad[outFlat] * primalData[n] / normVal;
|
|
643
|
+
for (let k = inputShape.length - 1; k >= 0; k--) {
|
|
644
|
+
if (++inCoords[k] < inputShape[k]) break;
|
|
645
|
+
inCoords[k] = 0;
|
|
646
|
+
}
|
|
647
|
+
}
|
|
648
|
+
});
|
|
649
|
+
return new TapedTensor([...outT.shape], new Float64Array(outT.data), this.tape, id);
|
|
650
|
+
}
|
|
651
|
+
if (p === 1) {
|
|
652
|
+
const { id } = this.tape.record([this.id], outT.data.length, (outputGrad) => {
|
|
653
|
+
const outShape = [...outT.shape];
|
|
654
|
+
const outStrides = Tensor.rowMajorStrides(outShape);
|
|
655
|
+
const inCoords = new Array(inputShape.length).fill(0);
|
|
656
|
+
for (let n = 0; n < primalData.length; n++) {
|
|
657
|
+
let outFlat = 0;
|
|
658
|
+
let outDim = 0;
|
|
659
|
+
for (let k = 0; k < inputShape.length; k++) if (axisSet.has(k)) {
|
|
660
|
+
if (keepDims) outDim++;
|
|
661
|
+
} else {
|
|
662
|
+
outFlat += inCoords[k] * outStrides[outDim];
|
|
663
|
+
outDim++;
|
|
664
|
+
}
|
|
665
|
+
const xi = primalData[n];
|
|
666
|
+
const sign = xi > 0 ? 1 : xi < 0 ? -1 : 0;
|
|
667
|
+
thisGradSlot[n] += outputGrad[outFlat] * sign;
|
|
668
|
+
for (let k = inputShape.length - 1; k >= 0; k--) {
|
|
669
|
+
if (++inCoords[k] < inputShape[k]) break;
|
|
670
|
+
inCoords[k] = 0;
|
|
671
|
+
}
|
|
672
|
+
}
|
|
673
|
+
});
|
|
674
|
+
return new TapedTensor([...outT.shape], new Float64Array(outT.data), this.tape, id);
|
|
675
|
+
}
|
|
676
|
+
{
|
|
677
|
+
const { id } = this.tape.record([this.id], outT.data.length, (outputGrad) => {
|
|
678
|
+
const outShape = [...outT.shape];
|
|
679
|
+
const outStrides = Tensor.rowMajorStrides(outShape);
|
|
680
|
+
const argmaxAbsSlot = new Int32Array(outT.data.length).fill(-1);
|
|
681
|
+
const inCoords = new Array(inputShape.length).fill(0);
|
|
682
|
+
for (let n = 0; n < primalData.length; n++) {
|
|
683
|
+
let outFlat = 0;
|
|
684
|
+
let outDim = 0;
|
|
685
|
+
for (let k = 0; k < inputShape.length; k++) if (axisSet.has(k)) {
|
|
686
|
+
if (keepDims) outDim++;
|
|
687
|
+
} else {
|
|
688
|
+
outFlat += inCoords[k] * outStrides[outDim];
|
|
689
|
+
outDim++;
|
|
690
|
+
}
|
|
691
|
+
const prevWinner = argmaxAbsSlot[outFlat];
|
|
692
|
+
if (prevWinner === -1 || Math.abs(primalData[n]) > Math.abs(primalData[prevWinner])) argmaxAbsSlot[outFlat] = n;
|
|
693
|
+
for (let k = inputShape.length - 1; k >= 0; k--) {
|
|
694
|
+
if (++inCoords[k] < inputShape[k]) break;
|
|
695
|
+
inCoords[k] = 0;
|
|
696
|
+
}
|
|
697
|
+
}
|
|
698
|
+
for (let outFlat = 0; outFlat < outT.data.length; outFlat++) {
|
|
699
|
+
const winner = argmaxAbsSlot[outFlat];
|
|
700
|
+
if (winner !== -1) {
|
|
701
|
+
const xi = primalData[winner];
|
|
702
|
+
const sign = xi > 0 ? 1 : xi < 0 ? -1 : 0;
|
|
703
|
+
thisGradSlot[winner] += outputGrad[outFlat] * sign;
|
|
704
|
+
}
|
|
705
|
+
}
|
|
706
|
+
});
|
|
707
|
+
return new TapedTensor([...outT.shape], new Float64Array(outT.data), this.tape, id);
|
|
708
|
+
}
|
|
709
|
+
}
|
|
710
|
+
log() {
|
|
711
|
+
const out = new Float64Array(this.primal.length);
|
|
712
|
+
for (let i = 0; i < out.length; i++) out[i] = Math.log(this.primal[i]);
|
|
713
|
+
const primalData = new Float64Array(this.primal);
|
|
714
|
+
const thisGradSlot = this.tape.getInputGrad(this.id);
|
|
715
|
+
const { id } = this.tape.record([this.id], out.length, (outputGrad) => {
|
|
716
|
+
for (let i = 0; i < outputGrad.length; i++) thisGradSlot[i] += outputGrad[i] / primalData[i];
|
|
717
|
+
});
|
|
718
|
+
return new TapedTensor(this.shape, out, this.tape, id);
|
|
719
|
+
}
|
|
720
|
+
exp() {
|
|
721
|
+
const out = new Float64Array(this.primal.length);
|
|
722
|
+
for (let i = 0; i < out.length; i++) out[i] = Math.exp(this.primal[i]);
|
|
723
|
+
const expData = new Float64Array(out);
|
|
724
|
+
const thisGradSlot = this.tape.getInputGrad(this.id);
|
|
725
|
+
const { id } = this.tape.record([this.id], out.length, (outputGrad) => {
|
|
726
|
+
for (let i = 0; i < outputGrad.length; i++) thisGradSlot[i] += outputGrad[i] * expData[i];
|
|
727
|
+
});
|
|
728
|
+
return new TapedTensor(this.shape, out, this.tape, id);
|
|
729
|
+
}
|
|
730
|
+
sin() {
|
|
731
|
+
const out = new Float64Array(this.primal.length);
|
|
732
|
+
const cosData = new Float64Array(this.primal.length);
|
|
733
|
+
for (let i = 0; i < out.length; i++) {
|
|
734
|
+
out[i] = Math.sin(this.primal[i]);
|
|
735
|
+
cosData[i] = Math.cos(this.primal[i]);
|
|
736
|
+
}
|
|
737
|
+
const thisGradSlot = this.tape.getInputGrad(this.id);
|
|
738
|
+
const { id } = this.tape.record([this.id], out.length, (outputGrad) => {
|
|
739
|
+
for (let i = 0; i < outputGrad.length; i++) thisGradSlot[i] += outputGrad[i] * cosData[i];
|
|
740
|
+
});
|
|
741
|
+
return new TapedTensor(this.shape, out, this.tape, id);
|
|
742
|
+
}
|
|
743
|
+
cos() {
|
|
744
|
+
const out = new Float64Array(this.primal.length);
|
|
745
|
+
const sinData = new Float64Array(this.primal.length);
|
|
746
|
+
for (let i = 0; i < out.length; i++) {
|
|
747
|
+
out[i] = Math.cos(this.primal[i]);
|
|
748
|
+
sinData[i] = Math.sin(this.primal[i]);
|
|
749
|
+
}
|
|
750
|
+
const thisGradSlot = this.tape.getInputGrad(this.id);
|
|
751
|
+
const { id } = this.tape.record([this.id], out.length, (outputGrad) => {
|
|
752
|
+
for (let i = 0; i < outputGrad.length; i++) thisGradSlot[i] += -outputGrad[i] * sinData[i];
|
|
753
|
+
});
|
|
754
|
+
return new TapedTensor(this.shape, out, this.tape, id);
|
|
755
|
+
}
|
|
756
|
+
tan() {
|
|
757
|
+
const out = new Float64Array(this.primal.length);
|
|
758
|
+
const cosData = new Float64Array(this.primal.length);
|
|
759
|
+
for (let i = 0; i < out.length; i++) {
|
|
760
|
+
out[i] = Math.tan(this.primal[i]);
|
|
761
|
+
cosData[i] = Math.cos(this.primal[i]);
|
|
762
|
+
}
|
|
763
|
+
const thisGradSlot = this.tape.getInputGrad(this.id);
|
|
764
|
+
const { id } = this.tape.record([this.id], out.length, (outputGrad) => {
|
|
765
|
+
for (let i = 0; i < outputGrad.length; i++) {
|
|
766
|
+
const c = cosData[i];
|
|
767
|
+
thisGradSlot[i] += outputGrad[i] / (c * c);
|
|
768
|
+
}
|
|
769
|
+
});
|
|
770
|
+
return new TapedTensor(this.shape, out, this.tape, id);
|
|
771
|
+
}
|
|
772
|
+
sqrt() {
|
|
773
|
+
const out = new Float64Array(this.primal.length);
|
|
774
|
+
for (let i = 0; i < out.length; i++) out[i] = Math.sqrt(this.primal[i]);
|
|
775
|
+
const sqrtData = new Float64Array(out);
|
|
776
|
+
const thisGradSlot = this.tape.getInputGrad(this.id);
|
|
777
|
+
const { id } = this.tape.record([this.id], out.length, (outputGrad) => {
|
|
778
|
+
for (let i = 0; i < outputGrad.length; i++) thisGradSlot[i] += outputGrad[i] / (2 * sqrtData[i]);
|
|
779
|
+
});
|
|
780
|
+
return new TapedTensor(this.shape, out, this.tape, id);
|
|
781
|
+
}
|
|
782
|
+
square() {
|
|
783
|
+
const out = new Float64Array(this.primal.length);
|
|
784
|
+
for (let i = 0; i < out.length; i++) out[i] = this.primal[i] * this.primal[i];
|
|
785
|
+
const primalData = new Float64Array(this.primal);
|
|
786
|
+
const thisGradSlot = this.tape.getInputGrad(this.id);
|
|
787
|
+
const { id } = this.tape.record([this.id], out.length, (outputGrad) => {
|
|
788
|
+
for (let i = 0; i < outputGrad.length; i++) thisGradSlot[i] += outputGrad[i] * 2 * primalData[i];
|
|
789
|
+
});
|
|
790
|
+
return new TapedTensor(this.shape, out, this.tape, id);
|
|
791
|
+
}
|
|
792
|
+
pow(k) {
|
|
793
|
+
if (typeof k === "number") {
|
|
794
|
+
const out = new Float64Array(this.primal.length);
|
|
795
|
+
for (let i = 0; i < out.length; i++) out[i] = Math.pow(this.primal[i], k);
|
|
796
|
+
const primalData = new Float64Array(this.primal);
|
|
797
|
+
const thisGradSlot = this.tape.getInputGrad(this.id);
|
|
798
|
+
const { id } = this.tape.record([this.id], out.length, (outputGrad) => {
|
|
799
|
+
for (let i = 0; i < outputGrad.length; i++) thisGradSlot[i] += outputGrad[i] * k * Math.pow(primalData[i], k - 1);
|
|
800
|
+
});
|
|
801
|
+
return new TapedTensor(this.shape, out, this.tape, id);
|
|
802
|
+
}
|
|
803
|
+
const other = k;
|
|
804
|
+
this.checkSameShape(other, "pow");
|
|
805
|
+
const out = new Float64Array(this.primal.length);
|
|
806
|
+
for (let i = 0; i < out.length; i++) out[i] = Math.pow(this.primal[i], other.primal[i]);
|
|
807
|
+
const aPrimal = new Float64Array(this.primal);
|
|
808
|
+
const bPrimal = new Float64Array(other.primal);
|
|
809
|
+
const yPrimal = new Float64Array(out);
|
|
810
|
+
const thisGradSlot = this.tape.getInputGrad(this.id);
|
|
811
|
+
const otherGradSlot = this.tape.getInputGrad(other.id);
|
|
812
|
+
const isAliased = this === other;
|
|
813
|
+
const { id } = this.tape.record([this.id, other.id], out.length, (outputGrad) => {
|
|
814
|
+
for (let i = 0; i < outputGrad.length; i++) {
|
|
815
|
+
const a = aPrimal[i];
|
|
816
|
+
const b = bPrimal[i];
|
|
817
|
+
const y = yPrimal[i];
|
|
818
|
+
const g = outputGrad[i];
|
|
819
|
+
if (isAliased) thisGradSlot[i] += g * y * (Math.log(a) + 1);
|
|
820
|
+
else {
|
|
821
|
+
thisGradSlot[i] += g * b * Math.pow(a, b - 1);
|
|
822
|
+
otherGradSlot[i] += g * y * Math.log(a);
|
|
823
|
+
}
|
|
824
|
+
}
|
|
825
|
+
});
|
|
826
|
+
return new TapedTensor(this.shape, out, this.tape, id);
|
|
827
|
+
}
|
|
828
|
+
reciprocal() {
|
|
829
|
+
const out = new Float64Array(this.primal.length);
|
|
830
|
+
for (let i = 0; i < out.length; i++) out[i] = 1 / this.primal[i];
|
|
831
|
+
const primalData = new Float64Array(this.primal);
|
|
832
|
+
const thisGradSlot = this.tape.getInputGrad(this.id);
|
|
833
|
+
const { id } = this.tape.record([this.id], out.length, (outputGrad) => {
|
|
834
|
+
for (let i = 0; i < outputGrad.length; i++) {
|
|
835
|
+
const xi = primalData[i];
|
|
836
|
+
thisGradSlot[i] += -outputGrad[i] / (xi * xi);
|
|
837
|
+
}
|
|
838
|
+
});
|
|
839
|
+
return new TapedTensor(this.shape, out, this.tape, id);
|
|
840
|
+
}
|
|
841
|
+
abs() {
|
|
842
|
+
const out = new Float64Array(this.primal.length);
|
|
843
|
+
for (let i = 0; i < out.length; i++) out[i] = Math.abs(this.primal[i]);
|
|
844
|
+
const primalData = new Float64Array(this.primal);
|
|
845
|
+
const thisGradSlot = this.tape.getInputGrad(this.id);
|
|
846
|
+
const { id } = this.tape.record([this.id], out.length, (outputGrad) => {
|
|
847
|
+
for (let i = 0; i < outputGrad.length; i++) {
|
|
848
|
+
const xi = primalData[i];
|
|
849
|
+
const sign = xi > 0 ? 1 : xi < 0 ? -1 : 0;
|
|
850
|
+
thisGradSlot[i] += outputGrad[i] * sign;
|
|
851
|
+
}
|
|
852
|
+
});
|
|
853
|
+
return new TapedTensor(this.shape, out, this.tape, id);
|
|
854
|
+
}
|
|
855
|
+
_unaryElementwise(primalFn, localGrad) {
|
|
856
|
+
const xData = new Float64Array(this.primal);
|
|
857
|
+
const out = new Float64Array(xData.length);
|
|
858
|
+
for (let i = 0; i < out.length; i++) out[i] = primalFn(xData[i]);
|
|
859
|
+
const yData = new Float64Array(out);
|
|
860
|
+
const thisGradSlot = this.tape.getInputGrad(this.id);
|
|
861
|
+
const { id } = this.tape.record([this.id], out.length, (outputGrad) => {
|
|
862
|
+
for (let i = 0; i < outputGrad.length; i++) thisGradSlot[i] += outputGrad[i] * localGrad(xData[i], yData[i]);
|
|
863
|
+
});
|
|
864
|
+
return new TapedTensor(this.shape, out, this.tape, id);
|
|
865
|
+
}
|
|
866
|
+
sinh() {
|
|
867
|
+
return this._unaryElementwise(Math.sinh, (x) => Math.cosh(x));
|
|
868
|
+
}
|
|
869
|
+
cosh() {
|
|
870
|
+
return this._unaryElementwise(Math.cosh, (x) => Math.sinh(x));
|
|
871
|
+
}
|
|
872
|
+
tanh() {
|
|
873
|
+
return this._unaryElementwise(Math.tanh, (_x, y) => 1 - y * y);
|
|
874
|
+
}
|
|
875
|
+
asin() {
|
|
876
|
+
return this._unaryElementwise(Math.asin, (x) => 1 / Math.sqrt(1 - x * x));
|
|
877
|
+
}
|
|
878
|
+
acos() {
|
|
879
|
+
return this._unaryElementwise(Math.acos, (x) => -1 / Math.sqrt(1 - x * x));
|
|
880
|
+
}
|
|
881
|
+
atan() {
|
|
882
|
+
return this._unaryElementwise(Math.atan, (x) => 1 / (1 + x * x));
|
|
883
|
+
}
|
|
884
|
+
asinh() {
|
|
885
|
+
return this._unaryElementwise(Math.asinh, (x) => 1 / Math.sqrt(x * x + 1));
|
|
886
|
+
}
|
|
887
|
+
acosh() {
|
|
888
|
+
return this._unaryElementwise(Math.acosh, (x) => 1 / Math.sqrt(x * x - 1));
|
|
889
|
+
}
|
|
890
|
+
atanh() {
|
|
891
|
+
return this._unaryElementwise(Math.atanh, (x) => 1 / (1 - x * x));
|
|
892
|
+
}
|
|
893
|
+
log2() {
|
|
894
|
+
return this._unaryElementwise(Math.log2, (x) => 1 / (x * Math.LN2));
|
|
895
|
+
}
|
|
896
|
+
log10() {
|
|
897
|
+
return this._unaryElementwise(Math.log10, (x) => 1 / (x * Math.LN10));
|
|
898
|
+
}
|
|
899
|
+
log1p() {
|
|
900
|
+
return this._unaryElementwise(Math.log1p, (x) => 1 / (1 + x));
|
|
901
|
+
}
|
|
902
|
+
expm1() {
|
|
903
|
+
return this._unaryElementwise(Math.expm1, (_x, y) => y + 1);
|
|
904
|
+
}
|
|
905
|
+
cbrt() {
|
|
906
|
+
return this._unaryElementwise(Math.cbrt, (_x, y) => 1 / (3 * y * y));
|
|
907
|
+
}
|
|
908
|
+
sign() {
|
|
909
|
+
return this._unaryElementwise(Math.sign, () => 0);
|
|
910
|
+
}
|
|
911
|
+
atan2(other) {
|
|
912
|
+
this.checkSameShape(other, "atan2");
|
|
913
|
+
const a = new Float64Array(this.primal);
|
|
914
|
+
const b = new Float64Array(other.primal);
|
|
915
|
+
const out = new Float64Array(a.length);
|
|
916
|
+
for (let i = 0; i < out.length; i++) out[i] = Math.atan2(a[i], b[i]);
|
|
917
|
+
const thisGradSlot = this.tape.getInputGrad(this.id);
|
|
918
|
+
const otherGradSlot = this.tape.getInputGrad(other.id);
|
|
919
|
+
const isAliased = this === other;
|
|
920
|
+
const { id } = this.tape.record([this.id, other.id], out.length, (outputGrad) => {
|
|
921
|
+
if (isAliased) return;
|
|
922
|
+
for (let i = 0; i < outputGrad.length; i++) {
|
|
923
|
+
const d = a[i] * a[i] + b[i] * b[i];
|
|
924
|
+
thisGradSlot[i] += outputGrad[i] * b[i] / d;
|
|
925
|
+
otherGradSlot[i] += -outputGrad[i] * a[i] / d;
|
|
926
|
+
}
|
|
927
|
+
});
|
|
928
|
+
return new TapedTensor(this.shape, out, this.tape, id);
|
|
929
|
+
}
|
|
930
|
+
tensordot(other, axes) {
|
|
931
|
+
const aShape = [...this.shape];
|
|
932
|
+
const bShape = [...other.shape];
|
|
933
|
+
const aRank = aShape.length;
|
|
934
|
+
const bRank = bShape.length;
|
|
935
|
+
const aContractedInPairOrder = [];
|
|
936
|
+
const bContractedInPairOrder = [];
|
|
937
|
+
for (const [a, b] of axes) {
|
|
938
|
+
aContractedInPairOrder.push(a);
|
|
939
|
+
bContractedInPairOrder.push(b);
|
|
940
|
+
}
|
|
941
|
+
const aContractedSet = new Set(aContractedInPairOrder);
|
|
942
|
+
const bContractedSet = new Set(bContractedInPairOrder);
|
|
943
|
+
const aFree = [];
|
|
944
|
+
for (let i = 0; i < aRank; i++) if (!aContractedSet.has(i)) aFree.push(i);
|
|
945
|
+
const bFree = [];
|
|
946
|
+
for (let i = 0; i < bRank; i++) if (!bContractedSet.has(i)) bFree.push(i);
|
|
947
|
+
const aPrimalT = new Tensor(aShape, new Float64Array(this.primal));
|
|
948
|
+
const bPrimalT = new Tensor(bShape, new Float64Array(other.primal));
|
|
949
|
+
const zPrimal = aPrimalT.tensordot(bPrimalT, axes);
|
|
950
|
+
const zShape = [...zPrimal.shape];
|
|
951
|
+
const naFree = aFree.length;
|
|
952
|
+
const nbFree = bFree.length;
|
|
953
|
+
const aPrimalData = new Float64Array(this.primal);
|
|
954
|
+
const bPrimalData = new Float64Array(other.primal);
|
|
955
|
+
const aGradSlot = this.tape.getInputGrad(this.id);
|
|
956
|
+
const bGradSlot = this.tape.getInputGrad(other.id);
|
|
957
|
+
const { id } = this.tape.record([this.id, other.id], zPrimal.data.length, (outputGrad) => {
|
|
958
|
+
const dZ = new Tensor(zShape, new Float64Array(outputGrad));
|
|
959
|
+
const aPrimalTbk = new Tensor(aShape, aPrimalData);
|
|
960
|
+
const bPrimalTbk = new Tensor(bShape, bPrimalData);
|
|
961
|
+
const dAaxes = [];
|
|
962
|
+
for (let i = 0; i < nbFree; i++) dAaxes.push([naFree + i, bFree[i]]);
|
|
963
|
+
const dAPartial = dZ.tensordot(bPrimalTbk, dAaxes);
|
|
964
|
+
const bContractedSortedByOriginal = [...bContractedInPairOrder].sort((x, y) => x - y);
|
|
965
|
+
const nPairs = aContractedInPairOrder.length;
|
|
966
|
+
const tailPerm = [];
|
|
967
|
+
for (let k = 0; k < nPairs; k++) {
|
|
968
|
+
const bOrigAxis = bContractedInPairOrder[k];
|
|
969
|
+
const idx = bContractedSortedByOriginal.indexOf(bOrigAxis);
|
|
970
|
+
tailPerm.push(idx);
|
|
971
|
+
}
|
|
972
|
+
const dAFirstPerm = [];
|
|
973
|
+
for (let i = 0; i < naFree; i++) dAFirstPerm.push(i);
|
|
974
|
+
for (let k = 0; k < nPairs; k++) dAFirstPerm.push(naFree + tailPerm[k]);
|
|
975
|
+
const dAReordered = dAFirstPerm.every((v, idx) => v === idx) ? dAPartial : dAPartial.transpose(dAFirstPerm);
|
|
976
|
+
const scatterA = new Array(aRank);
|
|
977
|
+
for (let a = 0; a < aRank; a++) {
|
|
978
|
+
const freeIdx = aFree.indexOf(a);
|
|
979
|
+
if (freeIdx !== -1) scatterA[a] = freeIdx;
|
|
980
|
+
else {
|
|
981
|
+
const contIdx = aContractedInPairOrder.indexOf(a);
|
|
982
|
+
scatterA[a] = naFree + contIdx;
|
|
983
|
+
}
|
|
984
|
+
}
|
|
985
|
+
const dAFinal = scatterA.every((v, idx) => v === idx) ? dAReordered : dAReordered.transpose(scatterA);
|
|
986
|
+
for (let i = 0; i < aGradSlot.length; i++) aGradSlot[i] += dAFinal.data[i];
|
|
987
|
+
const dBaxes = [];
|
|
988
|
+
for (let i = 0; i < naFree; i++) dBaxes.push([aFree[i], i]);
|
|
989
|
+
const dBPartial = aPrimalTbk.tensordot(dZ, dBaxes);
|
|
990
|
+
const aContractedSortedByOriginal = [...aContractedInPairOrder].sort((x, y) => x - y);
|
|
991
|
+
const dBHeadPerm = [];
|
|
992
|
+
for (let k = 0; k < nPairs; k++) {
|
|
993
|
+
const aOrigAxis = aContractedInPairOrder[k];
|
|
994
|
+
const idx = aContractedSortedByOriginal.indexOf(aOrigAxis);
|
|
995
|
+
dBHeadPerm.push(idx);
|
|
996
|
+
}
|
|
997
|
+
const dBFirstPerm = [];
|
|
998
|
+
for (let k = 0; k < nPairs; k++) dBFirstPerm.push(dBHeadPerm[k]);
|
|
999
|
+
for (let i = 0; i < nbFree; i++) dBFirstPerm.push(nPairs + i);
|
|
1000
|
+
const dBReordered = dBFirstPerm.every((v, idx) => v === idx) ? dBPartial : dBPartial.transpose(dBFirstPerm);
|
|
1001
|
+
const scatterB = new Array(bRank);
|
|
1002
|
+
for (let b = 0; b < bRank; b++) {
|
|
1003
|
+
const freeIdx = bFree.indexOf(b);
|
|
1004
|
+
if (freeIdx !== -1) scatterB[b] = nPairs + freeIdx;
|
|
1005
|
+
else {
|
|
1006
|
+
const contIdx = bContractedInPairOrder.indexOf(b);
|
|
1007
|
+
scatterB[b] = contIdx;
|
|
1008
|
+
}
|
|
1009
|
+
}
|
|
1010
|
+
const dBFinal = scatterB.every((v, idx) => v === idx) ? dBReordered : dBReordered.transpose(scatterB);
|
|
1011
|
+
for (let i = 0; i < bGradSlot.length; i++) bGradSlot[i] += dBFinal.data[i];
|
|
1012
|
+
});
|
|
1013
|
+
return new TapedTensor(zShape, new Float64Array(zPrimal.data), this.tape, id);
|
|
1014
|
+
}
|
|
1015
|
+
svd() {
|
|
1016
|
+
if (this.shape.length !== 2) throw new Error(`TapedTensor.svd: expected rank-2 input, got rank ${this.shape.length}. Reshape the tensor to 2-D first.`);
|
|
1017
|
+
const m = this.shape[0];
|
|
1018
|
+
const n = this.shape[1];
|
|
1019
|
+
const k = Math.min(m, n);
|
|
1020
|
+
const inputT = new Tensor([m, n], new Float64Array(this.primal));
|
|
1021
|
+
const svdResult = tensorSvd(inputT, [0]);
|
|
1022
|
+
if (svdResult.truncatedDim !== k) throw new Error(`TapedTensor.svd: internal — tensorSvd returned truncatedDim=${svdResult.truncatedDim}, expected ${k}. (A zero singular value was dropped by the default cutoff?)`);
|
|
1023
|
+
const uPrimal = new Float64Array(svdResult.U.data);
|
|
1024
|
+
const sPrimal = new Float64Array(svdResult.S.data);
|
|
1025
|
+
const vtPrimal = new Float64Array(svdResult.V.data);
|
|
1026
|
+
const dUBuf = new Float64Array(m * k);
|
|
1027
|
+
const dSBuf = new Float64Array(k);
|
|
1028
|
+
const dVtBuf = new Float64Array(k * n);
|
|
1029
|
+
const thisGradSlot = this.tape.getInputGrad(this.id);
|
|
1030
|
+
const sharedBackward = () => {
|
|
1031
|
+
const dA = _svdBackward(uPrimal, sPrimal, vtPrimal, dUBuf, dSBuf, dVtBuf, m, n, k);
|
|
1032
|
+
for (let i = 0; i < thisGradSlot.length; i++) thisGradSlot[i] += dA[i];
|
|
1033
|
+
};
|
|
1034
|
+
const { id: uId } = this.tape.record([this.id], m * k, (outputGrad) => {
|
|
1035
|
+
for (let i = 0; i < dUBuf.length; i++) dUBuf[i] += outputGrad[i];
|
|
1036
|
+
sharedBackward();
|
|
1037
|
+
});
|
|
1038
|
+
const { id: sId } = this.tape.record([this.id], k, (outputGrad) => {
|
|
1039
|
+
for (let i = 0; i < dSBuf.length; i++) dSBuf[i] += outputGrad[i];
|
|
1040
|
+
});
|
|
1041
|
+
const { id: vId } = this.tape.record([this.id], k * n, (outputGrad) => {
|
|
1042
|
+
for (let i = 0; i < dVtBuf.length; i++) dVtBuf[i] += outputGrad[i];
|
|
1043
|
+
});
|
|
1044
|
+
return {
|
|
1045
|
+
U: new TapedTensor([m, k], uPrimal, this.tape, uId),
|
|
1046
|
+
S: new TapedTensor([k], sPrimal, this.tape, sId),
|
|
1047
|
+
V: new TapedTensor([k, n], vtPrimal, this.tape, vId)
|
|
1048
|
+
};
|
|
1049
|
+
}
|
|
1050
|
+
eig(opts) {
|
|
1051
|
+
if (!opts || typeof opts.symmetric !== "boolean") throw new Error("TapedTensor.eig: opts.symmetric must be set to true or false. Pass { symmetric: true } for symmetric A, { symmetric: false } for general A.");
|
|
1052
|
+
if (this.shape.length !== 2 || this.shape[0] !== this.shape[1]) throw new Error(`TapedTensor.eig: expected rank-2 square input, got shape [${this.shape}].`);
|
|
1053
|
+
const n = this.shape[0];
|
|
1054
|
+
if (opts.symmetric === true) return this._eigSymmetric(n);
|
|
1055
|
+
return this._eigGeneral(n);
|
|
1056
|
+
}
|
|
1057
|
+
_eigSymmetric(n) {
|
|
1058
|
+
const inputT = new Tensor([n, n], new Float64Array(this.primal));
|
|
1059
|
+
const eigResult = tensorEig(inputT, [0], {
|
|
1060
|
+
symmetric: true,
|
|
1061
|
+
computeVectors: true
|
|
1062
|
+
});
|
|
1063
|
+
const lambdaPrimal = new Float64Array(eigResult.eigenvalues.data);
|
|
1064
|
+
if (!eigResult.eigenvectors) throw new Error("TapedTensor.eig: tensorEig did not return eigenvectors");
|
|
1065
|
+
const uPrimal = new Float64Array(eigResult.eigenvectors.data);
|
|
1066
|
+
const dLambdaBuf = new Float64Array(n);
|
|
1067
|
+
const dUBuf = new Float64Array(n * n);
|
|
1068
|
+
const thisGradSlot = this.tape.getInputGrad(this.id);
|
|
1069
|
+
const sharedBackward = () => {
|
|
1070
|
+
const dA = _eigSymBackward(uPrimal, lambdaPrimal, dUBuf, dLambdaBuf, n);
|
|
1071
|
+
for (let i = 0; i < thisGradSlot.length; i++) thisGradSlot[i] += dA[i];
|
|
1072
|
+
};
|
|
1073
|
+
const { id: lId } = this.tape.record([this.id], n, (outputGrad) => {
|
|
1074
|
+
for (let i = 0; i < dLambdaBuf.length; i++) dLambdaBuf[i] += outputGrad[i];
|
|
1075
|
+
sharedBackward();
|
|
1076
|
+
});
|
|
1077
|
+
const { id: uId } = this.tape.record([this.id], n * n, (outputGrad) => {
|
|
1078
|
+
for (let i = 0; i < dUBuf.length; i++) dUBuf[i] += outputGrad[i];
|
|
1079
|
+
});
|
|
1080
|
+
return {
|
|
1081
|
+
eigvals: new TapedTensor([n], lambdaPrimal, this.tape, lId),
|
|
1082
|
+
eigvecs: new TapedTensor([n, n], uPrimal, this.tape, uId)
|
|
1083
|
+
};
|
|
1084
|
+
}
|
|
1085
|
+
_eigGeneral(n) {
|
|
1086
|
+
const inputT = new Tensor([n, n], new Float64Array(this.primal));
|
|
1087
|
+
const eigResult = tensorEig(inputT, [0], {
|
|
1088
|
+
symmetric: false,
|
|
1089
|
+
computeVectors: true
|
|
1090
|
+
});
|
|
1091
|
+
if (eigResult.eigenvaluesImaginary) {
|
|
1092
|
+
const imData = eigResult.eigenvaluesImaginary.data;
|
|
1093
|
+
for (let i = 0; i < n; i++) if (imData[i] !== 0) throw new Error("TapedTensor.eig({ symmetric: false }): input has complex eigenvalues. Reverse-mode AD over the general non-symmetric eig is restricted to real eigenvalues (the underlying matrix primitive does not return complex eigenvectors). Either symmetrise the input, or restrict to inputs with a real spectrum.");
|
|
1094
|
+
}
|
|
1095
|
+
if (!eigResult.eigenvectors) throw new Error("TapedTensor.eig: tensorEig did not return eigenvectors");
|
|
1096
|
+
const lambdaPrimal = new Float64Array(eigResult.eigenvalues.data);
|
|
1097
|
+
const vPrimal = new Float64Array(eigResult.eigenvectors.data);
|
|
1098
|
+
const vInvOrNull = _invertWithCondCheck(vPrimal, n, 0x5af3107a4000);
|
|
1099
|
+
if (vInvOrNull === null) throw new Error("TapedTensor.eig({ symmetric: false }): input appears defective (eigenvector matrix V has cond_∞(V) > 1e14). The general-case adjoint requires diagonalisability. Consider perturbing the input or using the symmetric path if A is symmetric.");
|
|
1100
|
+
const vInv = vInvOrNull;
|
|
1101
|
+
const dLambdaBuf = new Float64Array(n);
|
|
1102
|
+
const dVBuf = new Float64Array(n * n);
|
|
1103
|
+
const thisGradSlot = this.tape.getInputGrad(this.id);
|
|
1104
|
+
const sharedBackward = () => {
|
|
1105
|
+
const dA = _eigGeneralBackward(vPrimal, vInv, lambdaPrimal, dVBuf, dLambdaBuf, n);
|
|
1106
|
+
for (let i = 0; i < thisGradSlot.length; i++) thisGradSlot[i] += dA[i];
|
|
1107
|
+
};
|
|
1108
|
+
const { id: lId } = this.tape.record([this.id], n, (outputGrad) => {
|
|
1109
|
+
for (let i = 0; i < dLambdaBuf.length; i++) dLambdaBuf[i] += outputGrad[i];
|
|
1110
|
+
sharedBackward();
|
|
1111
|
+
});
|
|
1112
|
+
const { id: vId } = this.tape.record([this.id], n * n, (outputGrad) => {
|
|
1113
|
+
for (let i = 0; i < dVBuf.length; i++) dVBuf[i] += outputGrad[i];
|
|
1114
|
+
});
|
|
1115
|
+
return {
|
|
1116
|
+
eigvals: new TapedTensor([n], lambdaPrimal, this.tape, lId),
|
|
1117
|
+
eigvecs: new TapedTensor([n, n], vPrimal, this.tape, vId)
|
|
1118
|
+
};
|
|
1119
|
+
}
|
|
1708
1120
|
};
|
|
1709
1121
|
function _svdBackward(U, s, Vt, dU, dS, dVt, m, n, k) {
|
|
1710
|
-
|
|
1711
|
-
|
|
1712
|
-
|
|
1713
|
-
|
|
1714
|
-
|
|
1715
|
-
|
|
1716
|
-
|
|
1717
|
-
|
|
1718
|
-
|
|
1719
|
-
|
|
1720
|
-
|
|
1721
|
-
|
|
1722
|
-
|
|
1723
|
-
|
|
1724
|
-
|
|
1725
|
-
|
|
1726
|
-
|
|
1727
|
-
|
|
1728
|
-
|
|
1729
|
-
|
|
1730
|
-
|
|
1731
|
-
|
|
1732
|
-
|
|
1733
|
-
|
|
1734
|
-
|
|
1735
|
-
|
|
1736
|
-
|
|
1737
|
-
|
|
1738
|
-
|
|
1739
|
-
|
|
1740
|
-
|
|
1741
|
-
|
|
1742
|
-
|
|
1743
|
-
|
|
1744
|
-
|
|
1745
|
-
|
|
1746
|
-
|
|
1747
|
-
|
|
1748
|
-
|
|
1749
|
-
|
|
1750
|
-
|
|
1751
|
-
|
|
1752
|
-
|
|
1753
|
-
|
|
1754
|
-
|
|
1755
|
-
|
|
1756
|
-
|
|
1757
|
-
|
|
1758
|
-
|
|
1759
|
-
|
|
1760
|
-
|
|
1761
|
-
let term = 0;
|
|
1762
|
-
if (Math.abs(denomDiff) > tolAbs) {
|
|
1763
|
-
term += (alpha + beta) / denomDiff;
|
|
1764
|
-
}
|
|
1765
|
-
if (Math.abs(denomSum) > tolAbs) {
|
|
1766
|
-
term += (alpha - beta) / denomSum;
|
|
1767
|
-
}
|
|
1768
|
-
C[i * k + j] = term;
|
|
1769
|
-
}
|
|
1770
|
-
}
|
|
1771
|
-
}
|
|
1772
|
-
const UC = _matmulMNK(U, C, m, k, k, false, false);
|
|
1773
|
-
const dA = _matmulMNK(UC, Vt, m, k, n, false, false);
|
|
1774
|
-
if (m > k) {
|
|
1775
|
-
const UUtdU = _matmulMNK(U, UtdU, m, k, k, false, false);
|
|
1776
|
-
const dUperp = new Float64Array(m * k);
|
|
1777
|
-
for (let i = 0; i < m * k; i++) dUperp[i] = dU[i] - UUtdU[i];
|
|
1778
|
-
for (let i = 0; i < m; i++) {
|
|
1779
|
-
for (let j = 0; j < k; j++) {
|
|
1780
|
-
const sj = s[j];
|
|
1781
|
-
if (Math.abs(sj) > tolAbs) {
|
|
1782
|
-
dUperp[i * k + j] /= sj;
|
|
1783
|
-
} else {
|
|
1784
|
-
dUperp[i * k + j] = 0;
|
|
1785
|
-
}
|
|
1786
|
-
}
|
|
1787
|
-
}
|
|
1788
|
-
const corr = _matmulMNK(dUperp, Vt, m, k, n, false, false);
|
|
1789
|
-
for (let i = 0; i < dA.length; i++) dA[i] += corr[i];
|
|
1790
|
-
}
|
|
1791
|
-
if (n > k) {
|
|
1792
|
-
const VVtdV = _matmulMNK(V, VtdV, n, k, k, false, false);
|
|
1793
|
-
const dVperp = new Float64Array(n * k);
|
|
1794
|
-
for (let i = 0; i < n * k; i++) dVperp[i] = dV[i] - VVtdV[i];
|
|
1795
|
-
for (let i = 0; i < n; i++) {
|
|
1796
|
-
for (let j = 0; j < k; j++) {
|
|
1797
|
-
const sj = s[j];
|
|
1798
|
-
if (Math.abs(sj) > tolAbs) {
|
|
1799
|
-
dVperp[i * k + j] /= sj;
|
|
1800
|
-
} else {
|
|
1801
|
-
dVperp[i * k + j] = 0;
|
|
1802
|
-
}
|
|
1803
|
-
}
|
|
1804
|
-
}
|
|
1805
|
-
const corr = _matmulMNK(U, dVperp, m, k, n, false, true);
|
|
1806
|
-
for (let i = 0; i < dA.length; i++) dA[i] += corr[i];
|
|
1807
|
-
}
|
|
1808
|
-
return dA;
|
|
1122
|
+
const REL_TOL = 1e-10;
|
|
1123
|
+
let sMax = 0;
|
|
1124
|
+
for (let i = 0; i < k; i++) {
|
|
1125
|
+
const a = Math.abs(s[i]);
|
|
1126
|
+
if (a > sMax) sMax = a;
|
|
1127
|
+
}
|
|
1128
|
+
const tolAbs = REL_TOL * (sMax > 0 ? sMax : 1);
|
|
1129
|
+
const V = new Float64Array(n * k);
|
|
1130
|
+
for (let i = 0; i < n; i++) for (let j = 0; j < k; j++) V[i * k + j] = Vt[j * n + i];
|
|
1131
|
+
const dV = new Float64Array(n * k);
|
|
1132
|
+
for (let i = 0; i < n; i++) for (let j = 0; j < k; j++) dV[i * k + j] = dVt[j * n + i];
|
|
1133
|
+
const UtdU = _matmulMNK(U, dU, m, k, k, true, false);
|
|
1134
|
+
const VtdV = _matmulMNK(V, dV, n, k, k, true, false);
|
|
1135
|
+
const C = new Float64Array(k * k);
|
|
1136
|
+
for (let i = 0; i < k; i++) for (let j = 0; j < k; j++) if (i === j) C[i * k + j] = dS[i];
|
|
1137
|
+
else {
|
|
1138
|
+
const alpha = (UtdU[i * k + j] - UtdU[j * k + i]) * .5;
|
|
1139
|
+
const beta = (VtdV[i * k + j] - VtdV[j * k + i]) * .5;
|
|
1140
|
+
const denomDiff = s[j] - s[i];
|
|
1141
|
+
const denomSum = s[j] + s[i];
|
|
1142
|
+
let term = 0;
|
|
1143
|
+
if (Math.abs(denomDiff) > tolAbs) term += (alpha + beta) / denomDiff;
|
|
1144
|
+
if (Math.abs(denomSum) > tolAbs) term += (alpha - beta) / denomSum;
|
|
1145
|
+
C[i * k + j] = term;
|
|
1146
|
+
}
|
|
1147
|
+
const dA = _matmulMNK(_matmulMNK(U, C, m, k, k, false, false), Vt, m, k, n, false, false);
|
|
1148
|
+
if (m > k) {
|
|
1149
|
+
const UUtdU = _matmulMNK(U, UtdU, m, k, k, false, false);
|
|
1150
|
+
const dUperp = new Float64Array(m * k);
|
|
1151
|
+
for (let i = 0; i < m * k; i++) dUperp[i] = dU[i] - UUtdU[i];
|
|
1152
|
+
for (let i = 0; i < m; i++) for (let j = 0; j < k; j++) {
|
|
1153
|
+
const sj = s[j];
|
|
1154
|
+
if (Math.abs(sj) > tolAbs) dUperp[i * k + j] /= sj;
|
|
1155
|
+
else dUperp[i * k + j] = 0;
|
|
1156
|
+
}
|
|
1157
|
+
const corr = _matmulMNK(dUperp, Vt, m, k, n, false, false);
|
|
1158
|
+
for (let i = 0; i < dA.length; i++) dA[i] += corr[i];
|
|
1159
|
+
}
|
|
1160
|
+
if (n > k) {
|
|
1161
|
+
const VVtdV = _matmulMNK(V, VtdV, n, k, k, false, false);
|
|
1162
|
+
const dVperp = new Float64Array(n * k);
|
|
1163
|
+
for (let i = 0; i < n * k; i++) dVperp[i] = dV[i] - VVtdV[i];
|
|
1164
|
+
for (let i = 0; i < n; i++) for (let j = 0; j < k; j++) {
|
|
1165
|
+
const sj = s[j];
|
|
1166
|
+
if (Math.abs(sj) > tolAbs) dVperp[i * k + j] /= sj;
|
|
1167
|
+
else dVperp[i * k + j] = 0;
|
|
1168
|
+
}
|
|
1169
|
+
const corr = _matmulMNK(U, dVperp, m, k, n, false, true);
|
|
1170
|
+
for (let i = 0; i < dA.length; i++) dA[i] += corr[i];
|
|
1171
|
+
}
|
|
1172
|
+
return dA;
|
|
1809
1173
|
}
|
|
1810
1174
|
function _eigSymBackward(U, lam, dU, dLam, n) {
|
|
1811
|
-
|
|
1812
|
-
|
|
1813
|
-
|
|
1814
|
-
|
|
1815
|
-
|
|
1816
|
-
|
|
1817
|
-
|
|
1818
|
-
|
|
1819
|
-
|
|
1820
|
-
|
|
1821
|
-
|
|
1822
|
-
|
|
1823
|
-
|
|
1824
|
-
|
|
1825
|
-
|
|
1826
|
-
|
|
1827
|
-
|
|
1828
|
-
|
|
1829
|
-
|
|
1830
|
-
}
|
|
1831
|
-
}
|
|
1832
|
-
}
|
|
1833
|
-
}
|
|
1834
|
-
const UM = _matmulMNK(U, M, n, n, n, false, false);
|
|
1835
|
-
const dAraw = _matmulMNK(UM, U, n, n, n, false, true);
|
|
1836
|
-
const dA = new Float64Array(n * n);
|
|
1837
|
-
for (let i = 0; i < n; i++) {
|
|
1838
|
-
for (let j = 0; j < n; j++) {
|
|
1839
|
-
dA[i * n + j] = (dAraw[i * n + j] + dAraw[j * n + i]) * 0.5;
|
|
1840
|
-
}
|
|
1841
|
-
}
|
|
1842
|
-
return dA;
|
|
1175
|
+
const REL_TOL = 1e-10;
|
|
1176
|
+
let lamMax = 0;
|
|
1177
|
+
for (let i = 0; i < n; i++) {
|
|
1178
|
+
const a = Math.abs(lam[i]);
|
|
1179
|
+
if (a > lamMax) lamMax = a;
|
|
1180
|
+
}
|
|
1181
|
+
const tolAbs = REL_TOL * (lamMax > 0 ? lamMax : 1);
|
|
1182
|
+
const UtdU = _matmulMNK(U, dU, n, n, n, true, false);
|
|
1183
|
+
const M = new Float64Array(n * n);
|
|
1184
|
+
for (let i = 0; i < n; i++) for (let j = 0; j < n; j++) if (i === j) M[i * n + j] = dLam[i];
|
|
1185
|
+
else {
|
|
1186
|
+
const denom = lam[j] - lam[i];
|
|
1187
|
+
if (Math.abs(denom) > tolAbs) M[i * n + j] = UtdU[i * n + j] / denom;
|
|
1188
|
+
else M[i * n + j] = 0;
|
|
1189
|
+
}
|
|
1190
|
+
const dAraw = _matmulMNK(_matmulMNK(U, M, n, n, n, false, false), U, n, n, n, false, true);
|
|
1191
|
+
const dA = new Float64Array(n * n);
|
|
1192
|
+
for (let i = 0; i < n; i++) for (let j = 0; j < n; j++) dA[i * n + j] = (dAraw[i * n + j] + dAraw[j * n + i]) * .5;
|
|
1193
|
+
return dA;
|
|
1843
1194
|
}
|
|
1844
1195
|
function _eigGeneralBackward(V, vInv, lam, dV, dLam, n) {
|
|
1845
|
-
|
|
1846
|
-
|
|
1847
|
-
|
|
1848
|
-
|
|
1849
|
-
|
|
1850
|
-
|
|
1851
|
-
|
|
1852
|
-
|
|
1853
|
-
|
|
1854
|
-
|
|
1855
|
-
|
|
1856
|
-
|
|
1857
|
-
|
|
1858
|
-
|
|
1859
|
-
|
|
1860
|
-
|
|
1861
|
-
|
|
1862
|
-
|
|
1863
|
-
|
|
1864
|
-
|
|
1865
|
-
M[i * n + j] = dLam[i];
|
|
1866
|
-
} else {
|
|
1867
|
-
const denom = lam[j] - lam[i];
|
|
1868
|
-
if (Math.abs(denom) > tolAbs) {
|
|
1869
|
-
M[i * n + j] = VtdV[i * n + j] / denom;
|
|
1870
|
-
} else {
|
|
1871
|
-
M[i * n + j] = 0;
|
|
1872
|
-
}
|
|
1873
|
-
}
|
|
1874
|
-
}
|
|
1875
|
-
}
|
|
1876
|
-
const VinvT_M = _matmulMNK(vInv, M, n, n, n, true, false);
|
|
1877
|
-
const dA = _matmulMNK(VinvT_M, V, n, n, n, false, true);
|
|
1878
|
-
return dA;
|
|
1196
|
+
const REL_TOL = 1e-10;
|
|
1197
|
+
let lamMax = 0;
|
|
1198
|
+
for (let i = 0; i < n; i++) {
|
|
1199
|
+
const a = Math.abs(lam[i]);
|
|
1200
|
+
if (a > lamMax) lamMax = a;
|
|
1201
|
+
}
|
|
1202
|
+
const tolAbs = REL_TOL * (lamMax > 0 ? lamMax : 1);
|
|
1203
|
+
const VtdV = _matmulMNK(V, dV, n, n, n, true, false);
|
|
1204
|
+
const VtV = _matmulMNK(V, V, n, n, n, true, false);
|
|
1205
|
+
const diagVtdV = new Float64Array(n);
|
|
1206
|
+
for (let i = 0; i < n; i++) diagVtdV[i] = VtdV[i * n + i];
|
|
1207
|
+
for (let i = 0; i < n; i++) for (let j = 0; j < n; j++) VtdV[i * n + j] -= VtV[i * n + j] * diagVtdV[j];
|
|
1208
|
+
const M = new Float64Array(n * n);
|
|
1209
|
+
for (let i = 0; i < n; i++) for (let j = 0; j < n; j++) if (i === j) M[i * n + j] = dLam[i];
|
|
1210
|
+
else {
|
|
1211
|
+
const denom = lam[j] - lam[i];
|
|
1212
|
+
if (Math.abs(denom) > tolAbs) M[i * n + j] = VtdV[i * n + j] / denom;
|
|
1213
|
+
else M[i * n + j] = 0;
|
|
1214
|
+
}
|
|
1215
|
+
return _matmulMNK(_matmulMNK(vInv, M, n, n, n, true, false), V, n, n, n, false, true);
|
|
1879
1216
|
}
|
|
1880
1217
|
function _invertWithCondCheck(V, n, maxCond) {
|
|
1881
|
-
|
|
1882
|
-
|
|
1883
|
-
|
|
1884
|
-
|
|
1885
|
-
|
|
1886
|
-
|
|
1887
|
-
|
|
1888
|
-
|
|
1889
|
-
|
|
1890
|
-
|
|
1891
|
-
|
|
1892
|
-
|
|
1893
|
-
|
|
1894
|
-
|
|
1895
|
-
|
|
1896
|
-
|
|
1897
|
-
|
|
1898
|
-
|
|
1899
|
-
|
|
1900
|
-
|
|
1901
|
-
|
|
1902
|
-
|
|
1903
|
-
|
|
1904
|
-
|
|
1905
|
-
|
|
1906
|
-
|
|
1907
|
-
|
|
1908
|
-
|
|
1909
|
-
|
|
1910
|
-
|
|
1911
|
-
|
|
1912
|
-
|
|
1913
|
-
|
|
1914
|
-
|
|
1915
|
-
|
|
1916
|
-
|
|
1917
|
-
|
|
1918
|
-
|
|
1919
|
-
|
|
1920
|
-
|
|
1921
|
-
|
|
1922
|
-
|
|
1923
|
-
|
|
1924
|
-
|
|
1925
|
-
|
|
1926
|
-
|
|
1927
|
-
|
|
1928
|
-
|
|
1929
|
-
|
|
1930
|
-
|
|
1931
|
-
|
|
1932
|
-
|
|
1933
|
-
|
|
1934
|
-
|
|
1935
|
-
|
|
1936
|
-
|
|
1937
|
-
|
|
1938
|
-
|
|
1939
|
-
|
|
1940
|
-
|
|
1941
|
-
|
|
1942
|
-
|
|
1943
|
-
|
|
1944
|
-
|
|
1945
|
-
|
|
1946
|
-
|
|
1947
|
-
|
|
1948
|
-
|
|
1949
|
-
|
|
1950
|
-
if (rowV > normV) normV = rowV;
|
|
1951
|
-
if (rowI > normInv) normInv = rowI;
|
|
1952
|
-
}
|
|
1953
|
-
const cond = normV * normInv;
|
|
1954
|
-
if (!Number.isFinite(cond) || cond > maxCond) return null;
|
|
1955
|
-
return inv;
|
|
1218
|
+
const LU = new Float64Array(V);
|
|
1219
|
+
const perm = new Int32Array(n);
|
|
1220
|
+
for (let i = 0; i < n; i++) perm[i] = i;
|
|
1221
|
+
let pivotEpsScale = 0;
|
|
1222
|
+
for (let i = 0; i < n * n; i++) {
|
|
1223
|
+
const a = Math.abs(V[i]);
|
|
1224
|
+
if (a > pivotEpsScale) pivotEpsScale = a;
|
|
1225
|
+
}
|
|
1226
|
+
const singularEps = pivotEpsScale === 0 ? 0 : pivotEpsScale * 1e-300;
|
|
1227
|
+
for (let k = 0; k < n; k++) {
|
|
1228
|
+
let maxVal = Math.abs(LU[k * n + k]);
|
|
1229
|
+
let maxIdx = k;
|
|
1230
|
+
for (let i = k + 1; i < n; i++) {
|
|
1231
|
+
const v = Math.abs(LU[i * n + k]);
|
|
1232
|
+
if (v > maxVal) {
|
|
1233
|
+
maxVal = v;
|
|
1234
|
+
maxIdx = i;
|
|
1235
|
+
}
|
|
1236
|
+
}
|
|
1237
|
+
if (maxIdx !== k) {
|
|
1238
|
+
for (let j = 0; j < n; j++) {
|
|
1239
|
+
const tmp = LU[k * n + j];
|
|
1240
|
+
LU[k * n + j] = LU[maxIdx * n + j];
|
|
1241
|
+
LU[maxIdx * n + j] = tmp;
|
|
1242
|
+
}
|
|
1243
|
+
const t = perm[k];
|
|
1244
|
+
perm[k] = perm[maxIdx];
|
|
1245
|
+
perm[maxIdx] = t;
|
|
1246
|
+
}
|
|
1247
|
+
const pivot = LU[k * n + k];
|
|
1248
|
+
if (!Number.isFinite(pivot) || Math.abs(pivot) <= singularEps) return null;
|
|
1249
|
+
for (let i = k + 1; i < n; i++) {
|
|
1250
|
+
LU[i * n + k] /= pivot;
|
|
1251
|
+
const lik = LU[i * n + k];
|
|
1252
|
+
for (let j = k + 1; j < n; j++) LU[i * n + j] -= lik * LU[k * n + j];
|
|
1253
|
+
}
|
|
1254
|
+
}
|
|
1255
|
+
const inv = new Float64Array(n * n);
|
|
1256
|
+
const y = new Float64Array(n);
|
|
1257
|
+
for (let col = 0; col < n; col++) {
|
|
1258
|
+
for (let i = 0; i < n; i++) y[i] = perm[i] === col ? 1 : 0;
|
|
1259
|
+
for (let i = 0; i < n; i++) {
|
|
1260
|
+
let s = y[i];
|
|
1261
|
+
for (let j = 0; j < i; j++) s -= LU[i * n + j] * y[j];
|
|
1262
|
+
y[i] = s;
|
|
1263
|
+
}
|
|
1264
|
+
for (let i = n - 1; i >= 0; i--) {
|
|
1265
|
+
let s = y[i];
|
|
1266
|
+
for (let j = i + 1; j < n; j++) s -= LU[i * n + j] * inv[j * n + col];
|
|
1267
|
+
inv[i * n + col] = s / LU[i * n + i];
|
|
1268
|
+
}
|
|
1269
|
+
}
|
|
1270
|
+
let normV = 0;
|
|
1271
|
+
let normInv = 0;
|
|
1272
|
+
for (let i = 0; i < n; i++) {
|
|
1273
|
+
let rowV = 0;
|
|
1274
|
+
let rowI = 0;
|
|
1275
|
+
for (let j = 0; j < n; j++) {
|
|
1276
|
+
rowV += Math.abs(V[i * n + j]);
|
|
1277
|
+
const inv_ij = inv[i * n + j];
|
|
1278
|
+
if (!Number.isFinite(inv_ij)) return null;
|
|
1279
|
+
rowI += Math.abs(inv_ij);
|
|
1280
|
+
}
|
|
1281
|
+
if (rowV > normV) normV = rowV;
|
|
1282
|
+
if (rowI > normInv) normInv = rowI;
|
|
1283
|
+
}
|
|
1284
|
+
const cond = normV * normInv;
|
|
1285
|
+
if (!Number.isFinite(cond) || cond > maxCond) return null;
|
|
1286
|
+
return inv;
|
|
1956
1287
|
}
|
|
1957
1288
|
function _matmulMNK(A, B, M, K, N, aT, bT) {
|
|
1958
|
-
|
|
1959
|
-
|
|
1960
|
-
|
|
1961
|
-
|
|
1962
|
-
|
|
1963
|
-
|
|
1964
|
-
|
|
1965
|
-
|
|
1966
|
-
|
|
1967
|
-
|
|
1968
|
-
|
|
1969
|
-
}
|
|
1970
|
-
return C;
|
|
1289
|
+
const C = new Float64Array(M * N);
|
|
1290
|
+
for (let i = 0; i < M; i++) for (let j = 0; j < N; j++) {
|
|
1291
|
+
let sum = 0;
|
|
1292
|
+
for (let q = 0; q < K; q++) {
|
|
1293
|
+
const aVal = aT ? A[q * M + i] : A[i * K + q];
|
|
1294
|
+
const bVal = bT ? B[j * K + q] : B[q * N + j];
|
|
1295
|
+
sum += aVal * bVal;
|
|
1296
|
+
}
|
|
1297
|
+
C[i * N + j] = sum;
|
|
1298
|
+
}
|
|
1299
|
+
return C;
|
|
1971
1300
|
}
|
|
1972
1301
|
function _resolveAxes(axis, rank) {
|
|
1973
|
-
|
|
1974
|
-
|
|
1975
|
-
|
|
1976
|
-
if (typeof axis === "number") {
|
|
1977
|
-
return [axis];
|
|
1978
|
-
}
|
|
1979
|
-
return [...axis].sort((a, b) => a - b);
|
|
1302
|
+
if (axis === void 0) return Array.from({ length: rank }, (_, i) => i);
|
|
1303
|
+
if (typeof axis === "number") return [axis];
|
|
1304
|
+
return [...axis].sort((a, b) => a - b);
|
|
1980
1305
|
}
|
|
1981
1306
|
function _scatterToOriginalAxes(tensor, targetLabels) {
|
|
1982
|
-
|
|
1983
|
-
|
|
1984
|
-
|
|
1985
|
-
|
|
1986
|
-
|
|
1987
|
-
|
|
1988
|
-
|
|
1989
|
-
|
|
1990
|
-
|
|
1991
|
-
|
|
1992
|
-
|
|
1993
|
-
}
|
|
1994
|
-
const isIdentity = perm.every((j, i) => j === i);
|
|
1995
|
-
if (isIdentity) {
|
|
1996
|
-
return new Float64Array(tensor.data);
|
|
1997
|
-
}
|
|
1998
|
-
const permuted = tensor.transpose(perm);
|
|
1999
|
-
return new Float64Array(permuted.data);
|
|
1307
|
+
const currentLabels = tensor.axisLabels;
|
|
1308
|
+
const rank = targetLabels.length;
|
|
1309
|
+
const perm = new Array(rank);
|
|
1310
|
+
for (let i = 0; i < rank; i++) {
|
|
1311
|
+
const j = currentLabels.findIndex((lbl) => lbl.matches(targetLabels[i]));
|
|
1312
|
+
if (j === -1) throw new Error(`_scatterToOriginalAxes: target label ${targetLabels[i].toString()} not found in tensor axisLabels`);
|
|
1313
|
+
perm[i] = j;
|
|
1314
|
+
}
|
|
1315
|
+
if (perm.every((j, i) => j === i)) return new Float64Array(tensor.data);
|
|
1316
|
+
const permuted = tensor.transpose(perm);
|
|
1317
|
+
return new Float64Array(permuted.data);
|
|
2000
1318
|
}
|
|
2001
1319
|
function _batchedMatmul(aData, bData, aShape, bShape) {
|
|
2002
|
-
|
|
2003
|
-
|
|
2004
|
-
|
|
2005
|
-
|
|
2006
|
-
|
|
2007
|
-
|
|
2008
|
-
|
|
2009
|
-
|
|
2010
|
-
|
|
2011
|
-
|
|
2012
|
-
|
|
2013
|
-
|
|
2014
|
-
|
|
2015
|
-
|
|
2016
|
-
|
|
2017
|
-
|
|
2018
|
-
|
|
2019
|
-
|
|
2020
|
-
|
|
2021
|
-
}
|
|
2022
|
-
}
|
|
2023
|
-
}
|
|
2024
|
-
return yData;
|
|
1320
|
+
const rank = aShape.length;
|
|
1321
|
+
const batchRank = rank - 2;
|
|
1322
|
+
const m = aShape[rank - 2];
|
|
1323
|
+
const k = aShape[rank - 1];
|
|
1324
|
+
const n = bShape[rank - 1];
|
|
1325
|
+
let batchSize = 1;
|
|
1326
|
+
for (let b = 0; b < batchRank; b++) batchSize *= aShape[b];
|
|
1327
|
+
const yData = new Float64Array(batchSize * m * n);
|
|
1328
|
+
for (let batch = 0; batch < batchSize; batch++) {
|
|
1329
|
+
const aOff = batch * m * k;
|
|
1330
|
+
const bOff = batch * k * n;
|
|
1331
|
+
const yOff = batch * m * n;
|
|
1332
|
+
for (let i = 0; i < m; i++) for (let j = 0; j < n; j++) {
|
|
1333
|
+
let sum = 0;
|
|
1334
|
+
for (let q = 0; q < k; q++) sum += aData[aOff + i * k + q] * bData[bOff + q * n + j];
|
|
1335
|
+
yData[yOff + i * n + j] = sum;
|
|
1336
|
+
}
|
|
1337
|
+
}
|
|
1338
|
+
return yData;
|
|
2025
1339
|
}
|
|
2026
1340
|
function _batchedMatmulGradA(dYData, bData, yShape, bShape) {
|
|
2027
|
-
|
|
2028
|
-
|
|
2029
|
-
|
|
2030
|
-
|
|
2031
|
-
|
|
2032
|
-
|
|
2033
|
-
|
|
2034
|
-
|
|
2035
|
-
|
|
2036
|
-
|
|
2037
|
-
|
|
2038
|
-
|
|
2039
|
-
|
|
2040
|
-
|
|
2041
|
-
|
|
2042
|
-
|
|
2043
|
-
|
|
2044
|
-
|
|
2045
|
-
|
|
2046
|
-
}
|
|
2047
|
-
}
|
|
2048
|
-
}
|
|
2049
|
-
return dAData;
|
|
1341
|
+
const rank = yShape.length;
|
|
1342
|
+
const batchRank = rank - 2;
|
|
1343
|
+
const m = yShape[rank - 2];
|
|
1344
|
+
const n = yShape[rank - 1];
|
|
1345
|
+
const k = bShape[rank - 2];
|
|
1346
|
+
let batchSize = 1;
|
|
1347
|
+
for (let b = 0; b < batchRank; b++) batchSize *= yShape[b];
|
|
1348
|
+
const dAData = new Float64Array(batchSize * m * k);
|
|
1349
|
+
for (let batch = 0; batch < batchSize; batch++) {
|
|
1350
|
+
const dYOff = batch * m * n;
|
|
1351
|
+
const bOff = batch * k * n;
|
|
1352
|
+
const dAOff = batch * m * k;
|
|
1353
|
+
for (let i = 0; i < m; i++) for (let q = 0; q < k; q++) {
|
|
1354
|
+
let sum = 0;
|
|
1355
|
+
for (let j = 0; j < n; j++) sum += dYData[dYOff + i * n + j] * bData[bOff + q * n + j];
|
|
1356
|
+
dAData[dAOff + i * k + q] = sum;
|
|
1357
|
+
}
|
|
1358
|
+
}
|
|
1359
|
+
return dAData;
|
|
2050
1360
|
}
|
|
2051
1361
|
function _batchedMatmulGradB(aData, dYData, aShape, yShape) {
|
|
2052
|
-
|
|
2053
|
-
|
|
2054
|
-
|
|
2055
|
-
|
|
2056
|
-
|
|
2057
|
-
|
|
2058
|
-
|
|
2059
|
-
|
|
2060
|
-
|
|
2061
|
-
|
|
2062
|
-
|
|
2063
|
-
|
|
2064
|
-
|
|
2065
|
-
|
|
2066
|
-
|
|
2067
|
-
|
|
2068
|
-
|
|
2069
|
-
|
|
2070
|
-
|
|
2071
|
-
}
|
|
2072
|
-
}
|
|
2073
|
-
}
|
|
2074
|
-
return dBData;
|
|
1362
|
+
const rank = aShape.length;
|
|
1363
|
+
const batchRank = rank - 2;
|
|
1364
|
+
const m = aShape[rank - 2];
|
|
1365
|
+
const k = aShape[rank - 1];
|
|
1366
|
+
const n = yShape[rank - 1];
|
|
1367
|
+
let batchSize = 1;
|
|
1368
|
+
for (let b = 0; b < batchRank; b++) batchSize *= aShape[b];
|
|
1369
|
+
const dBData = new Float64Array(batchSize * k * n);
|
|
1370
|
+
for (let batch = 0; batch < batchSize; batch++) {
|
|
1371
|
+
const aOff = batch * m * k;
|
|
1372
|
+
const dYOff = batch * m * n;
|
|
1373
|
+
const dBOff = batch * k * n;
|
|
1374
|
+
for (let q = 0; q < k; q++) for (let j = 0; j < n; j++) {
|
|
1375
|
+
let sum = 0;
|
|
1376
|
+
for (let i = 0; i < m; i++) sum += aData[aOff + i * k + q] * dYData[dYOff + i * n + j];
|
|
1377
|
+
dBData[dBOff + q * n + j] = sum;
|
|
1378
|
+
}
|
|
1379
|
+
}
|
|
1380
|
+
return dBData;
|
|
2075
1381
|
}
|
|
2076
|
-
|
|
2077
|
-
|
|
2078
|
-
import { Tensor as Tensor4 } from "@danielsimonjr/mathts-tensor";
|
|
1382
|
+
//#endregion
|
|
1383
|
+
//#region src/reverse-grad.ts
|
|
2079
1384
|
function reverseGrad(fn, x, cotangent) {
|
|
2080
|
-
|
|
2081
|
-
|
|
2082
|
-
|
|
2083
|
-
|
|
2084
|
-
|
|
2085
|
-
|
|
2086
|
-
|
|
2087
|
-
|
|
2088
|
-
|
|
2089
|
-
|
|
2090
|
-
|
|
2091
|
-
|
|
2092
|
-
|
|
2093
|
-
|
|
2094
|
-
|
|
2095
|
-
|
|
2096
|
-
|
|
2097
|
-
|
|
2098
|
-
|
|
2099
|
-
|
|
2100
|
-
ct = cotangent;
|
|
2101
|
-
}
|
|
2102
|
-
tape.backward(yTaped.id, new Float64Array(ct.data));
|
|
2103
|
-
const xGradSlot = tape.getInputGrad(xTaped.id);
|
|
2104
|
-
return {
|
|
2105
|
-
value,
|
|
2106
|
-
gradient: new Tensor4(x.shape, new Float64Array(xGradSlot))
|
|
2107
|
-
};
|
|
1385
|
+
const tape = new Tape();
|
|
1386
|
+
const xTaped = TapedTensor.fromTensorAsInput(x, tape);
|
|
1387
|
+
const yRaw = fn(xTaped);
|
|
1388
|
+
if (!(yRaw instanceof TapedTensor)) throw new Error("reverseGrad: fn must be AD-traceable — its return must propagate through TapedTensor arithmetic (use add/sub/mul/scale on the argument). A fresh Tensor return loses the tape and silently corrupts the gradient.");
|
|
1389
|
+
const yTaped = yRaw;
|
|
1390
|
+
const value = yTaped.toPrimalTensor();
|
|
1391
|
+
let ct;
|
|
1392
|
+
if (cotangent === void 0) {
|
|
1393
|
+
const data = new Float64Array(value.data.length).fill(1);
|
|
1394
|
+
ct = new Tensor(value.shape, data);
|
|
1395
|
+
} else {
|
|
1396
|
+
if (cotangent.shape.length !== value.shape.length || !cotangent.shape.every((v, i) => v === value.shape[i])) throw new Error(`reverseGrad: cotangent shape [${cotangent.shape}] != value shape [${value.shape}]`);
|
|
1397
|
+
ct = cotangent;
|
|
1398
|
+
}
|
|
1399
|
+
tape.backward(yTaped.id, new Float64Array(ct.data));
|
|
1400
|
+
const xGradSlot = tape.getInputGrad(xTaped.id);
|
|
1401
|
+
return {
|
|
1402
|
+
value,
|
|
1403
|
+
gradient: new Tensor(x.shape, new Float64Array(xGradSlot))
|
|
1404
|
+
};
|
|
2108
1405
|
}
|
|
2109
|
-
|
|
2110
|
-
|
|
1406
|
+
//#endregion
|
|
1407
|
+
//#region src/grad.ts
|
|
2111
1408
|
function toFloat64(x) {
|
|
2112
|
-
|
|
2113
|
-
|
|
1409
|
+
if (typeof x === "number") return Float64Array.of(x);
|
|
1410
|
+
return x instanceof Float64Array ? new Float64Array(x) : Float64Array.from(x);
|
|
2114
1411
|
}
|
|
2115
1412
|
function valueAndGrad(fn, x) {
|
|
2116
|
-
|
|
2117
|
-
|
|
2118
|
-
|
|
2119
|
-
|
|
2120
|
-
|
|
2121
|
-
|
|
2122
|
-
|
|
2123
|
-
|
|
2124
|
-
|
|
2125
|
-
|
|
2126
|
-
tape.backward(out.id, Float64Array.of(1));
|
|
2127
|
-
return { value: out.primal[0], grad: new Float64Array(tape.getInputGrad(id)) };
|
|
1413
|
+
const xs = toFloat64(x);
|
|
1414
|
+
const tape = new Tape();
|
|
1415
|
+
const { id } = tape.allocate(xs.length);
|
|
1416
|
+
const out = fn(new TapedTensor([xs.length], xs, tape, id));
|
|
1417
|
+
if (out.primal.length !== 1) throw new Error(`valueAndGrad: fn must return a scalar (length 1); got length ${out.primal.length}. Reduce with .sum()/.mean(), or use jacobian() for vector outputs.`);
|
|
1418
|
+
tape.backward(out.id, Float64Array.of(1));
|
|
1419
|
+
return {
|
|
1420
|
+
value: out.primal[0],
|
|
1421
|
+
grad: new Float64Array(tape.getInputGrad(id))
|
|
1422
|
+
};
|
|
2128
1423
|
}
|
|
2129
1424
|
function grad(fn) {
|
|
2130
|
-
|
|
1425
|
+
return (x) => valueAndGrad(fn, x).grad;
|
|
2131
1426
|
}
|
|
2132
1427
|
function derivative(fn, x0) {
|
|
2133
|
-
|
|
1428
|
+
return valueAndGrad(fn, x0).grad[0];
|
|
2134
1429
|
}
|
|
2135
1430
|
function jacobian(fn, x) {
|
|
2136
|
-
|
|
2137
|
-
|
|
2138
|
-
|
|
2139
|
-
|
|
2140
|
-
|
|
2141
|
-
|
|
2142
|
-
|
|
2143
|
-
|
|
2144
|
-
|
|
2145
|
-
|
|
2146
|
-
|
|
2147
|
-
|
|
2148
|
-
|
|
2149
|
-
|
|
2150
|
-
|
|
2151
|
-
return rows;
|
|
1431
|
+
const xs = toFloat64(x);
|
|
1432
|
+
const probeTape = new Tape();
|
|
1433
|
+
const { id: probeId } = probeTape.allocate(xs.length);
|
|
1434
|
+
const m = fn(new TapedTensor([xs.length], new Float64Array(xs), probeTape, probeId)).primal.length;
|
|
1435
|
+
const rows = [];
|
|
1436
|
+
for (let r = 0; r < m; r++) {
|
|
1437
|
+
const tape = new Tape();
|
|
1438
|
+
const { id } = tape.allocate(xs.length);
|
|
1439
|
+
const out = fn(new TapedTensor([xs.length], new Float64Array(xs), tape, id));
|
|
1440
|
+
const seed = new Float64Array(m);
|
|
1441
|
+
seed[r] = 1;
|
|
1442
|
+
tape.backward(out.id, seed);
|
|
1443
|
+
rows.push(Array.from(tape.getInputGrad(id)));
|
|
1444
|
+
}
|
|
1445
|
+
return rows;
|
|
2152
1446
|
}
|
|
2153
|
-
|
|
2154
|
-
|
|
2155
|
-
Tape,
|
|
2156
|
-
TapedTensor,
|
|
2157
|
-
derivative,
|
|
2158
|
-
forwardGrad,
|
|
2159
|
-
grad,
|
|
2160
|
-
jacobian,
|
|
2161
|
-
reverseGrad,
|
|
2162
|
-
valueAndGrad
|
|
2163
|
-
};
|
|
1447
|
+
//#endregion
|
|
1448
|
+
export { DualTensor, Tape, TapedTensor, derivative, forwardGrad, grad, jacobian, reverseGrad, valueAndGrad };
|