tilework 0.18.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE +21 -0
- data/lib/tilework/backend/cpu/dsl_codegen.rb +509 -0
- data/lib/tilework/backend/cpu/ops_codegen.rb +451 -0
- data/lib/tilework/backend/cpu.rb +156 -0
- data/lib/tilework/buffer.rb +753 -0
- data/lib/tilework/capabilities.rb +183 -0
- data/lib/tilework/compat/cuda.h +66 -0
- data/lib/tilework/compat/hip.h +35 -0
- data/lib/tilework/compat/metal.h +45 -0
- data/lib/tilework/conformance.rb +2296 -0
- data/lib/tilework/core_ext.rb +21 -0
- data/lib/tilework/dialect.rb +67 -0
- data/lib/tilework/dsl.rb +722 -0
- data/lib/tilework/dtype/bfloat.rb +31 -0
- data/lib/tilework/dtype/cast.rb +88 -0
- data/lib/tilework/dtype/half.rb +98 -0
- data/lib/tilework/graph/builder.rb +1073 -0
- data/lib/tilework/graph/fusion.rb +520 -0
- data/lib/tilework/graph/grad.rb +443 -0
- data/lib/tilework/graph/high_level.rb +266 -0
- data/lib/tilework/graph/kernels.rb +403 -0
- data/lib/tilework/graph/node.rb +27 -0
- data/lib/tilework/graph/op.rb +290 -0
- data/lib/tilework/graph/optim.rb +183 -0
- data/lib/tilework/graph/plan.rb +1086 -0
- data/lib/tilework/graph/pool.rb +178 -0
- data/lib/tilework/graph/region.rb +210 -0
- data/lib/tilework/graph/residency.rb +449 -0
- data/lib/tilework/graph/state.rb +64 -0
- data/lib/tilework/graph/tensor.rb +124 -0
- data/lib/tilework/graph.rb +213 -0
- data/lib/tilework/kernel.rb +163 -0
- data/lib/tilework/math/conv.rb +270 -0
- data/lib/tilework/math/fft.rb +172 -0
- data/lib/tilework/math/nn.rb +1103 -0
- data/lib/tilework/math/philox.rb +194 -0
- data/lib/tilework/math/random.rb +232 -0
- data/lib/tilework/math/tensor_core.rb +269 -0
- data/lib/tilework/ops.rb +682 -0
- data/lib/tilework/runtime/offload.rb +498 -0
- data/lib/tilework/runtime/submission.rb +163 -0
- data/lib/tilework/runtime/telemetry.rb +112 -0
- data/lib/tilework/scope.rb +33 -0
- data/lib/tilework/version.rb +5 -0
- data/lib/tilework.rb +345 -0
- metadata +92 -0
|
@@ -0,0 +1,443 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Tilework
|
|
4
|
+
module Graph
|
|
5
|
+
# The contract for reverse mode (#22): a vjp table with **one row per op**,
|
|
6
|
+
# keyed by the same names as `Op::REGISTRY`.
|
|
7
|
+
#
|
|
8
|
+
# It is a table for the same reason as Op::REGISTRY: adding an op and
|
|
9
|
+
# forgetting its gradient should fail in `Grad.fetch`, not silently produce
|
|
10
|
+
# a wrong plan. Rows declaring "not differentiable" live in the table too
|
|
11
|
+
# (`opaque`'s contents are unknown, etc.), so failures carry a reason.
|
|
12
|
+
#
|
|
13
|
+
# **Rows are written in the builder's primitives only.** Backward nodes are
|
|
14
|
+
# ordinary nodes, so fusion, placement and liveness all apply, and their
|
|
15
|
+
# gradients can be differentiated again (docs/autograd.md).
|
|
16
|
+
#
|
|
17
|
+
# Grad.define(:unary) { |g, node, dy, inputs| [...] }
|
|
18
|
+
#
|
|
19
|
+
# g the builder (where new nodes are recorded)
|
|
20
|
+
# node the forward node
|
|
21
|
+
# dy the cotangent arriving at this node's output (Tensor)
|
|
22
|
+
# inputs node.inputs wrapped as Tensors
|
|
23
|
+
#
|
|
24
|
+
# Returns one cotangent per input. Inputs no gradient flows into (indices,
|
|
25
|
+
# masks, comparison outputs) get nil.
|
|
26
|
+
module Grad
|
|
27
|
+
REGISTRY = {}
|
|
28
|
+
|
|
29
|
+
module_function
|
|
30
|
+
|
|
31
|
+
def define(name, &vjp)
|
|
32
|
+
raise Error, "the vjp of #{name.inspect} is already defined" if REGISTRY.key?(name)
|
|
33
|
+
raise ArgumentError, "define needs a block" if vjp.nil?
|
|
34
|
+
|
|
35
|
+
REGISTRY[name] = vjp
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
def fetch(name)
|
|
39
|
+
REGISTRY.fetch(name) do
|
|
40
|
+
raise Error, "no vjp is defined for the graph op #{name.inspect}"
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
def defined?(name) = REGISTRY.key?(name)
|
|
45
|
+
def names = REGISTRY.keys
|
|
46
|
+
|
|
47
|
+
def apply(builder, node, dy, inputs) = fetch(node.op).call(builder, node, dy, inputs)
|
|
48
|
+
|
|
49
|
+
# --- Hand-written vjps for high-level ops --------------------------------
|
|
50
|
+
#
|
|
51
|
+
# The default is "decompose, then differentiate", which gives correct
|
|
52
|
+
# gradients, but expressions with row-wise reductions cost more round
|
|
53
|
+
# trips decomposed, so **only those faster written as an expression** are
|
|
54
|
+
# added here. Functions in this table stay :nn nodes even in a
|
|
55
|
+
# differentiable graph (builder.rb #nn).
|
|
56
|
+
#
|
|
57
|
+
# Adding one follows a single procedure: **the decomposition is the
|
|
58
|
+
# oracle**. With `Graph.lower_high_level = true` its gradient comes from
|
|
59
|
+
# primitive vjps automatically and can be compared with the hand-written
|
|
60
|
+
# row (conformance's graph_hand_written_vjps_match_the_decomposed_ones).
|
|
61
|
+
# Finite differences alone cannot tell whether the fast path or the
|
|
62
|
+
# decomposition is wrong.
|
|
63
|
+
NN_REGISTRY = {}
|
|
64
|
+
|
|
65
|
+
def define_nn(fn, &vjp)
|
|
66
|
+
raise Error, "the vjp of NN.#{fn} is already defined" if NN_REGISTRY.key?(fn)
|
|
67
|
+
|
|
68
|
+
NN_REGISTRY[fn] = vjp
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
def nn_defined?(fn) = NN_REGISTRY.key?(fn)
|
|
72
|
+
def nn_names = NN_REGISTRY.keys
|
|
73
|
+
|
|
74
|
+
# Splits a :nn node's inputs into positional and keyword arguments.
|
|
75
|
+
def nn_arguments(node, inputs)
|
|
76
|
+
arity = node.attrs[:arity]
|
|
77
|
+
[inputs.first(arity), node.attrs[:keys].zip(inputs.drop(arity)).to_h]
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
# Shared by functions with row-wise reductions: the last dim is columns, the rest rows.
|
|
81
|
+
def row_count(shape) = shape[0..-2].inject(1, :*)
|
|
82
|
+
|
|
83
|
+
# The gamma / beta gradient folds over all rows, leaving only columns.
|
|
84
|
+
def fold_rows(tensor)
|
|
85
|
+
flat = tensor.reshape(row_count(tensor.shape), tensor.shape[-1])
|
|
86
|
+
flat.sum(axis: 0)
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
# --- Helpers for the rows ----------------------------------------------
|
|
90
|
+
|
|
91
|
+
# Folds a cotangent of the broadcast shape back to the original shape.
|
|
92
|
+
# The rule is broadcasting reversed (NumPy right alignment): added leading
|
|
93
|
+
# axes are folded away, and axes that were 1 are folded with keepdims.
|
|
94
|
+
def unbroadcast(dy, shape)
|
|
95
|
+
extra = dy.shape.length - shape.length
|
|
96
|
+
extra.times { dy = dy.sum(axis: 0) }
|
|
97
|
+
shape.each_with_index do |dim, axis|
|
|
98
|
+
dy = dy.sum(axis: axis, keepdims: true) if dim == 1 && dy.shape[axis] != 1
|
|
99
|
+
end
|
|
100
|
+
dy.shape == shape ? dy : dy.reshape(*shape)
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
# The inverse of a reduction: spreads values back along the folded axis.
|
|
104
|
+
def spread(dy, node, shape)
|
|
105
|
+
axis = node.attrs[:axis]
|
|
106
|
+
return dy.broadcast_to(shape) if axis.nil?
|
|
107
|
+
|
|
108
|
+
# If it was folded with keepdims: false, restore the axis before broadcasting.
|
|
109
|
+
dy = dy.reshape(*shape.each_with_index.map { |dim, i| i == axis ? 1 : dim }) unless node.attrs[:keepdims]
|
|
110
|
+
dy.broadcast_to(shape)
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
# The inverse of padding (a crop). `slice` takes only the outermost dim,
|
|
114
|
+
# so cropping an arbitrary axis builds an index table and pulls it with
|
|
115
|
+
# `gather` — the same trick conv2d's decomposition uses for its im2col
|
|
116
|
+
# table, with no constants brought in from the host.
|
|
117
|
+
def crop(builder, dy, before, shape)
|
|
118
|
+
total = shape.inject(1, :*)
|
|
119
|
+
padded = suffix_products(dy.shape)
|
|
120
|
+
index = builder.iota([total], axis: 0)
|
|
121
|
+
flat = nil
|
|
122
|
+
suffix_products(shape).each_with_index do |stride, axis|
|
|
123
|
+
along = index / stride
|
|
124
|
+
coord = along - ((along / shape[axis]) * shape[axis]) # along % shape[axis]
|
|
125
|
+
term = (coord + before[axis]) * padded[axis]
|
|
126
|
+
flat = flat.nil? ? term : flat + term
|
|
127
|
+
end
|
|
128
|
+
builder.gather(dy.reshape(dy.length), flat).reshape(*shape)
|
|
129
|
+
end
|
|
130
|
+
|
|
131
|
+
# Per axis, how far the flat index advances for one step.
|
|
132
|
+
def suffix_products(shape)
|
|
133
|
+
shape.each_index.map { |i| shape[(i + 1)..].inject(1, :*) }
|
|
134
|
+
end
|
|
135
|
+
|
|
136
|
+
# Non-differentiable ops are collected here with a reason.
|
|
137
|
+
def undefined!(what, reason)
|
|
138
|
+
raise Error, "#{what} has no gradient: #{reason}"
|
|
139
|
+
end
|
|
140
|
+
end
|
|
141
|
+
|
|
142
|
+
# --- The table -----------------------------------------------------------
|
|
143
|
+
#
|
|
144
|
+
# Leaves. They have no inputs (or pass no gradient), so backward stops here.
|
|
145
|
+
|
|
146
|
+
Grad.define(:input) { |_g, _node, _dy, _inputs| [] }
|
|
147
|
+
Grad.define(:const) { |_g, _node, _dy, _inputs| [] }
|
|
148
|
+
Grad.define(:iota) { |_g, _node, _dy, _inputs| [] }
|
|
149
|
+
Grad.define(:random) { |_g, _node, _dy, _inputs| [] }
|
|
150
|
+
Grad.define(:state_init) { |_g, _node, _dy, _inputs| [] }
|
|
151
|
+
|
|
152
|
+
# Reads are treated as leaves: the gradient is taken with respect to the
|
|
153
|
+
# read's result (pass the g.read(ref) Tensor in wrt:). Folding back into the
|
|
154
|
+
# state itself involves versions and regions, so it is not done here.
|
|
155
|
+
Grad.define(:state_read) { |_g, _node, _dy, _inputs| [nil] }
|
|
156
|
+
|
|
157
|
+
# Index reductions and comparisons output integers, so no gradient flows.
|
|
158
|
+
Grad.define(:arg_reduce) { |_g, _node, _dy, _inputs| [nil] }
|
|
159
|
+
|
|
160
|
+
# --- Elementwise ---------------------------------------------------------
|
|
161
|
+
|
|
162
|
+
# Binary ops. With a scalar there is one input, and attrs hold scalar and
|
|
163
|
+
# reversed (reversed means swapped operands, as in `2.0 - x`).
|
|
164
|
+
Grad.define(:binary) do |_g, node, dy, inputs|
|
|
165
|
+
# Comparisons only return 0/1, so no gradient flows.
|
|
166
|
+
next Array.new(node.inputs.length) if node.attrs[:kind] == :compare
|
|
167
|
+
|
|
168
|
+
op = node.attrs[:op]
|
|
169
|
+
if node.attrs.key?(:scalar)
|
|
170
|
+
x = inputs[0]
|
|
171
|
+
s = node.attrs[:scalar]
|
|
172
|
+
rev = node.attrs[:reversed]
|
|
173
|
+
[case op
|
|
174
|
+
when :+ then dy
|
|
175
|
+
when :- then rev ? -1.0 * dy : dy # rev: s - x
|
|
176
|
+
when :* then dy * s
|
|
177
|
+
when :/ then rev ? -1.0 * dy * s / (x * x) : dy / s
|
|
178
|
+
end]
|
|
179
|
+
else
|
|
180
|
+
a, b = inputs
|
|
181
|
+
case op
|
|
182
|
+
when :+ then [dy, dy]
|
|
183
|
+
when :- then [dy, -1.0 * dy]
|
|
184
|
+
when :* then [dy * b, dy * a]
|
|
185
|
+
when :/ then [dy / b, -1.0 * dy * a / (b * b)]
|
|
186
|
+
end
|
|
187
|
+
end
|
|
188
|
+
end
|
|
189
|
+
|
|
190
|
+
# Unary math functions. Those that can use their output do (exp / tanh),
|
|
191
|
+
# which saves keeping one more value.
|
|
192
|
+
Grad.define(:unary) do |g, node, dy, inputs|
|
|
193
|
+
x = inputs[0]
|
|
194
|
+
y = Tensor.new(g, node)
|
|
195
|
+
[case node.attrs[:fn]
|
|
196
|
+
when :exp then dy * y
|
|
197
|
+
when :log then dy / x
|
|
198
|
+
when :sqrt then dy / (2.0 * y)
|
|
199
|
+
when :sin then dy * x.cos
|
|
200
|
+
when :cos then -1.0 * dy * x.sin
|
|
201
|
+
when :tanh then dy * (1.0 - (y * y))
|
|
202
|
+
when :abs then dy * g.where(x >= 0.0, 1.0, -1.0)
|
|
203
|
+
# Step functions: the derivative is 0 almost everywhere, so return that.
|
|
204
|
+
when :floor, :ceil, :round then g.fill(x.shape, 0.0, dtype: dy.dtype)
|
|
205
|
+
else Grad.undefined!(node.attrs[:fn], "no derivative is registered for it")
|
|
206
|
+
end]
|
|
207
|
+
end
|
|
208
|
+
|
|
209
|
+
# It only selects, so the cotangent is routed by the same condition; none
|
|
210
|
+
# flows into the condition itself.
|
|
211
|
+
Grad.define(:where) do |g, node, dy, inputs|
|
|
212
|
+
zero = 0.0
|
|
213
|
+
grads = [nil]
|
|
214
|
+
grads << g.where(inputs[0], dy, zero) unless node.attrs.key?(:a_scalar)
|
|
215
|
+
grads << g.where(inputs[0], zero, dy) unless node.attrs.key?(:b_scalar)
|
|
216
|
+
grads
|
|
217
|
+
end
|
|
218
|
+
|
|
219
|
+
# Only changes the storage type. Conversions involving integers are step
|
|
220
|
+
# functions, so no gradient flows.
|
|
221
|
+
Grad.define(:cast) do |_g, node, dy, inputs|
|
|
222
|
+
float = %i[float32 float16]
|
|
223
|
+
next [nil] unless float.include?(node.dtype) && float.include?(inputs[0].dtype)
|
|
224
|
+
|
|
225
|
+
[dy.cast(inputs[0].dtype)]
|
|
226
|
+
end
|
|
227
|
+
|
|
228
|
+
# --- Reductions ----------------------------------------------------------
|
|
229
|
+
|
|
230
|
+
# sum / mean just spread back along the folded axis. min / max pass only to
|
|
231
|
+
# the winning element (on ties **every** winner gets the same cotangent —
|
|
232
|
+
# the one place that does not match finite differences exactly, so avoid ties).
|
|
233
|
+
Grad.define(:reduce) do |g, node, dy, inputs|
|
|
234
|
+
x = inputs[0]
|
|
235
|
+
spread = Grad.spread(dy, node, x.shape)
|
|
236
|
+
[case node.attrs[:kind]
|
|
237
|
+
when :sum then spread
|
|
238
|
+
when :mean
|
|
239
|
+
axis = node.attrs[:axis]
|
|
240
|
+
spread / (axis.nil? ? x.length : x.shape[axis]).to_f
|
|
241
|
+
when :min, :max
|
|
242
|
+
y = Grad.spread(Tensor.new(g, node), node, x.shape)
|
|
243
|
+
g.where(x.eq(y), spread, 0.0)
|
|
244
|
+
end]
|
|
245
|
+
end
|
|
246
|
+
|
|
247
|
+
# --- Shape and indexing --------------------------------------------------
|
|
248
|
+
|
|
249
|
+
Grad.define(:reshape) { |_g, _node, dy, inputs| [dy.reshape(*inputs[0].shape)] }
|
|
250
|
+
|
|
251
|
+
Grad.define(:broadcast) { |_g, _node, dy, inputs| [Grad.unbroadcast(dy, inputs[0].shape)] }
|
|
252
|
+
|
|
253
|
+
Grad.define(:permute) do |_g, node, dy, inputs|
|
|
254
|
+
axes = node.attrs[:axes]
|
|
255
|
+
inverse = Array.new(axes.length)
|
|
256
|
+
axes.each_with_index { |from, to| inverse[from] = to }
|
|
257
|
+
[dy.permute(*inverse)]
|
|
258
|
+
end
|
|
259
|
+
|
|
260
|
+
# The inverse of a slice puts it back in place and zero-fills the rest.
|
|
261
|
+
# slice takes only the outermost dim, so the fill is outermost-only too.
|
|
262
|
+
Grad.define(:slice) do |g, node, dy, inputs|
|
|
263
|
+
shape = inputs[0].shape
|
|
264
|
+
rows = node.shape[0]
|
|
265
|
+
start = node.attrs[:offset] / shape.drop(1).inject(1, :*)
|
|
266
|
+
before = [start] + Array.new(shape.length - 1, 0)
|
|
267
|
+
after = [shape[0] - start - rows] + Array.new(shape.length - 1, 0)
|
|
268
|
+
[g.pad(dy, before: before, after: after, value: 0.0)]
|
|
269
|
+
end
|
|
270
|
+
|
|
271
|
+
# Drops what the border added.
|
|
272
|
+
Grad.define(:pad) do |g, node, dy, inputs|
|
|
273
|
+
[Grad.crop(g, dy, node.attrs[:before], inputs[0].shape)]
|
|
274
|
+
end
|
|
275
|
+
|
|
276
|
+
# The inverse of concat slices out each piece's range.
|
|
277
|
+
Grad.define(:concat) do |g, node, dy, inputs|
|
|
278
|
+
axis = node.attrs[:axis]
|
|
279
|
+
offset = 0
|
|
280
|
+
inputs.map do |piece|
|
|
281
|
+
before = Array.new(piece.shape.length, 0)
|
|
282
|
+
before[axis] = offset
|
|
283
|
+
offset += piece.shape[axis]
|
|
284
|
+
Grad.crop(g, dy, before, piece.shape)
|
|
285
|
+
end
|
|
286
|
+
end
|
|
287
|
+
|
|
288
|
+
# The inverse of table lookup is scatter-add. It scans the index list per
|
|
289
|
+
# output element without atomics (Kernels.scatter_add), so values do not
|
|
290
|
+
# vary across runs and no [table rows, index count] mask is materialized.
|
|
291
|
+
Grad.define(:gather) do |g, node, dy, inputs|
|
|
292
|
+
source, indices = inputs
|
|
293
|
+
[g.scatter_add(indices.reshape(indices.length), dy.reshape(*source.shape.take(node.attrs[:axis]),
|
|
294
|
+
indices.length,
|
|
295
|
+
*source.shape.drop(node.attrs[:axis] + 1)),
|
|
296
|
+
shape: source.shape, axis: node.attrs[:axis]),
|
|
297
|
+
nil]
|
|
298
|
+
end
|
|
299
|
+
|
|
300
|
+
# scatter_add's own inverse is gather (reading back from where it added).
|
|
301
|
+
# One line so second derivatives do not stop at an embedding layer.
|
|
302
|
+
Grad.define(:scatter_add) do |g, node, dy, inputs|
|
|
303
|
+
[nil, g.gather(dy, inputs[0], axis: node.attrs[:axis])]
|
|
304
|
+
end
|
|
305
|
+
|
|
306
|
+
# --- Matrix multiply -----------------------------------------------------
|
|
307
|
+
#
|
|
308
|
+
# dA = dY Bᵀ, dB = Aᵀ dY. Transposes are written with permute — folding
|
|
309
|
+
# them into GEMM transpose flags is a later optimization, and this plain
|
|
310
|
+
# form is its reference (docs/autograd.md, "matmul, and #22's wall 2").
|
|
311
|
+
Grad.define(:matmul) do |_g, node, dy, inputs|
|
|
312
|
+
a, b = inputs
|
|
313
|
+
swap = node.attrs[:mode] == :batched ? [0, 2, 1] : [1, 0]
|
|
314
|
+
[dy.matmul(b.permute(*swap)), a.permute(*swap).matmul(dy)]
|
|
315
|
+
end
|
|
316
|
+
|
|
317
|
+
# --- Not differentiable --------------------------------------------------
|
|
318
|
+
|
|
319
|
+
# High-level ops default to "decompose, then differentiate". Recording with
|
|
320
|
+
# Graph.build(differentiable: true) yields decompositions, so two kinds stay
|
|
321
|
+
# :nn — functions with a hand-written vjp (NN_REGISTRY below) and functions
|
|
322
|
+
# without a decomposition (gelu_exact / groupnorm). The former go to their
|
|
323
|
+
# row; the latter fail with a reason.
|
|
324
|
+
Grad.define(:nn) do |g, node, dy, inputs|
|
|
325
|
+
fn = node.attrs[:fn]
|
|
326
|
+
vjp = Grad::NN_REGISTRY[fn]
|
|
327
|
+
if vjp.nil?
|
|
328
|
+
Grad.undefined!("NN.#{fn}",
|
|
329
|
+
"it has no decomposition into primitives, so it cannot be differentiated " \
|
|
330
|
+
"(record the graph with Graph.build(differentiable: true) for the ops that do)")
|
|
331
|
+
end
|
|
332
|
+
|
|
333
|
+
positional, keyed = Grad.nn_arguments(node, inputs)
|
|
334
|
+
grads = vjp.call(g, node, dy, positional, keyed)
|
|
335
|
+
# The result is {input -> cotangent}; put it back in the node's input
|
|
336
|
+
# order (positional, then keyword).
|
|
337
|
+
positional.each_index.map { |i| grads[i] } + node.attrs[:keys].map { |key| grads[key] }
|
|
338
|
+
end
|
|
339
|
+
|
|
340
|
+
# --- Hand-written vjps ---------------------------------------------------
|
|
341
|
+
#
|
|
342
|
+
# Each is "row-wise reduction + elementwise", so row fusion folds it into
|
|
343
|
+
# one or two kernels. The only difference from differentiating the
|
|
344
|
+
# decomposition is the number of round trips; the values agree
|
|
345
|
+
# (conformance compares them).
|
|
346
|
+
|
|
347
|
+
# dx = (dy - Σ(dy·y)) · y; one fold per row is enough.
|
|
348
|
+
Grad.define_nn(:softmax) do |_g, node, dy, positional, _keyed|
|
|
349
|
+
y = Tensor.new(positional[0].graph, node)
|
|
350
|
+
{ 0 => (dy - (dy * y).sum(axis: -1, keepdims: true)) * y }
|
|
351
|
+
end
|
|
352
|
+
|
|
353
|
+
# y = x̂·γ + β, x̂ = (x - μ)/σ.
|
|
354
|
+
# dx = (g - mean(g) - x̂·mean(g·x̂)) / σ, g = dy·γ.
|
|
355
|
+
# The γ / β gradients are folded over rows.
|
|
356
|
+
Grad.define_nn(:layernorm) do |_g, node, dy, positional, keyed|
|
|
357
|
+
x = positional[0]
|
|
358
|
+
eps = node.attrs[:opts].fetch(:eps)
|
|
359
|
+
centered = x - x.mean(axis: -1, keepdims: true)
|
|
360
|
+
sigma = ((centered * centered).mean(axis: -1, keepdims: true) + eps).sqrt
|
|
361
|
+
hat = centered / sigma
|
|
362
|
+
inner = keyed[:gamma] ? dy * keyed[:gamma] : dy
|
|
363
|
+
|
|
364
|
+
grads = { 0 => (inner - inner.mean(axis: -1, keepdims: true) -
|
|
365
|
+
(hat * (inner * hat).mean(axis: -1, keepdims: true))) / sigma }
|
|
366
|
+
grads[:gamma] = Grad.fold_rows(dy * hat) if keyed[:gamma]
|
|
367
|
+
grads[:beta] = Grad.fold_rows(dy) if keyed[:beta]
|
|
368
|
+
grads
|
|
369
|
+
end
|
|
370
|
+
|
|
371
|
+
# y = x·γ/r, r = sqrt(mean(x²) + eps). ∂r/∂x_j = x_j/(n·r), so
|
|
372
|
+
# dx = (g - x·Σ(g·x)/(n·r²)) / r, g = dy·γ.
|
|
373
|
+
Grad.define_nn(:rmsnorm) do |_g, node, dy, positional, keyed|
|
|
374
|
+
x = positional[0]
|
|
375
|
+
eps = node.attrs[:opts].fetch(:eps)
|
|
376
|
+
r = ((x * x).mean(axis: -1, keepdims: true) + eps).sqrt
|
|
377
|
+
inner = keyed[:gamma] ? dy * keyed[:gamma] : dy
|
|
378
|
+
|
|
379
|
+
grads = { 0 => (inner - (x * ((inner * x).mean(axis: -1, keepdims: true) / (r * r)))) / r }
|
|
380
|
+
grads[:gamma] = Grad.fold_rows(dy * (x / r)) if keyed[:gamma]
|
|
381
|
+
grads
|
|
382
|
+
end
|
|
383
|
+
|
|
384
|
+
# (softmax(x) - onehot) · scale. **This form is why cross_entropy takes
|
|
385
|
+
# logits** — differentiating log(softmax(x)) reaches the same value but
|
|
386
|
+
# passes through reciprocals of probabilities and loses digits.
|
|
387
|
+
Grad.define_nn(:cross_entropy) do |g, node, dy, positional, _keyed|
|
|
388
|
+
logits, labels = positional
|
|
389
|
+
cols = logits.shape[-1]
|
|
390
|
+
rows_shape = logits.shape[0..-2]
|
|
391
|
+
reduction = node.attrs[:opts].fetch(:reduction)
|
|
392
|
+
|
|
393
|
+
clamped = g.where(labels < 0, 0, labels)
|
|
394
|
+
clamped = g.where(clamped > (cols - 1), cols - 1, clamped)
|
|
395
|
+
onehot = g.where(g.iota(logits.shape, axis: -1).eq(clamped.reshape(*rows_shape, 1)), 1.0, 0.0)
|
|
396
|
+
|
|
397
|
+
# The reduction contributes a scale; for :none each row's cotangent goes to that row.
|
|
398
|
+
scale = case reduction
|
|
399
|
+
when :none then dy.reshape(*rows_shape, 1)
|
|
400
|
+
when :sum then dy
|
|
401
|
+
else dy / Grad.row_count(logits.shape).to_f
|
|
402
|
+
end
|
|
403
|
+
{ 0 => (Tilework::NN.softmax(logits) - onehot) * scale, 1 => nil }
|
|
404
|
+
end
|
|
405
|
+
|
|
406
|
+
# mse is elementwise only, so its decomposition is fast enough and it has no hand-written row.
|
|
407
|
+
|
|
408
|
+
# Convolution uses the delegate's backward kernels (cuDNN / MPSGraph) as
|
|
409
|
+
# is. On a backend without convolution :conv2d is never recorded (it is
|
|
410
|
+
# opened into im2col), so primitive vjps handle it there (which is why the
|
|
411
|
+
# gather row is needed).
|
|
412
|
+
Grad.define(:conv2d) do |g, node, dy, inputs|
|
|
413
|
+
x, weight = inputs
|
|
414
|
+
pad = node.attrs[:pad]
|
|
415
|
+
stride = node.attrs[:stride]
|
|
416
|
+
dilation = node.attrs[:dilation]
|
|
417
|
+
groups = node.attrs[:groups]
|
|
418
|
+
[g.conv2d_backward(:data, weight, dy, x, pad: pad, stride: stride, dilation: dilation, groups: groups),
|
|
419
|
+
g.conv2d_backward(:filter, x, dy, weight, pad: pad, stride: stride, dilation: dilation, groups: groups)]
|
|
420
|
+
end
|
|
421
|
+
|
|
422
|
+
# The backward nodes themselves are not differentiated again (there is no
|
|
423
|
+
# second-order convolution kernel).
|
|
424
|
+
%i[conv2d_backward_data conv2d_backward_filter].each do |op|
|
|
425
|
+
Grad.define(op) do |_g, _node, _dy, _inputs|
|
|
426
|
+
Grad.undefined!(op.to_s, "the delegated convolution kernels have no second derivative")
|
|
427
|
+
end
|
|
428
|
+
end
|
|
429
|
+
|
|
430
|
+
Grad.define(:opaque) do |_g, node, _dy, _inputs|
|
|
431
|
+
Grad.undefined!("opaque #{node.attrs[:name].inspect}", "the graph cannot see inside it")
|
|
432
|
+
end
|
|
433
|
+
|
|
434
|
+
# Writes to mutable state follow memory, not values. Backward through them
|
|
435
|
+
# would need folding across versions and regions, so they fail for now
|
|
436
|
+
# (#22 Step 3).
|
|
437
|
+
%i[state_write state_merge].each do |op|
|
|
438
|
+
Grad.define(op) do |_g, _node, _dy, _inputs|
|
|
439
|
+
Grad.undefined!(op.to_s, "differentiating through a state write is not supported yet (#22 Step 3)")
|
|
440
|
+
end
|
|
441
|
+
end
|
|
442
|
+
end
|
|
443
|
+
end
|