tilework 0.18.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE +21 -0
- data/lib/tilework/backend/cpu/dsl_codegen.rb +509 -0
- data/lib/tilework/backend/cpu/ops_codegen.rb +451 -0
- data/lib/tilework/backend/cpu.rb +156 -0
- data/lib/tilework/buffer.rb +753 -0
- data/lib/tilework/capabilities.rb +183 -0
- data/lib/tilework/compat/cuda.h +66 -0
- data/lib/tilework/compat/hip.h +35 -0
- data/lib/tilework/compat/metal.h +45 -0
- data/lib/tilework/conformance.rb +2296 -0
- data/lib/tilework/core_ext.rb +21 -0
- data/lib/tilework/dialect.rb +67 -0
- data/lib/tilework/dsl.rb +722 -0
- data/lib/tilework/dtype/bfloat.rb +31 -0
- data/lib/tilework/dtype/cast.rb +88 -0
- data/lib/tilework/dtype/half.rb +98 -0
- data/lib/tilework/graph/builder.rb +1073 -0
- data/lib/tilework/graph/fusion.rb +520 -0
- data/lib/tilework/graph/grad.rb +443 -0
- data/lib/tilework/graph/high_level.rb +266 -0
- data/lib/tilework/graph/kernels.rb +403 -0
- data/lib/tilework/graph/node.rb +27 -0
- data/lib/tilework/graph/op.rb +290 -0
- data/lib/tilework/graph/optim.rb +183 -0
- data/lib/tilework/graph/plan.rb +1086 -0
- data/lib/tilework/graph/pool.rb +178 -0
- data/lib/tilework/graph/region.rb +210 -0
- data/lib/tilework/graph/residency.rb +449 -0
- data/lib/tilework/graph/state.rb +64 -0
- data/lib/tilework/graph/tensor.rb +124 -0
- data/lib/tilework/graph.rb +213 -0
- data/lib/tilework/kernel.rb +163 -0
- data/lib/tilework/math/conv.rb +270 -0
- data/lib/tilework/math/fft.rb +172 -0
- data/lib/tilework/math/nn.rb +1103 -0
- data/lib/tilework/math/philox.rb +194 -0
- data/lib/tilework/math/random.rb +232 -0
- data/lib/tilework/math/tensor_core.rb +269 -0
- data/lib/tilework/ops.rb +682 -0
- data/lib/tilework/runtime/offload.rb +498 -0
- data/lib/tilework/runtime/submission.rb +163 -0
- data/lib/tilework/runtime/telemetry.rb +112 -0
- data/lib/tilework/scope.rb +33 -0
- data/lib/tilework/version.rb +5 -0
- data/lib/tilework.rb +345 -0
- metadata +92 -0
|
@@ -0,0 +1,1103 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Tilework
|
|
4
|
+
# Neural-network operations implemented as fused kernels.
|
|
5
|
+
#
|
|
6
|
+
# Why the library owns them (when `Ops` combinations could express them):
|
|
7
|
+
#
|
|
8
|
+
# 1. **Ops multiply.** Each operation has a fixed cost of about 90 µs, so
|
|
9
|
+
# writing softmax as `exp` / `sum` / `/` takes several round trips per
|
|
10
|
+
# row; fused, it is one kernel.
|
|
11
|
+
# 2. **Everyone gets the launch domain wrong.** Omitting `over:` in a
|
|
12
|
+
# row-wise reduction makes every column thread compute the whole row (6.9x
|
|
13
|
+
# waste at 346x346). External applications would make the same mistake.
|
|
14
|
+
#
|
|
15
|
+
# The implementation is **one row (or one group) per block**, with the
|
|
16
|
+
# block's threads cooperating through shared memory. Arithmetic is always in
|
|
17
|
+
# float; float16 buffers are storage only (the output dtype follows the input).
|
|
18
|
+
module NN
|
|
19
|
+
BLOCK = 256
|
|
20
|
+
|
|
21
|
+
# Upper bounds on K/V columns held per flash-attention block and on head
|
|
22
|
+
# dimension. K/V/Q and the output accumulator total about 17 KiB, well
|
|
23
|
+
# within the 32 KiB of threadgroup/shared memory on both Metal and CUDA.
|
|
24
|
+
# Heads above the bound fall back to the GEMM path (typical head_dim is
|
|
25
|
+
# 64 or 128).
|
|
26
|
+
FLASH_TILE_KEYS = 16
|
|
27
|
+
FLASH_MAX_HEAD_DIM = 128
|
|
28
|
+
FLASH_MIN_SCORES = 4096
|
|
29
|
+
|
|
30
|
+
# Reduction identities. The float minimum avoids -INFINITY (a finite lower
|
|
31
|
+
# bound suffices rather than guaranteeing identical spelling and behavior
|
|
32
|
+
# across MSL/CUDA).
|
|
33
|
+
NEG_MAX = "-3.402823466e+38f"
|
|
34
|
+
|
|
35
|
+
module_function
|
|
36
|
+
|
|
37
|
+
# Applies a numerically stable softmax over the final dimension.
|
|
38
|
+
#
|
|
39
|
+
# @param x [Buffer, Graph::Tensor] tensor with at least two dimensions
|
|
40
|
+
# @return [Buffer, Graph::Tensor] probabilities with the same shape and dtype
|
|
41
|
+
#
|
|
42
|
+
# Any rank from 2D up — all leading dims fold into rows, and the shape
|
|
43
|
+
# comes back unchanged.
|
|
44
|
+
#
|
|
45
|
+
# probs = Tilework::NN.softmax(logits) # [rows, cols] -> [rows, cols]
|
|
46
|
+
# probs = Tilework::NN.softmax(scores) # [heads, n, m] works the same
|
|
47
|
+
#
|
|
48
|
+
# The max is subtracted before exp, so large logits do not overflow
|
|
49
|
+
# (`exp(1000)` is inf).
|
|
50
|
+
def softmax(x)
|
|
51
|
+
rows, cols = rowwise_dims!(x, "softmax")
|
|
52
|
+
return Graph.record_nn(:softmax, [x], shape: x.shape, dtype: x.dtype) if Graph.symbolic?(x)
|
|
53
|
+
return cpu_softmax(x, rows, cols) if Tilework.host_fallback?
|
|
54
|
+
|
|
55
|
+
out = Ops.allocate_output(x.dtype, x.length, x.shape)
|
|
56
|
+
kernel = Tilework.compile(softmax_src(x.dtype), name: "nn_softmax")
|
|
57
|
+
Ops.launch_kernel(kernel, grid: rows, block: BLOCK, args: [x, out, rows, cols])
|
|
58
|
+
out
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
# Applies layer normalization over the final dimension.
|
|
62
|
+
#
|
|
63
|
+
# @param x [Buffer, Graph::Tensor]
|
|
64
|
+
# @param gamma [Buffer, Graph::Tensor, nil] optional scale of length `x.shape.last`
|
|
65
|
+
# @param beta [Buffer, Graph::Tensor, nil] optional bias of length `x.shape.last`
|
|
66
|
+
# @param eps [Float] numerical stability term
|
|
67
|
+
# @return [Buffer, Graph::Tensor]
|
|
68
|
+
#
|
|
69
|
+
# A standard transformer component. Like softmax, any rank from 2D up
|
|
70
|
+
# ([batch, tokens, d] passes as is).
|
|
71
|
+
#
|
|
72
|
+
# y = Tilework::NN.layernorm(x, gamma: g, beta: b) # g, b of length cols
|
|
73
|
+
#
|
|
74
|
+
# gamma / beta may be omitted (omitting both means no affine transform).
|
|
75
|
+
def layernorm(x, gamma: nil, beta: nil, eps: 1e-5)
|
|
76
|
+
rows, cols = rowwise_dims!(x, "layernorm")
|
|
77
|
+
check_affine!(gamma, beta, cols, x.dtype)
|
|
78
|
+
if Graph.symbolic?(x, gamma, beta)
|
|
79
|
+
return Graph.record_nn(:layernorm, [x], tensors: { gamma: gamma, beta: beta },
|
|
80
|
+
opts: { eps: eps }, shape: x.shape, dtype: x.dtype)
|
|
81
|
+
end
|
|
82
|
+
return cpu_layernorm(x, rows, cols, gamma, beta, eps) if Tilework.host_fallback?
|
|
83
|
+
|
|
84
|
+
affine = !gamma.nil? || !beta.nil?
|
|
85
|
+
out = Ops.allocate_output(x.dtype, x.length, x.shape)
|
|
86
|
+
kernel = Tilework.compile(layernorm_src(x.dtype, affine), name: "nn_layernorm")
|
|
87
|
+
args = [x, out]
|
|
88
|
+
temporaries = []
|
|
89
|
+
if affine
|
|
90
|
+
scale = gamma || ones_like(cols, x.dtype).tap { |buffer| temporaries << buffer }
|
|
91
|
+
bias = beta || zeros_like(cols, x.dtype).tap { |buffer| temporaries << buffer }
|
|
92
|
+
args += [scale, bias]
|
|
93
|
+
end
|
|
94
|
+
begin
|
|
95
|
+
Ops.launch_kernel(kernel, grid: rows, block: BLOCK, args: args + [rows, cols, eps.to_f])
|
|
96
|
+
ensure
|
|
97
|
+
temporaries.each(&:free)
|
|
98
|
+
end
|
|
99
|
+
out
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
# Applies RMS normalization over the final dimension.
|
|
103
|
+
#
|
|
104
|
+
# @param x [Buffer, Graph::Tensor]
|
|
105
|
+
# @param gamma [Buffer, Graph::Tensor, nil] optional scale of length `x.shape.last`
|
|
106
|
+
# @param eps [Float] numerical stability term
|
|
107
|
+
# @return [Buffer, Graph::Tensor]
|
|
108
|
+
#
|
|
109
|
+
# Divides by the root mean square without subtracting the mean, as used by
|
|
110
|
+
# T5 / LLaMA-family models.
|
|
111
|
+
#
|
|
112
|
+
# y = Tilework::NN.rmsnorm(x, gamma: g, eps: 1e-6) # g of length cols
|
|
113
|
+
#
|
|
114
|
+
# Unlike layernorm **there is no mean subtraction**, so no shift (beta)
|
|
115
|
+
# either. The sum of squares accumulates in float (in half it stops
|
|
116
|
+
# growing around 2048).
|
|
117
|
+
def rmsnorm(x, gamma: nil, eps: 1e-6)
|
|
118
|
+
rows, cols = rowwise_dims!(x, "rmsnorm")
|
|
119
|
+
check_affine!(gamma, nil, cols, x.dtype)
|
|
120
|
+
if Graph.symbolic?(x, gamma)
|
|
121
|
+
return Graph.record_nn(:rmsnorm, [x], tensors: { gamma: gamma },
|
|
122
|
+
opts: { eps: eps }, shape: x.shape, dtype: x.dtype)
|
|
123
|
+
end
|
|
124
|
+
return cpu_rmsnorm(x, rows, cols, gamma, eps) if Tilework.host_fallback?
|
|
125
|
+
|
|
126
|
+
out = Ops.allocate_output(x.dtype, x.length, x.shape)
|
|
127
|
+
kernel = Tilework.compile(rmsnorm_src(x.dtype, !gamma.nil?), name: "nn_rmsnorm")
|
|
128
|
+
args = [x, out]
|
|
129
|
+
args += [gamma] if gamma
|
|
130
|
+
Ops.launch_kernel(kernel, grid: rows, block: BLOCK, args: args + [rows, cols, eps.to_f])
|
|
131
|
+
out
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
# Applies group normalization to an NCHW tensor.
|
|
135
|
+
#
|
|
136
|
+
# @param x [Buffer, Graph::Tensor] `[N, C, H, W]` tensor
|
|
137
|
+
# @param groups [Integer] positive divisor of `C`
|
|
138
|
+
# @param gamma [Buffer, Graph::Tensor, nil] optional per-channel scale
|
|
139
|
+
# @param beta [Buffer, Graph::Tensor, nil] optional per-channel bias
|
|
140
|
+
# @param eps [Float] numerical stability term
|
|
141
|
+
# @return [Buffer, Graph::Tensor]
|
|
142
|
+
#
|
|
143
|
+
# Splits channels into `groups` and normalizes each group over all of
|
|
144
|
+
# (C/groups) x H x W — used heavily in diffusion-model UNets.
|
|
145
|
+
#
|
|
146
|
+
# y = Tilework::NN.groupnorm(x, groups: 32, gamma: g, beta: b)
|
|
147
|
+
#
|
|
148
|
+
# gamma / beta are **per channel** (length C). groups == C is instance
|
|
149
|
+
# norm; groups == 1 is equivalent to layer norm.
|
|
150
|
+
def groupnorm(x, groups:, gamma: nil, beta: nil, eps: 1e-5)
|
|
151
|
+
n, c, h, w = nchw_dims!(x, "groupnorm")
|
|
152
|
+
unless groups.is_a?(Integer) && groups.positive? && (c % groups).zero?
|
|
153
|
+
raise ArgumentError, "groups #{groups.inspect} must be a positive divisor of #{c} channels"
|
|
154
|
+
end
|
|
155
|
+
|
|
156
|
+
check_affine!(gamma, beta, c, x.dtype)
|
|
157
|
+
if Graph.symbolic?(x, gamma, beta)
|
|
158
|
+
return Graph.record_nn(:groupnorm, [x], tensors: { gamma: gamma, beta: beta },
|
|
159
|
+
opts: { groups: groups, eps: eps },
|
|
160
|
+
shape: x.shape, dtype: x.dtype)
|
|
161
|
+
end
|
|
162
|
+
return cpu_groupnorm(x, [n, c, h, w], groups, gamma, beta, eps) if Tilework.host_fallback?
|
|
163
|
+
|
|
164
|
+
affine = !gamma.nil? || !beta.nil?
|
|
165
|
+
out = Ops.allocate_output(x.dtype, x.length, x.shape)
|
|
166
|
+
kernel = Tilework.compile(groupnorm_src(x.dtype, affine), name: "nn_groupnorm")
|
|
167
|
+
args = [x, out]
|
|
168
|
+
temporaries = []
|
|
169
|
+
if affine
|
|
170
|
+
scale = gamma || ones_like(c, x.dtype).tap { |buffer| temporaries << buffer }
|
|
171
|
+
bias = beta || zeros_like(c, x.dtype).tap { |buffer| temporaries << buffer }
|
|
172
|
+
args += [scale, bias]
|
|
173
|
+
end
|
|
174
|
+
begin
|
|
175
|
+
Ops.launch_kernel(kernel, grid: n * groups, block: BLOCK, args: args + [n, c, h * w, groups, eps.to_f])
|
|
176
|
+
ensure
|
|
177
|
+
temporaries.each(&:free)
|
|
178
|
+
end
|
|
179
|
+
out
|
|
180
|
+
end
|
|
181
|
+
|
|
182
|
+
# --- Activations ---------------------------------------------------------
|
|
183
|
+
|
|
184
|
+
# Applies the SiLU (Swish) activation.
|
|
185
|
+
# @param x [Buffer, Graph::Tensor]
|
|
186
|
+
# @return [Buffer, Graph::Tensor]
|
|
187
|
+
#
|
|
188
|
+
# x * sigmoid(x), standard in diffusion models and recent LLMs. sigmoid is
|
|
189
|
+
# rearranged to use exp(-|x|), so very negative x does not overflow (a naive
|
|
190
|
+
# 1/(1+exp(-x)) gives exp(100) = inf at x = -100).
|
|
191
|
+
def silu(x) = activation(x, :silu)
|
|
192
|
+
|
|
193
|
+
# Applies GELU using the tanh approximation by default.
|
|
194
|
+
# @param x [Buffer, Graph::Tensor]
|
|
195
|
+
# @param exact [Boolean] use the slower `erf` formulation
|
|
196
|
+
# @return [Buffer, Graph::Tensor]
|
|
197
|
+
#
|
|
198
|
+
# The tanh approximation is the default of GPT-2 and many implementations.
|
|
199
|
+
def gelu(x, exact: false) = activation(x, exact ? :gelu_exact : :gelu_tanh)
|
|
200
|
+
|
|
201
|
+
# Applies the ReLU activation.
|
|
202
|
+
# @param x [Buffer, Graph::Tensor]
|
|
203
|
+
# @return [Buffer, Graph::Tensor]
|
|
204
|
+
def relu(x) = activation(x, :relu)
|
|
205
|
+
|
|
206
|
+
# Applies the sigmoid activation.
|
|
207
|
+
# @param x [Buffer, Graph::Tensor]
|
|
208
|
+
# @return [Buffer, Graph::Tensor]
|
|
209
|
+
def sigmoid(x) = activation(x, :sigmoid)
|
|
210
|
+
|
|
211
|
+
# --- Losses ------------------------------------------------------------
|
|
212
|
+
#
|
|
213
|
+
# The entry point of training. **They end in a reduction**, so like
|
|
214
|
+
# `Ops.reduce` there are two ways to receive the result (see #reduce in ops.rb):
|
|
215
|
+
#
|
|
216
|
+
# NN.mse(y, t) # => Float; the queue drains here
|
|
217
|
+
# NN.mse(y, t, device: true) # => a one-element Buffer, left on the device
|
|
218
|
+
# NN.mse(y, t, reduction: :none) # => per-element / per-row loss Buffer
|
|
219
|
+
#
|
|
220
|
+
# In a graph they always run in the device: true form (dropping to the host
|
|
221
|
+
# would end the plan). Backward (#22) requires that form, so **keeping the
|
|
222
|
+
# loss a Tensor** is the point.
|
|
223
|
+
#
|
|
224
|
+
# reduction is :mean (default) / :sum / :none. It changes the gradient's
|
|
225
|
+
# magnitude, so stating it explicitly beats relying on the default.
|
|
226
|
+
|
|
227
|
+
LOSS_REDUCTIONS = %i[mean sum none].freeze
|
|
228
|
+
|
|
229
|
+
# Calculates mean squared error.
|
|
230
|
+
#
|
|
231
|
+
# @param pred [Buffer, Graph::Tensor]
|
|
232
|
+
# @param target [Buffer, Graph::Tensor] same shape and dtype as `pred`
|
|
233
|
+
# @param reduction [:mean, :sum, :none] reduction mode
|
|
234
|
+
# @param device [Boolean] return a one-element buffer instead of a host scalar
|
|
235
|
+
# @return [Float, Buffer, Graph::Tensor]
|
|
236
|
+
#
|
|
237
|
+
# loss = Tilework::NN.mse(pred, target)
|
|
238
|
+
#
|
|
239
|
+
# reduction: :none returns per-element squared errors (the input's shape).
|
|
240
|
+
def mse(pred, target, reduction: :mean, device: false)
|
|
241
|
+
check_loss_pair!(pred, target)
|
|
242
|
+
check_reduction!(reduction)
|
|
243
|
+
if Graph.symbolic?(pred, target)
|
|
244
|
+
return Graph.record_nn(:mse, [pred, target], opts: { reduction: reduction, device: true },
|
|
245
|
+
shape: loss_shape(pred.shape, reduction, rowwise: false),
|
|
246
|
+
dtype: loss_dtype(pred.dtype))
|
|
247
|
+
end
|
|
248
|
+
|
|
249
|
+
reduce_loss(squared_error(pred, target), reduction, device: device)
|
|
250
|
+
end
|
|
251
|
+
|
|
252
|
+
# Calculates cross entropy from unnormalized logits and int32 class labels.
|
|
253
|
+
#
|
|
254
|
+
# @param logits [Buffer, Graph::Tensor] tensor whose final dimension is classes
|
|
255
|
+
# @param labels [Buffer, Graph::Tensor] int32 tensor with the final logits dimension removed
|
|
256
|
+
# @param reduction [:mean, :sum, :none] reduction mode
|
|
257
|
+
# @param device [Boolean] return a one-element buffer instead of a host scalar
|
|
258
|
+
# @return [Float, Buffer, Graph::Tensor]
|
|
259
|
+
#
|
|
260
|
+
# loss = Tilework::NN.cross_entropy(logits, labels)
|
|
261
|
+
#
|
|
262
|
+
# **Takes logits** (values before softmax). It computes **log-sum-exp in one
|
|
263
|
+
# kernel** rather than log after softmax: a naive `log(softmax(x))` hits
|
|
264
|
+
# -inf where a probability underflows to 0, and its gradient does not take
|
|
265
|
+
# the plain (softmax(x) - onehot) / n form.
|
|
266
|
+
#
|
|
267
|
+
# Out-of-range labels are clamped to the ends (the same rule as `gather`) —
|
|
268
|
+
# an out-of-range read on a GPU is undefined rather than merely wrong, so
|
|
269
|
+
# clamping beats breaking silently.
|
|
270
|
+
def cross_entropy(logits, labels, reduction: :mean, device: false)
|
|
271
|
+
rows, cols = rowwise_dims!(logits, "cross_entropy")
|
|
272
|
+
check_labels!(labels, logits, rows)
|
|
273
|
+
check_reduction!(reduction)
|
|
274
|
+
if Graph.symbolic?(logits, labels)
|
|
275
|
+
return Graph.record_nn(:cross_entropy, [logits, labels],
|
|
276
|
+
opts: { reduction: reduction, device: true },
|
|
277
|
+
shape: loss_shape(logits.shape, reduction, rowwise: true),
|
|
278
|
+
dtype: loss_dtype(logits.dtype))
|
|
279
|
+
end
|
|
280
|
+
|
|
281
|
+
rowwise = if Tilework.host_fallback?
|
|
282
|
+
cpu_cross_entropy(logits, labels, rows, cols)
|
|
283
|
+
else
|
|
284
|
+
out = Ops.allocate_output(loss_dtype(logits.dtype), rows, [rows])
|
|
285
|
+
kernel = Tilework.compile(cross_entropy_src(logits.dtype), name: "nn_cross_entropy")
|
|
286
|
+
Ops.launch_kernel(kernel, grid: rows, block: BLOCK, args: [logits, labels, out, rows, cols])
|
|
287
|
+
out
|
|
288
|
+
end
|
|
289
|
+
reduce_loss(rowwise, reduction, device: device)
|
|
290
|
+
end
|
|
291
|
+
|
|
292
|
+
# Transposes a two-dimensional tensor.
|
|
293
|
+
#
|
|
294
|
+
# @param x [Buffer, Graph::Tensor] `[rows, columns]` tensor
|
|
295
|
+
# @return [Buffer, Graph::Tensor] `[columns, rows]` tensor
|
|
296
|
+
#
|
|
297
|
+
# **Launches over the output's domain.** Launching over the input's domain
|
|
298
|
+
# and writing `out[c, r]` leaves the tail of the output unwritten when rows
|
|
299
|
+
# and columns differ.
|
|
300
|
+
def transpose(x)
|
|
301
|
+
rows, cols = matrix_dims!(x, "transpose")
|
|
302
|
+
return Graph.record_nn(:transpose, [x], shape: [cols, rows], dtype: x.dtype) if Graph.symbolic?(x)
|
|
303
|
+
return cpu_transpose(x, rows, cols) if Tilework.host_fallback?
|
|
304
|
+
|
|
305
|
+
out = Ops.allocate_output(x.dtype, x.length, [cols, rows])
|
|
306
|
+
kernel = Tilework.compile(transpose_src(x.dtype), name: "nn_transpose")
|
|
307
|
+
# The grid covers the output [cols, rows]: x is the column and y the row, as in the DSL.
|
|
308
|
+
grid = [(rows / 16.0).ceil, (cols / 16.0).ceil]
|
|
309
|
+
Ops.launch_kernel(kernel, grid: grid, block: [16, 16], args: [x, out, rows, cols])
|
|
310
|
+
out
|
|
311
|
+
end
|
|
312
|
+
|
|
313
|
+
# Calculates scaled dot-product multi-head attention.
|
|
314
|
+
#
|
|
315
|
+
# @param q [Buffer, Graph::Tensor] query matrix `[n, d]`
|
|
316
|
+
# @param k [Buffer, Graph::Tensor] key matrix `[m, d]`
|
|
317
|
+
# @param v [Buffer, Graph::Tensor] value matrix `[m, d]`
|
|
318
|
+
# @param heads [Integer] number of heads dividing `d`
|
|
319
|
+
# @param scale [Float, nil] score scale; defaults to `1 / sqrt(d / heads)`
|
|
320
|
+
# @param bias [Buffer, Graph::Tensor, nil] optional broadcastable score bias
|
|
321
|
+
# @return [Buffer, Graph::Tensor] attention output `[n, d]`
|
|
322
|
+
#
|
|
323
|
+
# q: [n, d], k/v: [m, d], d = heads * head_dim; returns [n, d].
|
|
324
|
+
#
|
|
325
|
+
# Heads are **small, evenly spaced matrices** inside `[*, d]`, so with a
|
|
326
|
+
# strided batched GEMM they never need slicing out or writing back: pass
|
|
327
|
+
# the original width d as ld and head_dim as the stride and compute on
|
|
328
|
+
# views. The score matrix [heads, n, m] is contiguous, so viewed as
|
|
329
|
+
# [heads*n, m] **one softmax covers every head**.
|
|
330
|
+
#
|
|
331
|
+
# For one 16-head attention this cuts launches from about 100 to 4.
|
|
332
|
+
#
|
|
333
|
+
# scale multiplies the scores (default 1/sqrt(head_dim)). It goes to GEMM's
|
|
334
|
+
# alpha, so it is **free**; writing `* (1.0/Math.sqrt(dh))` yourself costs
|
|
335
|
+
# another full round trip over the score matrix.
|
|
336
|
+
#
|
|
337
|
+
# bias is added to the scores **before** softmax, shaped [heads * n, m]
|
|
338
|
+
# (each head's [n, m] stacked vertically — the scores' own layout). T5's
|
|
339
|
+
# relative position bias and additive padding masks are this.
|
|
340
|
+
#
|
|
341
|
+
# If padding sits at the end, **passing k/v cut to the valid length** is
|
|
342
|
+
# faster than masking with bias (it removes the computation itself).
|
|
343
|
+
def multi_head_attention(q, k, v, heads:, scale: nil, bias: nil)
|
|
344
|
+
n, d = matrix_dims!(q, "multi_head_attention (q)")
|
|
345
|
+
m, dk = matrix_dims!(k, "multi_head_attention (k)")
|
|
346
|
+
mv, dv = matrix_dims!(v, "multi_head_attention (v)")
|
|
347
|
+
unless d == dk && d == dv && m == mv
|
|
348
|
+
raise ArgumentError,
|
|
349
|
+
"q [#{n}, #{d}] に対し k は [m, #{d}]、v は [m, #{d}] でなければならない " \
|
|
350
|
+
"(k: [#{m}, #{dk}], v: [#{mv}, #{dv}])"
|
|
351
|
+
end
|
|
352
|
+
raise ArgumentError, "d (#{d}) は heads (#{heads}) で割り切れなければならない" unless (d % heads).zero?
|
|
353
|
+
|
|
354
|
+
dh = d / heads
|
|
355
|
+
scale ||= 1.0 / Math.sqrt(dh)
|
|
356
|
+
check_attention_bias!(bias, heads, n, m)
|
|
357
|
+
if Graph.symbolic?(q, k, v, bias)
|
|
358
|
+
# scale is fixed here before becoming an attribute (decided when the plan is built).
|
|
359
|
+
return Graph.record_nn(:multi_head_attention, [q, k, v], tensors: { bias: bias },
|
|
360
|
+
opts: { heads: heads, scale: scale },
|
|
361
|
+
shape: [n, d], dtype: :float32)
|
|
362
|
+
end
|
|
363
|
+
return flash_attention(q, k, v, heads, dh, n, m, d, scale, bias) if flash_attention?(dh, n, m)
|
|
364
|
+
|
|
365
|
+
unless TensorCore.gemm_batched? && q.dtype == k.dtype && q.dtype == v.dtype &&
|
|
366
|
+
Tilework.capabilities.gemm_dtypes.include?(q.dtype)
|
|
367
|
+
return attention_per_head(q, k, v, heads, dh, n, m, d, scale, bias)
|
|
368
|
+
end
|
|
369
|
+
|
|
370
|
+
# Column-major Cc = Kc^T * Ac. Kc/Ac are views inside the parent [*, d], so ld = d.
|
|
371
|
+
scores = Buffer.new(:float32, heads * n * m, shape: [heads * n, m])
|
|
372
|
+
TensorCore.gemm_batched(k, q, scores,
|
|
373
|
+
m: m, k: dh, n: n,
|
|
374
|
+
lda: d, ldb: d, ldc: m,
|
|
375
|
+
stride_a: dh, stride_b: dh, stride_c: n * m,
|
|
376
|
+
batch: heads, trans_a: true, alpha: scale)
|
|
377
|
+
if bias
|
|
378
|
+
biased = scores + bias
|
|
379
|
+
scores.free
|
|
380
|
+
scores = biased
|
|
381
|
+
end
|
|
382
|
+
probs = softmax(scores)
|
|
383
|
+
scores.free
|
|
384
|
+
|
|
385
|
+
# Column-major ctx^T = Vc * Pc. With ldc = d and stride_c = dh it writes
|
|
386
|
+
# **directly** into the h-th section of the output [n, d] (no write-back kernel).
|
|
387
|
+
probs_for_gemm = if Ops::LOW_PRECISION_DTYPES.include?(v.dtype)
|
|
388
|
+
probs.cast(v.dtype)
|
|
389
|
+
else
|
|
390
|
+
probs
|
|
391
|
+
end
|
|
392
|
+
out = Ops.allocate_output(:float32, n * d, [n, d])
|
|
393
|
+
TensorCore.gemm_batched(v, probs_for_gemm, out,
|
|
394
|
+
m: dh, k: m, n: n,
|
|
395
|
+
lda: d, ldb: m, ldc: d,
|
|
396
|
+
stride_a: dh, stride_b: n * m, stride_c: dh,
|
|
397
|
+
batch: heads)
|
|
398
|
+
probs_for_gemm.free unless probs_for_gemm.equal?(probs)
|
|
399
|
+
probs.free
|
|
400
|
+
out
|
|
401
|
+
rescue StandardError
|
|
402
|
+
[scores, probs_for_gemm, probs, out].compact.uniq.each { |buffer| buffer.free unless buffer.freed? }
|
|
403
|
+
raise
|
|
404
|
+
end
|
|
405
|
+
|
|
406
|
+
# ========================================================================
|
|
407
|
+
# Internal
|
|
408
|
+
# ========================================================================
|
|
409
|
+
|
|
410
|
+
# Attention that never puts the [heads*n, m] scores in VRAM. Each block is
|
|
411
|
+
# "one query row of one head": K/V tiles go through shared memory while an
|
|
412
|
+
# online softmax updates the running max / running sum and the output
|
|
413
|
+
# accumulator (the same numerical skeleton as FlashAttention). The working
|
|
414
|
+
# set is O(tile * dh), and no Buffer proportional to n*m is allocated.
|
|
415
|
+
def flash_attention(q, k, v, heads, dh, n, m, d, scale, bias)
|
|
416
|
+
out = Ops.allocate_output(:float32, n * d, [n, d])
|
|
417
|
+
kernel = Tilework.compile(flash_attention_src(q.dtype, k.dtype, v.dtype, bias&.dtype),
|
|
418
|
+
name: "nn_flash_attention")
|
|
419
|
+
args = [q, k, v]
|
|
420
|
+
args << bias if bias
|
|
421
|
+
Ops.launch_kernel(kernel, grid: heads * n, block: BLOCK,
|
|
422
|
+
args: args + [out, heads, n, m, d, dh, scale.to_f])
|
|
423
|
+
out
|
|
424
|
+
rescue StandardError
|
|
425
|
+
out&.free
|
|
426
|
+
raise
|
|
427
|
+
end
|
|
428
|
+
|
|
429
|
+
def flash_attention?(dh, n, m)
|
|
430
|
+
!Tilework.host_fallback? && Tilework.capabilities.dialect &&
|
|
431
|
+
dh <= FLASH_MAX_HEAD_DIM && (n * m) >= FLASH_MIN_SCORES
|
|
432
|
+
end
|
|
433
|
+
|
|
434
|
+
def flash_attention_src(q_dtype, k_dtype, v_dtype, bias_dtype)
|
|
435
|
+
qt = ctype(q_dtype)
|
|
436
|
+
kt = ctype(k_dtype)
|
|
437
|
+
vt = ctype(v_dtype)
|
|
438
|
+
bias_param = bias_dtype ? ",\n device const #{ctype(bias_dtype)}* bias" : ""
|
|
439
|
+
bias_read = bias_dtype ? " + #{read(bias_dtype, 'bias[(head * n + query) * m + key]')}" : ""
|
|
440
|
+
<<~KERNEL
|
|
441
|
+
GPU_KERNEL void nn_flash_attention(GPU_ARGS,
|
|
442
|
+
device const #{qt}* q,
|
|
443
|
+
device const #{kt}* k,
|
|
444
|
+
device const #{vt}* v#{bias_param},
|
|
445
|
+
device float* out,
|
|
446
|
+
constant int& heads,
|
|
447
|
+
constant int& n,
|
|
448
|
+
constant int& m,
|
|
449
|
+
constant int& d,
|
|
450
|
+
constant int& dh,
|
|
451
|
+
constant float& scale) {
|
|
452
|
+
int row = GROUP_ID_X;
|
|
453
|
+
int lid = LOCAL_ID_X;
|
|
454
|
+
int head = row / n;
|
|
455
|
+
int query = row % n;
|
|
456
|
+
if (head >= heads) return;
|
|
457
|
+
|
|
458
|
+
GPU_SHARED float dot[#{BLOCK}];
|
|
459
|
+
GPU_SHARED float qtile[#{FLASH_MAX_HEAD_DIM}];
|
|
460
|
+
GPU_SHARED float ktile[#{FLASH_TILE_KEYS * FLASH_MAX_HEAD_DIM}];
|
|
461
|
+
GPU_SHARED float vtile[#{FLASH_TILE_KEYS * FLASH_MAX_HEAD_DIM}];
|
|
462
|
+
GPU_SHARED float acc[#{FLASH_MAX_HEAD_DIM}];
|
|
463
|
+
// state = max, sum, old-scale, new-score-weight
|
|
464
|
+
GPU_SHARED float state[4];
|
|
465
|
+
|
|
466
|
+
for (int j = lid; j < dh; j += #{BLOCK}) {
|
|
467
|
+
qtile[j] = #{read(q_dtype, 'q[query * d + head * dh + j]')};
|
|
468
|
+
acc[j] = 0.0f;
|
|
469
|
+
}
|
|
470
|
+
if (lid == 0) { state[0] = #{NEG_MAX}; state[1] = 0.0f; }
|
|
471
|
+
GPU_BARRIER;
|
|
472
|
+
|
|
473
|
+
for (int tile = 0; tile < m; tile += #{FLASH_TILE_KEYS}) {
|
|
474
|
+
int count = m - tile < #{FLASH_TILE_KEYS} ? m - tile : #{FLASH_TILE_KEYS};
|
|
475
|
+
for (int i = lid; i < count * dh; i += #{BLOCK}) {
|
|
476
|
+
int t = i / dh;
|
|
477
|
+
int j = i % dh;
|
|
478
|
+
ktile[i] = #{read(k_dtype, 'k[(tile + t) * d + head * dh + j]')};
|
|
479
|
+
vtile[i] = #{read(v_dtype, 'v[(tile + t) * d + head * dh + j]')};
|
|
480
|
+
}
|
|
481
|
+
GPU_BARRIER;
|
|
482
|
+
|
|
483
|
+
for (int t = 0; t < count; t++) {
|
|
484
|
+
int key = tile + t;
|
|
485
|
+
float partial = 0.0f;
|
|
486
|
+
for (int j = lid; j < dh; j += #{BLOCK}) partial += qtile[j] * ktile[t * dh + j];
|
|
487
|
+
dot[lid] = partial;
|
|
488
|
+
#{block_reduce('dot[lid] + dot[lid + s]').gsub('sh[', 'dot[')}
|
|
489
|
+
|
|
490
|
+
if (lid == 0) {
|
|
491
|
+
float score = dot[0] * scale#{bias_read};
|
|
492
|
+
float next_max = fmax(state[0], score);
|
|
493
|
+
state[2] = exp(state[0] - next_max);
|
|
494
|
+
state[3] = exp(score - next_max);
|
|
495
|
+
state[0] = next_max;
|
|
496
|
+
state[1] = state[1] * state[2] + state[3];
|
|
497
|
+
}
|
|
498
|
+
GPU_BARRIER;
|
|
499
|
+
for (int j = lid; j < dh; j += #{BLOCK}) {
|
|
500
|
+
acc[j] = acc[j] * state[2] + state[3] * vtile[t * dh + j];
|
|
501
|
+
}
|
|
502
|
+
GPU_BARRIER;
|
|
503
|
+
}
|
|
504
|
+
}
|
|
505
|
+
|
|
506
|
+
for (int j = lid; j < dh; j += #{BLOCK}) out[query * d + head * dh + j] = acc[j] / state[1];
|
|
507
|
+
}
|
|
508
|
+
KERNEL
|
|
509
|
+
end
|
|
510
|
+
|
|
511
|
+
# For backends without batched GEMM (such as :cpu): materializes heads one
|
|
512
|
+
# at a time. Slow, but the result is the same.
|
|
513
|
+
def attention_per_head(q, k, v, heads, dh, n, m, d, scale, bias = nil)
|
|
514
|
+
# head_scatter covers [n, d] across all heads, so a reused pool slot (not
|
|
515
|
+
# zero-filled) leaves no residue.
|
|
516
|
+
out = Ops.allocate_output(:float32, n * d, [n, d])
|
|
517
|
+
heads.times do |h|
|
|
518
|
+
qh = head_slice(q, h, dh, n, d)
|
|
519
|
+
kh = head_slice(k, h, dh, m, d)
|
|
520
|
+
vh = head_slice(v, h, dh, m, d)
|
|
521
|
+
kt = transpose(kh)
|
|
522
|
+
scores = qh.matmul(kt) * scale
|
|
523
|
+
if bias
|
|
524
|
+
# bias is [heads * n, m]; the h-th [n, m] is contiguous by rows, so a
|
|
525
|
+
# view can take it (do not let it outlive its parent)
|
|
526
|
+
biased = scores + bias.view(h * n * m, n * m, shape: [n, m])
|
|
527
|
+
scores.free
|
|
528
|
+
scores = biased
|
|
529
|
+
end
|
|
530
|
+
probs = softmax(scores)
|
|
531
|
+
ctx = probs.matmul(vh)
|
|
532
|
+
head_scatter(ctx, out, h, dh, n, d)
|
|
533
|
+
[qh, kh, vh, kt, scores, probs, ctx].each(&:free)
|
|
534
|
+
end
|
|
535
|
+
out
|
|
536
|
+
rescue StandardError
|
|
537
|
+
[qh, kh, vh, kt, scores, probs, ctx, out].compact.uniq.each { |buffer| buffer.free unless buffer.freed? }
|
|
538
|
+
raise
|
|
539
|
+
end
|
|
540
|
+
|
|
541
|
+
def check_attention_bias!(bias, heads, n, m)
|
|
542
|
+
return if bias.nil?
|
|
543
|
+
|
|
544
|
+
shaped!(bias, "bias")
|
|
545
|
+
return if bias.shape == [heads * n, m]
|
|
546
|
+
|
|
547
|
+
raise ArgumentError,
|
|
548
|
+
"bias must be [heads * n, m] = #{[heads * n, m].inspect}, got #{bias.shape.inspect}"
|
|
549
|
+
end
|
|
550
|
+
|
|
551
|
+
# Extracts the h-th head section of [rows, d] as [rows, dh].
|
|
552
|
+
def head_slice(x, head, dh, rows, _d)
|
|
553
|
+
@head_gather ||= Tilework.kernel("nn_head_gather", out: 1) do |src, o, hd, dhh|
|
|
554
|
+
j = global_id_x
|
|
555
|
+
r = global_id_y
|
|
556
|
+
o[r, j] = src[r, (hd * dhh) + j] if j < dhh && r < src.rows
|
|
557
|
+
end
|
|
558
|
+
@head_gather.call(x, head, dh, over: [dh, rows])
|
|
559
|
+
end
|
|
560
|
+
|
|
561
|
+
# The reverse direction. dst is not out:, so the guard is written by hand.
|
|
562
|
+
def head_scatter(src, dst, head, dh, rows, _d)
|
|
563
|
+
@head_scatter ||= Tilework.kernel("nn_head_scatter") do |s, o, hd, dhh|
|
|
564
|
+
j = global_id_x
|
|
565
|
+
r = global_id_y
|
|
566
|
+
o[r, (hd * dhh) + j] = s[r, j] if j < dhh && r < s.rows
|
|
567
|
+
end
|
|
568
|
+
@head_scatter.call(src, dst, head, dh, over: [dh, rows])
|
|
569
|
+
end
|
|
570
|
+
|
|
571
|
+
def ctype(dtype) = Ops::CTYPE.fetch(dtype)
|
|
572
|
+
|
|
573
|
+
# Half is a storage type, so it is widened to float before entering an expression.
|
|
574
|
+
def read(dtype, expr) = Ops.read_dtype(dtype, expr)
|
|
575
|
+
|
|
576
|
+
# Checks that look only at shape and dtype accept Buffers and Graph::Tensors
|
|
577
|
+
# alike, so recording into a graph goes through **the same checks** —
|
|
578
|
+
# splitting them would let expressions through on one path only.
|
|
579
|
+
def shaped!(x, op)
|
|
580
|
+
return x if x.is_a?(Buffer) || x.is_a?(Graph::Tensor)
|
|
581
|
+
|
|
582
|
+
raise TypeError, "#{op} requires a Tilework::Buffer or Tilework::Graph::Tensor, got #{x.class}"
|
|
583
|
+
end
|
|
584
|
+
|
|
585
|
+
def matrix_dims!(x, op)
|
|
586
|
+
shaped!(x, op)
|
|
587
|
+
unless x.shape.length == 2
|
|
588
|
+
raise ArgumentError, "#{op} requires a 2D buffer, got #{x.shape.inspect}"
|
|
589
|
+
end
|
|
590
|
+
|
|
591
|
+
x.shape
|
|
592
|
+
end
|
|
593
|
+
|
|
594
|
+
# The launch domain for row-wise reductions (softmax / layernorm /
|
|
595
|
+
# rmsnorm): **the last dim is columns and everything before folds into
|
|
596
|
+
# rows**. Kernels only see a flat [rows, cols], so 3D and up pass as is
|
|
597
|
+
# ([batch, tokens, d] or attention's [heads, n, m]); the output keeps the
|
|
598
|
+
# input's shape, so no dims are lost.
|
|
599
|
+
def rowwise_dims!(x, op)
|
|
600
|
+
shaped!(x, op)
|
|
601
|
+
unless x.shape.length >= 2
|
|
602
|
+
raise ArgumentError, "#{op} requires a 2D or higher buffer, got #{x.shape.inspect}"
|
|
603
|
+
end
|
|
604
|
+
|
|
605
|
+
[x.shape[0..-2].inject(:*), x.shape[-1]]
|
|
606
|
+
end
|
|
607
|
+
|
|
608
|
+
def nchw_dims!(x, op)
|
|
609
|
+
shaped!(x, op)
|
|
610
|
+
unless x.shape.length == 4
|
|
611
|
+
raise ArgumentError, "#{op} requires a 4D NCHW buffer, got #{x.shape.inspect}"
|
|
612
|
+
end
|
|
613
|
+
|
|
614
|
+
x.shape
|
|
615
|
+
end
|
|
616
|
+
|
|
617
|
+
def check_affine!(gamma, beta, expected, dtype)
|
|
618
|
+
[["gamma", gamma], ["beta", beta]].each do |name, buf|
|
|
619
|
+
next if buf.nil?
|
|
620
|
+
|
|
621
|
+
shaped!(buf, name)
|
|
622
|
+
unless buf.length == expected
|
|
623
|
+
raise ArgumentError, "#{name} must have #{expected} elements, got #{buf.length}"
|
|
624
|
+
end
|
|
625
|
+
unless buf.dtype == dtype
|
|
626
|
+
raise ArgumentError, "#{name} dtype #{buf.dtype.inspect} does not match the input's #{dtype.inspect}"
|
|
627
|
+
end
|
|
628
|
+
end
|
|
629
|
+
end
|
|
630
|
+
|
|
631
|
+
# --- Shared loss helpers -------------------------------------------------
|
|
632
|
+
|
|
633
|
+
def check_reduction!(reduction)
|
|
634
|
+
return if LOSS_REDUCTIONS.include?(reduction)
|
|
635
|
+
|
|
636
|
+
raise ArgumentError, "reduction must be one of #{LOSS_REDUCTIONS.inspect}, got #{reduction.inspect}"
|
|
637
|
+
end
|
|
638
|
+
|
|
639
|
+
def check_loss_pair!(pred, target)
|
|
640
|
+
shaped!(pred, "mse (pred)")
|
|
641
|
+
shaped!(target, "mse (target)")
|
|
642
|
+
unless pred.shape == target.shape
|
|
643
|
+
raise ArgumentError, "mse requires matching shapes (pred #{pred.shape.inspect}, " \
|
|
644
|
+
"target #{target.shape.inspect})"
|
|
645
|
+
end
|
|
646
|
+
unless pred.dtype == target.dtype
|
|
647
|
+
raise ArgumentError, "mse requires matching dtypes (pred #{pred.dtype.inspect}, " \
|
|
648
|
+
"target #{target.dtype.inspect})"
|
|
649
|
+
end
|
|
650
|
+
return if FLOAT_DTYPES.include?(pred.dtype)
|
|
651
|
+
|
|
652
|
+
raise ArgumentError, "mse requires a float dtype, got #{pred.dtype.inspect}"
|
|
653
|
+
end
|
|
654
|
+
|
|
655
|
+
# Labels are class indices (:int32). Probability-style inputs (one-hot /
|
|
656
|
+
# label smoothing) are floats of the same shape, so a mix-up fails by name,
|
|
657
|
+
# saying what is expected.
|
|
658
|
+
def check_labels!(labels, logits, rows)
|
|
659
|
+
shaped!(labels, "cross_entropy (labels)")
|
|
660
|
+
unless labels.dtype == :int32
|
|
661
|
+
raise ArgumentError, "cross_entropy labels must be :int32 class indices, got " \
|
|
662
|
+
"#{labels.dtype.inspect} (probability targets are not supported yet)"
|
|
663
|
+
end
|
|
664
|
+
expected = logits.shape[0..-2]
|
|
665
|
+
return if labels.shape == expected || labels.length == rows
|
|
666
|
+
|
|
667
|
+
raise ArgumentError, "cross_entropy labels must have shape #{expected.inspect} " \
|
|
668
|
+
"(logits #{logits.shape.inspect}), got #{labels.shape.inspect}"
|
|
669
|
+
end
|
|
670
|
+
|
|
671
|
+
FLOAT_DTYPES = %i[float32 float16 bfloat16].freeze
|
|
672
|
+
|
|
673
|
+
# Losses are always kept in float. Accumulating in half piles up error as
|
|
674
|
+
# the count grows, so even :none (no reduction) widens to float32 (the same
|
|
675
|
+
# rule as Ops' accumulation).
|
|
676
|
+
def loss_dtype(dtype) = Ops::LOW_PRECISION_DTYPES.include?(dtype) ? :float32 : dtype
|
|
677
|
+
|
|
678
|
+
def loss_shape(shape, reduction, rowwise:)
|
|
679
|
+
return [1] unless reduction == :none
|
|
680
|
+
|
|
681
|
+
rowwise ? shape[0..-2] : shape
|
|
682
|
+
end
|
|
683
|
+
|
|
684
|
+
# Receives the reduced value; :none returns it as is.
|
|
685
|
+
def reduce_loss(values, reduction, device:)
|
|
686
|
+
return values if reduction == :none
|
|
687
|
+
|
|
688
|
+
result = if reduction == :mean
|
|
689
|
+
Ops.mean_op(values, device: device)
|
|
690
|
+
else
|
|
691
|
+
Ops.reduce(:sum, values, device: device)
|
|
692
|
+
end
|
|
693
|
+
values.free
|
|
694
|
+
result
|
|
695
|
+
end
|
|
696
|
+
|
|
697
|
+
def squared_error(pred, target)
|
|
698
|
+
dtype = loss_dtype(pred.dtype)
|
|
699
|
+
if Tilework.host_fallback?
|
|
700
|
+
values = pred.to_a.zip(target.to_a).map { |a, b| (a - b)**2 }
|
|
701
|
+
out = Ops.allocate_output(dtype, values.length, pred.shape)
|
|
702
|
+
out.write(values)
|
|
703
|
+
return out
|
|
704
|
+
end
|
|
705
|
+
|
|
706
|
+
out = Ops.allocate_output(dtype, pred.length, pred.shape)
|
|
707
|
+
kernel = Tilework.compile(squared_error_src(pred.dtype, dtype), name: "nn_squared_error")
|
|
708
|
+
Ops.launch_kernel(kernel, grid: (pred.length / 256.0).ceil, block: 256, args: [pred, target, out, pred.length])
|
|
709
|
+
out
|
|
710
|
+
end
|
|
711
|
+
|
|
712
|
+
def squared_error_src(dtype, acc)
|
|
713
|
+
t = ctype(dtype)
|
|
714
|
+
<<~KERNEL
|
|
715
|
+
GPU_KERNEL void nn_squared_error(GPU_ARGS,
|
|
716
|
+
device const #{t}* a,
|
|
717
|
+
device const #{t}* b,
|
|
718
|
+
device #{ctype(acc)}* y,
|
|
719
|
+
constant int& n) {
|
|
720
|
+
int i = GLOBAL_ID_X;
|
|
721
|
+
if (i < n) {
|
|
722
|
+
float d = #{read(dtype, 'a[i]')} - #{read(dtype, 'b[i]')};
|
|
723
|
+
y[i] = d * d;
|
|
724
|
+
}
|
|
725
|
+
}
|
|
726
|
+
KERNEL
|
|
727
|
+
end
|
|
728
|
+
|
|
729
|
+
# One row per block, the same skeleton as softmax (max → sum), but only one
|
|
730
|
+
# loss per row is written, so no third pass is needed.
|
|
731
|
+
def cross_entropy_src(dtype)
|
|
732
|
+
t = ctype(dtype)
|
|
733
|
+
<<~KERNEL
|
|
734
|
+
GPU_KERNEL void nn_cross_entropy(GPU_ARGS,
|
|
735
|
+
device const #{t}* x,
|
|
736
|
+
device const int* labels,
|
|
737
|
+
device #{ctype(loss_dtype(dtype))}* loss,
|
|
738
|
+
constant int& rows,
|
|
739
|
+
constant int& cols) {
|
|
740
|
+
int row = GROUP_ID_X;
|
|
741
|
+
int lid = LOCAL_ID_X;
|
|
742
|
+
if (row >= rows) return;
|
|
743
|
+
GPU_SHARED float sh[#{BLOCK}];
|
|
744
|
+
int base = row * cols;
|
|
745
|
+
|
|
746
|
+
// 1. Row max (stabilizes log-sum-exp)
|
|
747
|
+
float m = #{NEG_MAX};
|
|
748
|
+
for (int i = lid; i < cols; i += #{BLOCK}) m = fmax(m, #{read(dtype, 'x[base + i]')});
|
|
749
|
+
sh[lid] = m;
|
|
750
|
+
#{block_reduce('fmax(sh[lid], sh[lid + s])')}
|
|
751
|
+
m = sh[0];
|
|
752
|
+
GPU_BARRIER; // sh is reused by the next reduction, so wait until everyone has read it
|
|
753
|
+
|
|
754
|
+
// 2. Sum of exp
|
|
755
|
+
float total = 0.0f;
|
|
756
|
+
for (int i = lid; i < cols; i += #{BLOCK}) total += exp(#{read(dtype, 'x[base + i]')} - m);
|
|
757
|
+
sh[lid] = total;
|
|
758
|
+
#{block_reduce('sh[lid] + sh[lid + s]')}
|
|
759
|
+
|
|
760
|
+
// 3. -log p[label] = log(sum exp(x - m)) - (x[label] - m)
|
|
761
|
+
if (lid == 0) {
|
|
762
|
+
int c = labels[row];
|
|
763
|
+
c = c < 0 ? 0 : (c >= cols ? cols - 1 : c); // clamp out-of-range to the ends (as gather does)
|
|
764
|
+
loss[row] = log(sh[0]) - (#{read(dtype, 'x[base + c]')} - m);
|
|
765
|
+
}
|
|
766
|
+
}
|
|
767
|
+
KERNEL
|
|
768
|
+
end
|
|
769
|
+
|
|
770
|
+
def cpu_cross_entropy(logits, labels, rows, cols)
|
|
771
|
+
flat = logits.to_a
|
|
772
|
+
classes = labels.to_a
|
|
773
|
+
values = Array.new(rows) do |row|
|
|
774
|
+
slice = flat[row * cols, cols]
|
|
775
|
+
m = slice.max
|
|
776
|
+
c = classes[row].clamp(0, cols - 1)
|
|
777
|
+
Math.log(slice.sum { |v| Math.exp(v - m) }) - (slice[c] - m)
|
|
778
|
+
end
|
|
779
|
+
out = Ops.allocate_output(loss_dtype(logits.dtype), rows, [rows])
|
|
780
|
+
out.write(values)
|
|
781
|
+
out
|
|
782
|
+
end
|
|
783
|
+
|
|
784
|
+
# Fills in when only one of gamma/beta is passed.
|
|
785
|
+
def ones_like(n, dtype)
|
|
786
|
+
buf = Buffer.new(dtype, n)
|
|
787
|
+
buf.write(Array.new(n, 1.0))
|
|
788
|
+
buf
|
|
789
|
+
end
|
|
790
|
+
|
|
791
|
+
def zeros_like(n, dtype) = Buffer.new(dtype, n) # zero-filled on allocation
|
|
792
|
+
|
|
793
|
+
# In-block reduction: call with values in sh[] and the result lands in
|
|
794
|
+
# sh[0]. To reuse sh afterwards, read it and then put a GPU_BARRIER in between.
|
|
795
|
+
def block_reduce(combine)
|
|
796
|
+
<<~SRC.chomp
|
|
797
|
+
GPU_BARRIER;
|
|
798
|
+
for (int s = #{BLOCK / 2}; s > 0; s >>= 1) {
|
|
799
|
+
if (lid < s) { sh[lid] = #{combine}; }
|
|
800
|
+
GPU_BARRIER;
|
|
801
|
+
}
|
|
802
|
+
SRC
|
|
803
|
+
end
|
|
804
|
+
|
|
805
|
+
def softmax_src(dtype)
|
|
806
|
+
t = ctype(dtype)
|
|
807
|
+
<<~KERNEL
|
|
808
|
+
GPU_KERNEL void nn_softmax(GPU_ARGS,
|
|
809
|
+
device const #{t}* x,
|
|
810
|
+
device #{t}* y,
|
|
811
|
+
constant int& rows,
|
|
812
|
+
constant int& cols) {
|
|
813
|
+
int row = GROUP_ID_X;
|
|
814
|
+
int lid = LOCAL_ID_X;
|
|
815
|
+
if (row >= rows) return;
|
|
816
|
+
GPU_SHARED float sh[#{BLOCK}];
|
|
817
|
+
int base = row * cols;
|
|
818
|
+
|
|
819
|
+
// 1. Row max. Without subtracting it, exp overflows on large logits.
|
|
820
|
+
float m = #{NEG_MAX};
|
|
821
|
+
for (int i = lid; i < cols; i += #{BLOCK}) m = fmax(m, #{read(dtype, 'x[base + i]')});
|
|
822
|
+
sh[lid] = m;
|
|
823
|
+
#{block_reduce('fmax(sh[lid], sh[lid + s])')}
|
|
824
|
+
m = sh[0];
|
|
825
|
+
GPU_BARRIER; // sh is reused by the next reduction, so wait until everyone has read it
|
|
826
|
+
|
|
827
|
+
// 2. Sum of exp
|
|
828
|
+
float total = 0.0f;
|
|
829
|
+
for (int i = lid; i < cols; i += #{BLOCK}) total += exp(#{read(dtype, 'x[base + i]')} - m);
|
|
830
|
+
sh[lid] = total;
|
|
831
|
+
#{block_reduce('sh[lid] + sh[lid + s]')}
|
|
832
|
+
float denom = sh[0];
|
|
833
|
+
|
|
834
|
+
// 3. Write back
|
|
835
|
+
for (int i = lid; i < cols; i += #{BLOCK}) {
|
|
836
|
+
y[base + i] = exp(#{read(dtype, 'x[base + i]')} - m) / denom;
|
|
837
|
+
}
|
|
838
|
+
}
|
|
839
|
+
KERNEL
|
|
840
|
+
end
|
|
841
|
+
|
|
842
|
+
def layernorm_src(dtype, affine)
|
|
843
|
+
t = ctype(dtype)
|
|
844
|
+
params = ["device const #{t}* x", "device #{t}* y"]
|
|
845
|
+
params += ["device const #{t}* gamma", "device const #{t}* beta"] if affine
|
|
846
|
+
params += ["constant int& rows", "constant int& cols", "constant float& eps"]
|
|
847
|
+
scale = affine ? " * #{read(dtype, 'gamma[i]')} + #{read(dtype, 'beta[i]')}" : ""
|
|
848
|
+
<<~KERNEL
|
|
849
|
+
GPU_KERNEL void nn_layernorm(GPU_ARGS,
|
|
850
|
+
#{params.join(",\n ")}) {
|
|
851
|
+
int row = GROUP_ID_X;
|
|
852
|
+
int lid = LOCAL_ID_X;
|
|
853
|
+
if (row >= rows) return;
|
|
854
|
+
GPU_SHARED float sh[#{BLOCK}];
|
|
855
|
+
int base = row * cols;
|
|
856
|
+
|
|
857
|
+
// 1. Mean
|
|
858
|
+
float total = 0.0f;
|
|
859
|
+
for (int i = lid; i < cols; i += #{BLOCK}) total += #{read(dtype, 'x[base + i]')};
|
|
860
|
+
sh[lid] = total;
|
|
861
|
+
#{block_reduce('sh[lid] + sh[lid + s]')}
|
|
862
|
+
float mean = sh[0] / (float)cols;
|
|
863
|
+
GPU_BARRIER;
|
|
864
|
+
|
|
865
|
+
// 2. Variance, recomputing deviations in a second pass (E[x^2] - E[x]^2 cancels catastrophically).
|
|
866
|
+
float sq = 0.0f;
|
|
867
|
+
for (int i = lid; i < cols; i += #{BLOCK}) {
|
|
868
|
+
float d = #{read(dtype, 'x[base + i]')} - mean;
|
|
869
|
+
sq += d * d;
|
|
870
|
+
}
|
|
871
|
+
sh[lid] = sq;
|
|
872
|
+
#{block_reduce('sh[lid] + sh[lid + s]')}
|
|
873
|
+
float inv = 1.0f / sqrt(sh[0] / (float)cols + eps);
|
|
874
|
+
|
|
875
|
+
// 3. Normalize
|
|
876
|
+
for (int i = lid; i < cols; i += #{BLOCK}) {
|
|
877
|
+
y[base + i] = (#{read(dtype, 'x[base + i]')} - mean) * inv#{scale};
|
|
878
|
+
}
|
|
879
|
+
}
|
|
880
|
+
KERNEL
|
|
881
|
+
end
|
|
882
|
+
|
|
883
|
+
# The same one-row-per-block reduction as layernorm, but with no mean
|
|
884
|
+
# subtraction it takes one pass (layernorm scans twice, for mean and variance).
|
|
885
|
+
def rmsnorm_src(dtype, affine)
|
|
886
|
+
t = ctype(dtype)
|
|
887
|
+
params = ["device const #{t}* x", "device #{t}* y"]
|
|
888
|
+
params += ["device const #{t}* gamma"] if affine
|
|
889
|
+
params += ["constant int& rows", "constant int& cols", "constant float& eps"]
|
|
890
|
+
scale = affine ? " * #{read(dtype, 'gamma[i]')}" : ""
|
|
891
|
+
<<~KERNEL
|
|
892
|
+
GPU_KERNEL void nn_rmsnorm(GPU_ARGS,
|
|
893
|
+
#{params.join(",\n ")}) {
|
|
894
|
+
int row = GROUP_ID_X;
|
|
895
|
+
int lid = LOCAL_ID_X;
|
|
896
|
+
if (row >= rows) return;
|
|
897
|
+
GPU_SHARED float sh[#{BLOCK}];
|
|
898
|
+
int base = row * cols;
|
|
899
|
+
|
|
900
|
+
// Sum of squares; always accumulate in float (in half it stops growing midway)
|
|
901
|
+
float sq = 0.0f;
|
|
902
|
+
for (int i = lid; i < cols; i += #{BLOCK}) {
|
|
903
|
+
float v = #{read(dtype, 'x[base + i]')};
|
|
904
|
+
sq += v * v;
|
|
905
|
+
}
|
|
906
|
+
sh[lid] = sq;
|
|
907
|
+
#{block_reduce('sh[lid] + sh[lid + s]')}
|
|
908
|
+
float inv = 1.0f / sqrt(sh[0] / (float)cols + eps);
|
|
909
|
+
|
|
910
|
+
for (int i = lid; i < cols; i += #{BLOCK}) {
|
|
911
|
+
y[base + i] = #{read(dtype, 'x[base + i]')} * inv#{scale};
|
|
912
|
+
}
|
|
913
|
+
}
|
|
914
|
+
KERNEL
|
|
915
|
+
end
|
|
916
|
+
|
|
917
|
+
def groupnorm_src(dtype, affine)
|
|
918
|
+
t = ctype(dtype)
|
|
919
|
+
params = ["device const #{t}* x", "device #{t}* y"]
|
|
920
|
+
params += ["device const #{t}* gamma", "device const #{t}* beta"] if affine
|
|
921
|
+
params += ["constant int& n", "constant int& c", "constant int& hw",
|
|
922
|
+
"constant int& groups", "constant float& eps"]
|
|
923
|
+
scale = affine ? " * #{read(dtype, 'gamma[ch]')} + #{read(dtype, 'beta[ch]')}" : ""
|
|
924
|
+
<<~KERNEL
|
|
925
|
+
GPU_KERNEL void nn_groupnorm(GPU_ARGS,
|
|
926
|
+
#{params.join(",\n ")}) {
|
|
927
|
+
// One block = one (image, group). The reduction spans all of (C/groups) * H * W.
|
|
928
|
+
int blk = GROUP_ID_X;
|
|
929
|
+
int lid = LOCAL_ID_X;
|
|
930
|
+
if (blk >= n * groups) return;
|
|
931
|
+
int img = blk / groups;
|
|
932
|
+
int grp = blk % groups;
|
|
933
|
+
int per = c / groups; // channels per group
|
|
934
|
+
int count = per * hw; // elements per group
|
|
935
|
+
int base = (img * c + grp * per) * hw;
|
|
936
|
+
GPU_SHARED float sh[#{BLOCK}];
|
|
937
|
+
|
|
938
|
+
float total = 0.0f;
|
|
939
|
+
for (int i = lid; i < count; i += #{BLOCK}) total += #{read(dtype, 'x[base + i]')};
|
|
940
|
+
sh[lid] = total;
|
|
941
|
+
#{block_reduce('sh[lid] + sh[lid + s]')}
|
|
942
|
+
float mean = sh[0] / (float)count;
|
|
943
|
+
GPU_BARRIER;
|
|
944
|
+
|
|
945
|
+
float sq = 0.0f;
|
|
946
|
+
for (int i = lid; i < count; i += #{BLOCK}) {
|
|
947
|
+
float d = #{read(dtype, 'x[base + i]')} - mean;
|
|
948
|
+
sq += d * d;
|
|
949
|
+
}
|
|
950
|
+
sh[lid] = sq;
|
|
951
|
+
#{block_reduce('sh[lid] + sh[lid + s]')}
|
|
952
|
+
float inv = 1.0f / sqrt(sh[0] / (float)count + eps);
|
|
953
|
+
|
|
954
|
+
for (int i = lid; i < count; i += #{BLOCK}) {
|
|
955
|
+
int ch = grp * per + i / hw; // gamma/beta are per channel
|
|
956
|
+
y[base + i] = (#{read(dtype, 'x[base + i]')} - mean) * inv#{scale};
|
|
957
|
+
}
|
|
958
|
+
}
|
|
959
|
+
KERNEL
|
|
960
|
+
end
|
|
961
|
+
|
|
962
|
+
# Every activation is one thread per element; only the expression differs.
|
|
963
|
+
ACTIVATIONS = {
|
|
964
|
+
relu: "fmax(v, 0.0f)",
|
|
965
|
+
# Writing 1/(1+exp(-v)) directly overflows exp for very negative v. This
|
|
966
|
+
# form uses only exp(-|v|), so it is safe for either sign.
|
|
967
|
+
sigmoid: "(v >= 0.0f ? 1.0f / (1.0f + exp(-v)) : exp(v) / (1.0f + exp(v)))",
|
|
968
|
+
silu: "v * (v >= 0.0f ? 1.0f / (1.0f + exp(-v)) : exp(v) / (1.0f + exp(v)))",
|
|
969
|
+
gelu_tanh: "0.5f * v * (1.0f + tanh(0.7978845608028654f * (v + 0.044715f * v * v * v)))",
|
|
970
|
+
gelu_exact: "0.5f * v * (1.0f + GPU_ERF(v * 0.7071067811865476f))",
|
|
971
|
+
}.freeze
|
|
972
|
+
|
|
973
|
+
def activation(x, kind)
|
|
974
|
+
shaped!(x, kind.to_s)
|
|
975
|
+
return Graph.record_nn(:activation, [x], args: [kind], shape: x.shape, dtype: x.dtype) if Graph.symbolic?(x)
|
|
976
|
+
return cpu_activation(x, kind) if Tilework.host_fallback?
|
|
977
|
+
|
|
978
|
+
expr = ACTIVATIONS.fetch(kind)
|
|
979
|
+
t = ctype(x.dtype)
|
|
980
|
+
src = <<~KERNEL
|
|
981
|
+
GPU_KERNEL void nn_#{kind}(GPU_ARGS,
|
|
982
|
+
device const #{t}* x,
|
|
983
|
+
device #{t}* y,
|
|
984
|
+
constant int& n) {
|
|
985
|
+
int i = GLOBAL_ID_X;
|
|
986
|
+
if (i < n) {
|
|
987
|
+
float v = #{read(x.dtype, 'x[i]')};
|
|
988
|
+
y[i] = #{expr};
|
|
989
|
+
}
|
|
990
|
+
}
|
|
991
|
+
KERNEL
|
|
992
|
+
out = Ops.allocate_output(x.dtype, x.length, x.shape)
|
|
993
|
+
kernel = Tilework.compile(src, name: "nn_#{kind}")
|
|
994
|
+
Ops.launch_kernel(kernel, grid: (x.length / 256.0).ceil, block: 256, args: [x, out, x.length])
|
|
995
|
+
out
|
|
996
|
+
end
|
|
997
|
+
|
|
998
|
+
def transpose_src(dtype)
|
|
999
|
+
t = ctype(dtype)
|
|
1000
|
+
<<~KERNEL
|
|
1001
|
+
GPU_KERNEL void nn_transpose(GPU_ARGS,
|
|
1002
|
+
device const #{t}* x,
|
|
1003
|
+
device #{t}* y,
|
|
1004
|
+
constant int& rows,
|
|
1005
|
+
constant int& cols) {
|
|
1006
|
+
// Indexed by the input [rows, cols] position; the output is [cols, rows].
|
|
1007
|
+
int r = GLOBAL_ID_X;
|
|
1008
|
+
int c = GLOBAL_ID_Y;
|
|
1009
|
+
if (r < rows && c < cols) y[c * rows + r] = x[r * cols + c];
|
|
1010
|
+
}
|
|
1011
|
+
KERNEL
|
|
1012
|
+
end
|
|
1013
|
+
|
|
1014
|
+
# --- Reference implementations for the :cpu backend ----------------------
|
|
1015
|
+
#
|
|
1016
|
+
# :cpu has no generator for shared memory, so these compute on the host.
|
|
1017
|
+
# Slow, but they also **produce the expected values for tests** (GPU
|
|
1018
|
+
# results are compared against them).
|
|
1019
|
+
|
|
1020
|
+
def rebuild(x, values, shape = x.shape)
|
|
1021
|
+
out = Ops.allocate_output(x.dtype, values.length, shape)
|
|
1022
|
+
out.write(values)
|
|
1023
|
+
out
|
|
1024
|
+
end
|
|
1025
|
+
|
|
1026
|
+
def cpu_softmax(x, rows, cols)
|
|
1027
|
+
values = x.to_a.each_slice(cols).flat_map do |row|
|
|
1028
|
+
m = row.max
|
|
1029
|
+
exps = row.map { |v| Math.exp(v - m) }
|
|
1030
|
+
total = exps.sum
|
|
1031
|
+
exps.map { |e| e / total }
|
|
1032
|
+
end
|
|
1033
|
+
rebuild(x, values)
|
|
1034
|
+
end
|
|
1035
|
+
|
|
1036
|
+
def cpu_layernorm(x, rows, cols, gamma, beta, eps)
|
|
1037
|
+
g = gamma&.to_a
|
|
1038
|
+
b = beta&.to_a
|
|
1039
|
+
values = x.to_a.each_slice(cols).flat_map do |row|
|
|
1040
|
+
mean = row.sum / cols.to_f
|
|
1041
|
+
var = row.sum { |v| (v - mean)**2 } / cols.to_f
|
|
1042
|
+
inv = 1.0 / Math.sqrt(var + eps)
|
|
1043
|
+
row.each_with_index.map do |v, i|
|
|
1044
|
+
((v - mean) * inv * (g ? g[i] : 1.0)) + (b ? b[i] : 0.0)
|
|
1045
|
+
end
|
|
1046
|
+
end
|
|
1047
|
+
rebuild(x, values)
|
|
1048
|
+
end
|
|
1049
|
+
|
|
1050
|
+
def cpu_rmsnorm(x, rows, cols, gamma, eps)
|
|
1051
|
+
g = gamma&.to_a
|
|
1052
|
+
values = x.to_a.each_slice(cols).flat_map do |row|
|
|
1053
|
+
inv = 1.0 / Math.sqrt((row.sum { |v| v * v } / cols.to_f) + eps)
|
|
1054
|
+
row.each_with_index.map { |v, i| v * inv * (g ? g[i] : 1.0) }
|
|
1055
|
+
end
|
|
1056
|
+
rebuild(x, values)
|
|
1057
|
+
end
|
|
1058
|
+
|
|
1059
|
+
def cpu_groupnorm(x, shape, groups, gamma, beta, eps)
|
|
1060
|
+
n, c, h, w = shape
|
|
1061
|
+
hw = h * w
|
|
1062
|
+
per = c / groups
|
|
1063
|
+
g = gamma&.to_a
|
|
1064
|
+
b = beta&.to_a
|
|
1065
|
+
flat = x.to_a
|
|
1066
|
+
out = Array.new(flat.length)
|
|
1067
|
+
n.times do |img|
|
|
1068
|
+
groups.times do |grp|
|
|
1069
|
+
base = ((img * c) + (grp * per)) * hw
|
|
1070
|
+
slice = flat[base, per * hw]
|
|
1071
|
+
mean = slice.sum / slice.length.to_f
|
|
1072
|
+
var = slice.sum { |v| (v - mean)**2 } / slice.length.to_f
|
|
1073
|
+
inv = 1.0 / Math.sqrt(var + eps)
|
|
1074
|
+
slice.each_with_index do |v, i|
|
|
1075
|
+
ch = (grp * per) + (i / hw)
|
|
1076
|
+
out[base + i] = ((v - mean) * inv * (g ? g[ch] : 1.0)) + (b ? b[ch] : 0.0)
|
|
1077
|
+
end
|
|
1078
|
+
end
|
|
1079
|
+
end
|
|
1080
|
+
rebuild(x, out)
|
|
1081
|
+
end
|
|
1082
|
+
|
|
1083
|
+
def cpu_activation(x, kind)
|
|
1084
|
+
sigmoid = ->(v) { v >= 0 ? 1.0 / (1.0 + Math.exp(-v)) : Math.exp(v) / (1.0 + Math.exp(v)) }
|
|
1085
|
+
fn = case kind
|
|
1086
|
+
when :relu then ->(v) { v.positive? ? v : 0.0 }
|
|
1087
|
+
when :sigmoid then sigmoid
|
|
1088
|
+
when :silu then ->(v) { v * sigmoid.(v) }
|
|
1089
|
+
when :gelu_tanh
|
|
1090
|
+
->(v) { 0.5 * v * (1.0 + Math.tanh(0.7978845608028654 * (v + (0.044715 * v * v * v)))) }
|
|
1091
|
+
when :gelu_exact then ->(v) { 0.5 * v * (1.0 + Math.erf(v * 0.7071067811865476)) }
|
|
1092
|
+
end
|
|
1093
|
+
rebuild(x, x.to_a.map(&fn))
|
|
1094
|
+
end
|
|
1095
|
+
|
|
1096
|
+
def cpu_transpose(x, rows, cols)
|
|
1097
|
+
flat = x.to_a
|
|
1098
|
+
values = Array.new(rows * cols)
|
|
1099
|
+
cols.times { |c| rows.times { |r| values[(c * rows) + r] = flat[(r * cols) + c] } }
|
|
1100
|
+
rebuild(x, values, [cols, rows])
|
|
1101
|
+
end
|
|
1102
|
+
end
|
|
1103
|
+
end
|