tilework 0.18.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE +21 -0
  3. data/lib/tilework/backend/cpu/dsl_codegen.rb +509 -0
  4. data/lib/tilework/backend/cpu/ops_codegen.rb +451 -0
  5. data/lib/tilework/backend/cpu.rb +156 -0
  6. data/lib/tilework/buffer.rb +753 -0
  7. data/lib/tilework/capabilities.rb +183 -0
  8. data/lib/tilework/compat/cuda.h +66 -0
  9. data/lib/tilework/compat/hip.h +35 -0
  10. data/lib/tilework/compat/metal.h +45 -0
  11. data/lib/tilework/conformance.rb +2296 -0
  12. data/lib/tilework/core_ext.rb +21 -0
  13. data/lib/tilework/dialect.rb +67 -0
  14. data/lib/tilework/dsl.rb +722 -0
  15. data/lib/tilework/dtype/bfloat.rb +31 -0
  16. data/lib/tilework/dtype/cast.rb +88 -0
  17. data/lib/tilework/dtype/half.rb +98 -0
  18. data/lib/tilework/graph/builder.rb +1073 -0
  19. data/lib/tilework/graph/fusion.rb +520 -0
  20. data/lib/tilework/graph/grad.rb +443 -0
  21. data/lib/tilework/graph/high_level.rb +266 -0
  22. data/lib/tilework/graph/kernels.rb +403 -0
  23. data/lib/tilework/graph/node.rb +27 -0
  24. data/lib/tilework/graph/op.rb +290 -0
  25. data/lib/tilework/graph/optim.rb +183 -0
  26. data/lib/tilework/graph/plan.rb +1086 -0
  27. data/lib/tilework/graph/pool.rb +178 -0
  28. data/lib/tilework/graph/region.rb +210 -0
  29. data/lib/tilework/graph/residency.rb +449 -0
  30. data/lib/tilework/graph/state.rb +64 -0
  31. data/lib/tilework/graph/tensor.rb +124 -0
  32. data/lib/tilework/graph.rb +213 -0
  33. data/lib/tilework/kernel.rb +163 -0
  34. data/lib/tilework/math/conv.rb +270 -0
  35. data/lib/tilework/math/fft.rb +172 -0
  36. data/lib/tilework/math/nn.rb +1103 -0
  37. data/lib/tilework/math/philox.rb +194 -0
  38. data/lib/tilework/math/random.rb +232 -0
  39. data/lib/tilework/math/tensor_core.rb +269 -0
  40. data/lib/tilework/ops.rb +682 -0
  41. data/lib/tilework/runtime/offload.rb +498 -0
  42. data/lib/tilework/runtime/submission.rb +163 -0
  43. data/lib/tilework/runtime/telemetry.rb +112 -0
  44. data/lib/tilework/scope.rb +33 -0
  45. data/lib/tilework/version.rb +5 -0
  46. data/lib/tilework.rb +345 -0
  47. metadata +92 -0
@@ -0,0 +1,1103 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Tilework
4
+ # Neural-network operations implemented as fused kernels.
5
+ #
6
+ # Why the library owns them (when `Ops` combinations could express them):
7
+ #
8
+ # 1. **Ops multiply.** Each operation has a fixed cost of about 90 µs, so
9
+ # writing softmax as `exp` / `sum` / `/` takes several round trips per
10
+ # row; fused, it is one kernel.
11
+ # 2. **Everyone gets the launch domain wrong.** Omitting `over:` in a
12
+ # row-wise reduction makes every column thread compute the whole row (6.9x
13
+ # waste at 346x346). External applications would make the same mistake.
14
+ #
15
+ # The implementation is **one row (or one group) per block**, with the
16
+ # block's threads cooperating through shared memory. Arithmetic is always in
17
+ # float; float16 buffers are storage only (the output dtype follows the input).
18
+ module NN
19
+ BLOCK = 256
20
+
21
+ # Upper bounds on K/V columns held per flash-attention block and on head
22
+ # dimension. K/V/Q and the output accumulator total about 17 KiB, well
23
+ # within the 32 KiB of threadgroup/shared memory on both Metal and CUDA.
24
+ # Heads above the bound fall back to the GEMM path (typical head_dim is
25
+ # 64 or 128).
26
+ FLASH_TILE_KEYS = 16
27
+ FLASH_MAX_HEAD_DIM = 128
28
+ FLASH_MIN_SCORES = 4096
29
+
30
+ # Reduction identities. The float minimum avoids -INFINITY (a finite lower
31
+ # bound suffices rather than guaranteeing identical spelling and behavior
32
+ # across MSL/CUDA).
33
+ NEG_MAX = "-3.402823466e+38f"
34
+
35
+ module_function
36
+
37
+ # Applies a numerically stable softmax over the final dimension.
38
+ #
39
+ # @param x [Buffer, Graph::Tensor] tensor with at least two dimensions
40
+ # @return [Buffer, Graph::Tensor] probabilities with the same shape and dtype
41
+ #
42
+ # Any rank from 2D up — all leading dims fold into rows, and the shape
43
+ # comes back unchanged.
44
+ #
45
+ # probs = Tilework::NN.softmax(logits) # [rows, cols] -> [rows, cols]
46
+ # probs = Tilework::NN.softmax(scores) # [heads, n, m] works the same
47
+ #
48
+ # The max is subtracted before exp, so large logits do not overflow
49
+ # (`exp(1000)` is inf).
50
+ def softmax(x)
51
+ rows, cols = rowwise_dims!(x, "softmax")
52
+ return Graph.record_nn(:softmax, [x], shape: x.shape, dtype: x.dtype) if Graph.symbolic?(x)
53
+ return cpu_softmax(x, rows, cols) if Tilework.host_fallback?
54
+
55
+ out = Ops.allocate_output(x.dtype, x.length, x.shape)
56
+ kernel = Tilework.compile(softmax_src(x.dtype), name: "nn_softmax")
57
+ Ops.launch_kernel(kernel, grid: rows, block: BLOCK, args: [x, out, rows, cols])
58
+ out
59
+ end
60
+
61
+ # Applies layer normalization over the final dimension.
62
+ #
63
+ # @param x [Buffer, Graph::Tensor]
64
+ # @param gamma [Buffer, Graph::Tensor, nil] optional scale of length `x.shape.last`
65
+ # @param beta [Buffer, Graph::Tensor, nil] optional bias of length `x.shape.last`
66
+ # @param eps [Float] numerical stability term
67
+ # @return [Buffer, Graph::Tensor]
68
+ #
69
+ # A standard transformer component. Like softmax, any rank from 2D up
70
+ # ([batch, tokens, d] passes as is).
71
+ #
72
+ # y = Tilework::NN.layernorm(x, gamma: g, beta: b) # g, b of length cols
73
+ #
74
+ # gamma / beta may be omitted (omitting both means no affine transform).
75
+ def layernorm(x, gamma: nil, beta: nil, eps: 1e-5)
76
+ rows, cols = rowwise_dims!(x, "layernorm")
77
+ check_affine!(gamma, beta, cols, x.dtype)
78
+ if Graph.symbolic?(x, gamma, beta)
79
+ return Graph.record_nn(:layernorm, [x], tensors: { gamma: gamma, beta: beta },
80
+ opts: { eps: eps }, shape: x.shape, dtype: x.dtype)
81
+ end
82
+ return cpu_layernorm(x, rows, cols, gamma, beta, eps) if Tilework.host_fallback?
83
+
84
+ affine = !gamma.nil? || !beta.nil?
85
+ out = Ops.allocate_output(x.dtype, x.length, x.shape)
86
+ kernel = Tilework.compile(layernorm_src(x.dtype, affine), name: "nn_layernorm")
87
+ args = [x, out]
88
+ temporaries = []
89
+ if affine
90
+ scale = gamma || ones_like(cols, x.dtype).tap { |buffer| temporaries << buffer }
91
+ bias = beta || zeros_like(cols, x.dtype).tap { |buffer| temporaries << buffer }
92
+ args += [scale, bias]
93
+ end
94
+ begin
95
+ Ops.launch_kernel(kernel, grid: rows, block: BLOCK, args: args + [rows, cols, eps.to_f])
96
+ ensure
97
+ temporaries.each(&:free)
98
+ end
99
+ out
100
+ end
101
+
102
+ # Applies RMS normalization over the final dimension.
103
+ #
104
+ # @param x [Buffer, Graph::Tensor]
105
+ # @param gamma [Buffer, Graph::Tensor, nil] optional scale of length `x.shape.last`
106
+ # @param eps [Float] numerical stability term
107
+ # @return [Buffer, Graph::Tensor]
108
+ #
109
+ # Divides by the root mean square without subtracting the mean, as used by
110
+ # T5 / LLaMA-family models.
111
+ #
112
+ # y = Tilework::NN.rmsnorm(x, gamma: g, eps: 1e-6) # g of length cols
113
+ #
114
+ # Unlike layernorm **there is no mean subtraction**, so no shift (beta)
115
+ # either. The sum of squares accumulates in float (in half it stops
116
+ # growing around 2048).
117
+ def rmsnorm(x, gamma: nil, eps: 1e-6)
118
+ rows, cols = rowwise_dims!(x, "rmsnorm")
119
+ check_affine!(gamma, nil, cols, x.dtype)
120
+ if Graph.symbolic?(x, gamma)
121
+ return Graph.record_nn(:rmsnorm, [x], tensors: { gamma: gamma },
122
+ opts: { eps: eps }, shape: x.shape, dtype: x.dtype)
123
+ end
124
+ return cpu_rmsnorm(x, rows, cols, gamma, eps) if Tilework.host_fallback?
125
+
126
+ out = Ops.allocate_output(x.dtype, x.length, x.shape)
127
+ kernel = Tilework.compile(rmsnorm_src(x.dtype, !gamma.nil?), name: "nn_rmsnorm")
128
+ args = [x, out]
129
+ args += [gamma] if gamma
130
+ Ops.launch_kernel(kernel, grid: rows, block: BLOCK, args: args + [rows, cols, eps.to_f])
131
+ out
132
+ end
133
+
134
+ # Applies group normalization to an NCHW tensor.
135
+ #
136
+ # @param x [Buffer, Graph::Tensor] `[N, C, H, W]` tensor
137
+ # @param groups [Integer] positive divisor of `C`
138
+ # @param gamma [Buffer, Graph::Tensor, nil] optional per-channel scale
139
+ # @param beta [Buffer, Graph::Tensor, nil] optional per-channel bias
140
+ # @param eps [Float] numerical stability term
141
+ # @return [Buffer, Graph::Tensor]
142
+ #
143
+ # Splits channels into `groups` and normalizes each group over all of
144
+ # (C/groups) x H x W — used heavily in diffusion-model UNets.
145
+ #
146
+ # y = Tilework::NN.groupnorm(x, groups: 32, gamma: g, beta: b)
147
+ #
148
+ # gamma / beta are **per channel** (length C). groups == C is instance
149
+ # norm; groups == 1 is equivalent to layer norm.
150
+ def groupnorm(x, groups:, gamma: nil, beta: nil, eps: 1e-5)
151
+ n, c, h, w = nchw_dims!(x, "groupnorm")
152
+ unless groups.is_a?(Integer) && groups.positive? && (c % groups).zero?
153
+ raise ArgumentError, "groups #{groups.inspect} must be a positive divisor of #{c} channels"
154
+ end
155
+
156
+ check_affine!(gamma, beta, c, x.dtype)
157
+ if Graph.symbolic?(x, gamma, beta)
158
+ return Graph.record_nn(:groupnorm, [x], tensors: { gamma: gamma, beta: beta },
159
+ opts: { groups: groups, eps: eps },
160
+ shape: x.shape, dtype: x.dtype)
161
+ end
162
+ return cpu_groupnorm(x, [n, c, h, w], groups, gamma, beta, eps) if Tilework.host_fallback?
163
+
164
+ affine = !gamma.nil? || !beta.nil?
165
+ out = Ops.allocate_output(x.dtype, x.length, x.shape)
166
+ kernel = Tilework.compile(groupnorm_src(x.dtype, affine), name: "nn_groupnorm")
167
+ args = [x, out]
168
+ temporaries = []
169
+ if affine
170
+ scale = gamma || ones_like(c, x.dtype).tap { |buffer| temporaries << buffer }
171
+ bias = beta || zeros_like(c, x.dtype).tap { |buffer| temporaries << buffer }
172
+ args += [scale, bias]
173
+ end
174
+ begin
175
+ Ops.launch_kernel(kernel, grid: n * groups, block: BLOCK, args: args + [n, c, h * w, groups, eps.to_f])
176
+ ensure
177
+ temporaries.each(&:free)
178
+ end
179
+ out
180
+ end
181
+
182
+ # --- Activations ---------------------------------------------------------
183
+
184
+ # Applies the SiLU (Swish) activation.
185
+ # @param x [Buffer, Graph::Tensor]
186
+ # @return [Buffer, Graph::Tensor]
187
+ #
188
+ # x * sigmoid(x), standard in diffusion models and recent LLMs. sigmoid is
189
+ # rearranged to use exp(-|x|), so very negative x does not overflow (a naive
190
+ # 1/(1+exp(-x)) gives exp(100) = inf at x = -100).
191
+ def silu(x) = activation(x, :silu)
192
+
193
+ # Applies GELU using the tanh approximation by default.
194
+ # @param x [Buffer, Graph::Tensor]
195
+ # @param exact [Boolean] use the slower `erf` formulation
196
+ # @return [Buffer, Graph::Tensor]
197
+ #
198
+ # The tanh approximation is the default of GPT-2 and many implementations.
199
+ def gelu(x, exact: false) = activation(x, exact ? :gelu_exact : :gelu_tanh)
200
+
201
+ # Applies the ReLU activation.
202
+ # @param x [Buffer, Graph::Tensor]
203
+ # @return [Buffer, Graph::Tensor]
204
+ def relu(x) = activation(x, :relu)
205
+
206
+ # Applies the sigmoid activation.
207
+ # @param x [Buffer, Graph::Tensor]
208
+ # @return [Buffer, Graph::Tensor]
209
+ def sigmoid(x) = activation(x, :sigmoid)
210
+
211
+ # --- Losses ------------------------------------------------------------
212
+ #
213
+ # The entry point of training. **They end in a reduction**, so like
214
+ # `Ops.reduce` there are two ways to receive the result (see #reduce in ops.rb):
215
+ #
216
+ # NN.mse(y, t) # => Float; the queue drains here
217
+ # NN.mse(y, t, device: true) # => a one-element Buffer, left on the device
218
+ # NN.mse(y, t, reduction: :none) # => per-element / per-row loss Buffer
219
+ #
220
+ # In a graph they always run in the device: true form (dropping to the host
221
+ # would end the plan). Backward (#22) requires that form, so **keeping the
222
+ # loss a Tensor** is the point.
223
+ #
224
+ # reduction is :mean (default) / :sum / :none. It changes the gradient's
225
+ # magnitude, so stating it explicitly beats relying on the default.
226
+
227
+ LOSS_REDUCTIONS = %i[mean sum none].freeze
228
+
229
+ # Calculates mean squared error.
230
+ #
231
+ # @param pred [Buffer, Graph::Tensor]
232
+ # @param target [Buffer, Graph::Tensor] same shape and dtype as `pred`
233
+ # @param reduction [:mean, :sum, :none] reduction mode
234
+ # @param device [Boolean] return a one-element buffer instead of a host scalar
235
+ # @return [Float, Buffer, Graph::Tensor]
236
+ #
237
+ # loss = Tilework::NN.mse(pred, target)
238
+ #
239
+ # reduction: :none returns per-element squared errors (the input's shape).
240
+ def mse(pred, target, reduction: :mean, device: false)
241
+ check_loss_pair!(pred, target)
242
+ check_reduction!(reduction)
243
+ if Graph.symbolic?(pred, target)
244
+ return Graph.record_nn(:mse, [pred, target], opts: { reduction: reduction, device: true },
245
+ shape: loss_shape(pred.shape, reduction, rowwise: false),
246
+ dtype: loss_dtype(pred.dtype))
247
+ end
248
+
249
+ reduce_loss(squared_error(pred, target), reduction, device: device)
250
+ end
251
+
252
+ # Calculates cross entropy from unnormalized logits and int32 class labels.
253
+ #
254
+ # @param logits [Buffer, Graph::Tensor] tensor whose final dimension is classes
255
+ # @param labels [Buffer, Graph::Tensor] int32 tensor with the final logits dimension removed
256
+ # @param reduction [:mean, :sum, :none] reduction mode
257
+ # @param device [Boolean] return a one-element buffer instead of a host scalar
258
+ # @return [Float, Buffer, Graph::Tensor]
259
+ #
260
+ # loss = Tilework::NN.cross_entropy(logits, labels)
261
+ #
262
+ # **Takes logits** (values before softmax). It computes **log-sum-exp in one
263
+ # kernel** rather than log after softmax: a naive `log(softmax(x))` hits
264
+ # -inf where a probability underflows to 0, and its gradient does not take
265
+ # the plain (softmax(x) - onehot) / n form.
266
+ #
267
+ # Out-of-range labels are clamped to the ends (the same rule as `gather`) —
268
+ # an out-of-range read on a GPU is undefined rather than merely wrong, so
269
+ # clamping beats breaking silently.
270
+ def cross_entropy(logits, labels, reduction: :mean, device: false)
271
+ rows, cols = rowwise_dims!(logits, "cross_entropy")
272
+ check_labels!(labels, logits, rows)
273
+ check_reduction!(reduction)
274
+ if Graph.symbolic?(logits, labels)
275
+ return Graph.record_nn(:cross_entropy, [logits, labels],
276
+ opts: { reduction: reduction, device: true },
277
+ shape: loss_shape(logits.shape, reduction, rowwise: true),
278
+ dtype: loss_dtype(logits.dtype))
279
+ end
280
+
281
+ rowwise = if Tilework.host_fallback?
282
+ cpu_cross_entropy(logits, labels, rows, cols)
283
+ else
284
+ out = Ops.allocate_output(loss_dtype(logits.dtype), rows, [rows])
285
+ kernel = Tilework.compile(cross_entropy_src(logits.dtype), name: "nn_cross_entropy")
286
+ Ops.launch_kernel(kernel, grid: rows, block: BLOCK, args: [logits, labels, out, rows, cols])
287
+ out
288
+ end
289
+ reduce_loss(rowwise, reduction, device: device)
290
+ end
291
+
292
+ # Transposes a two-dimensional tensor.
293
+ #
294
+ # @param x [Buffer, Graph::Tensor] `[rows, columns]` tensor
295
+ # @return [Buffer, Graph::Tensor] `[columns, rows]` tensor
296
+ #
297
+ # **Launches over the output's domain.** Launching over the input's domain
298
+ # and writing `out[c, r]` leaves the tail of the output unwritten when rows
299
+ # and columns differ.
300
+ def transpose(x)
301
+ rows, cols = matrix_dims!(x, "transpose")
302
+ return Graph.record_nn(:transpose, [x], shape: [cols, rows], dtype: x.dtype) if Graph.symbolic?(x)
303
+ return cpu_transpose(x, rows, cols) if Tilework.host_fallback?
304
+
305
+ out = Ops.allocate_output(x.dtype, x.length, [cols, rows])
306
+ kernel = Tilework.compile(transpose_src(x.dtype), name: "nn_transpose")
307
+ # The grid covers the output [cols, rows]: x is the column and y the row, as in the DSL.
308
+ grid = [(rows / 16.0).ceil, (cols / 16.0).ceil]
309
+ Ops.launch_kernel(kernel, grid: grid, block: [16, 16], args: [x, out, rows, cols])
310
+ out
311
+ end
312
+
313
+ # Calculates scaled dot-product multi-head attention.
314
+ #
315
+ # @param q [Buffer, Graph::Tensor] query matrix `[n, d]`
316
+ # @param k [Buffer, Graph::Tensor] key matrix `[m, d]`
317
+ # @param v [Buffer, Graph::Tensor] value matrix `[m, d]`
318
+ # @param heads [Integer] number of heads dividing `d`
319
+ # @param scale [Float, nil] score scale; defaults to `1 / sqrt(d / heads)`
320
+ # @param bias [Buffer, Graph::Tensor, nil] optional broadcastable score bias
321
+ # @return [Buffer, Graph::Tensor] attention output `[n, d]`
322
+ #
323
+ # q: [n, d], k/v: [m, d], d = heads * head_dim; returns [n, d].
324
+ #
325
+ # Heads are **small, evenly spaced matrices** inside `[*, d]`, so with a
326
+ # strided batched GEMM they never need slicing out or writing back: pass
327
+ # the original width d as ld and head_dim as the stride and compute on
328
+ # views. The score matrix [heads, n, m] is contiguous, so viewed as
329
+ # [heads*n, m] **one softmax covers every head**.
330
+ #
331
+ # For one 16-head attention this cuts launches from about 100 to 4.
332
+ #
333
+ # scale multiplies the scores (default 1/sqrt(head_dim)). It goes to GEMM's
334
+ # alpha, so it is **free**; writing `* (1.0/Math.sqrt(dh))` yourself costs
335
+ # another full round trip over the score matrix.
336
+ #
337
+ # bias is added to the scores **before** softmax, shaped [heads * n, m]
338
+ # (each head's [n, m] stacked vertically — the scores' own layout). T5's
339
+ # relative position bias and additive padding masks are this.
340
+ #
341
+ # If padding sits at the end, **passing k/v cut to the valid length** is
342
+ # faster than masking with bias (it removes the computation itself).
343
+ def multi_head_attention(q, k, v, heads:, scale: nil, bias: nil)
344
+ n, d = matrix_dims!(q, "multi_head_attention (q)")
345
+ m, dk = matrix_dims!(k, "multi_head_attention (k)")
346
+ mv, dv = matrix_dims!(v, "multi_head_attention (v)")
347
+ unless d == dk && d == dv && m == mv
348
+ raise ArgumentError,
349
+ "q [#{n}, #{d}] に対し k は [m, #{d}]、v は [m, #{d}] でなければならない " \
350
+ "(k: [#{m}, #{dk}], v: [#{mv}, #{dv}])"
351
+ end
352
+ raise ArgumentError, "d (#{d}) は heads (#{heads}) で割り切れなければならない" unless (d % heads).zero?
353
+
354
+ dh = d / heads
355
+ scale ||= 1.0 / Math.sqrt(dh)
356
+ check_attention_bias!(bias, heads, n, m)
357
+ if Graph.symbolic?(q, k, v, bias)
358
+ # scale is fixed here before becoming an attribute (decided when the plan is built).
359
+ return Graph.record_nn(:multi_head_attention, [q, k, v], tensors: { bias: bias },
360
+ opts: { heads: heads, scale: scale },
361
+ shape: [n, d], dtype: :float32)
362
+ end
363
+ return flash_attention(q, k, v, heads, dh, n, m, d, scale, bias) if flash_attention?(dh, n, m)
364
+
365
+ unless TensorCore.gemm_batched? && q.dtype == k.dtype && q.dtype == v.dtype &&
366
+ Tilework.capabilities.gemm_dtypes.include?(q.dtype)
367
+ return attention_per_head(q, k, v, heads, dh, n, m, d, scale, bias)
368
+ end
369
+
370
+ # Column-major Cc = Kc^T * Ac. Kc/Ac are views inside the parent [*, d], so ld = d.
371
+ scores = Buffer.new(:float32, heads * n * m, shape: [heads * n, m])
372
+ TensorCore.gemm_batched(k, q, scores,
373
+ m: m, k: dh, n: n,
374
+ lda: d, ldb: d, ldc: m,
375
+ stride_a: dh, stride_b: dh, stride_c: n * m,
376
+ batch: heads, trans_a: true, alpha: scale)
377
+ if bias
378
+ biased = scores + bias
379
+ scores.free
380
+ scores = biased
381
+ end
382
+ probs = softmax(scores)
383
+ scores.free
384
+
385
+ # Column-major ctx^T = Vc * Pc. With ldc = d and stride_c = dh it writes
386
+ # **directly** into the h-th section of the output [n, d] (no write-back kernel).
387
+ probs_for_gemm = if Ops::LOW_PRECISION_DTYPES.include?(v.dtype)
388
+ probs.cast(v.dtype)
389
+ else
390
+ probs
391
+ end
392
+ out = Ops.allocate_output(:float32, n * d, [n, d])
393
+ TensorCore.gemm_batched(v, probs_for_gemm, out,
394
+ m: dh, k: m, n: n,
395
+ lda: d, ldb: m, ldc: d,
396
+ stride_a: dh, stride_b: n * m, stride_c: dh,
397
+ batch: heads)
398
+ probs_for_gemm.free unless probs_for_gemm.equal?(probs)
399
+ probs.free
400
+ out
401
+ rescue StandardError
402
+ [scores, probs_for_gemm, probs, out].compact.uniq.each { |buffer| buffer.free unless buffer.freed? }
403
+ raise
404
+ end
405
+
406
+ # ========================================================================
407
+ # Internal
408
+ # ========================================================================
409
+
410
+ # Attention that never puts the [heads*n, m] scores in VRAM. Each block is
411
+ # "one query row of one head": K/V tiles go through shared memory while an
412
+ # online softmax updates the running max / running sum and the output
413
+ # accumulator (the same numerical skeleton as FlashAttention). The working
414
+ # set is O(tile * dh), and no Buffer proportional to n*m is allocated.
415
+ def flash_attention(q, k, v, heads, dh, n, m, d, scale, bias)
416
+ out = Ops.allocate_output(:float32, n * d, [n, d])
417
+ kernel = Tilework.compile(flash_attention_src(q.dtype, k.dtype, v.dtype, bias&.dtype),
418
+ name: "nn_flash_attention")
419
+ args = [q, k, v]
420
+ args << bias if bias
421
+ Ops.launch_kernel(kernel, grid: heads * n, block: BLOCK,
422
+ args: args + [out, heads, n, m, d, dh, scale.to_f])
423
+ out
424
+ rescue StandardError
425
+ out&.free
426
+ raise
427
+ end
428
+
429
+ def flash_attention?(dh, n, m)
430
+ !Tilework.host_fallback? && Tilework.capabilities.dialect &&
431
+ dh <= FLASH_MAX_HEAD_DIM && (n * m) >= FLASH_MIN_SCORES
432
+ end
433
+
434
+ def flash_attention_src(q_dtype, k_dtype, v_dtype, bias_dtype)
435
+ qt = ctype(q_dtype)
436
+ kt = ctype(k_dtype)
437
+ vt = ctype(v_dtype)
438
+ bias_param = bias_dtype ? ",\n device const #{ctype(bias_dtype)}* bias" : ""
439
+ bias_read = bias_dtype ? " + #{read(bias_dtype, 'bias[(head * n + query) * m + key]')}" : ""
440
+ <<~KERNEL
441
+ GPU_KERNEL void nn_flash_attention(GPU_ARGS,
442
+ device const #{qt}* q,
443
+ device const #{kt}* k,
444
+ device const #{vt}* v#{bias_param},
445
+ device float* out,
446
+ constant int& heads,
447
+ constant int& n,
448
+ constant int& m,
449
+ constant int& d,
450
+ constant int& dh,
451
+ constant float& scale) {
452
+ int row = GROUP_ID_X;
453
+ int lid = LOCAL_ID_X;
454
+ int head = row / n;
455
+ int query = row % n;
456
+ if (head >= heads) return;
457
+
458
+ GPU_SHARED float dot[#{BLOCK}];
459
+ GPU_SHARED float qtile[#{FLASH_MAX_HEAD_DIM}];
460
+ GPU_SHARED float ktile[#{FLASH_TILE_KEYS * FLASH_MAX_HEAD_DIM}];
461
+ GPU_SHARED float vtile[#{FLASH_TILE_KEYS * FLASH_MAX_HEAD_DIM}];
462
+ GPU_SHARED float acc[#{FLASH_MAX_HEAD_DIM}];
463
+ // state = max, sum, old-scale, new-score-weight
464
+ GPU_SHARED float state[4];
465
+
466
+ for (int j = lid; j < dh; j += #{BLOCK}) {
467
+ qtile[j] = #{read(q_dtype, 'q[query * d + head * dh + j]')};
468
+ acc[j] = 0.0f;
469
+ }
470
+ if (lid == 0) { state[0] = #{NEG_MAX}; state[1] = 0.0f; }
471
+ GPU_BARRIER;
472
+
473
+ for (int tile = 0; tile < m; tile += #{FLASH_TILE_KEYS}) {
474
+ int count = m - tile < #{FLASH_TILE_KEYS} ? m - tile : #{FLASH_TILE_KEYS};
475
+ for (int i = lid; i < count * dh; i += #{BLOCK}) {
476
+ int t = i / dh;
477
+ int j = i % dh;
478
+ ktile[i] = #{read(k_dtype, 'k[(tile + t) * d + head * dh + j]')};
479
+ vtile[i] = #{read(v_dtype, 'v[(tile + t) * d + head * dh + j]')};
480
+ }
481
+ GPU_BARRIER;
482
+
483
+ for (int t = 0; t < count; t++) {
484
+ int key = tile + t;
485
+ float partial = 0.0f;
486
+ for (int j = lid; j < dh; j += #{BLOCK}) partial += qtile[j] * ktile[t * dh + j];
487
+ dot[lid] = partial;
488
+ #{block_reduce('dot[lid] + dot[lid + s]').gsub('sh[', 'dot[')}
489
+
490
+ if (lid == 0) {
491
+ float score = dot[0] * scale#{bias_read};
492
+ float next_max = fmax(state[0], score);
493
+ state[2] = exp(state[0] - next_max);
494
+ state[3] = exp(score - next_max);
495
+ state[0] = next_max;
496
+ state[1] = state[1] * state[2] + state[3];
497
+ }
498
+ GPU_BARRIER;
499
+ for (int j = lid; j < dh; j += #{BLOCK}) {
500
+ acc[j] = acc[j] * state[2] + state[3] * vtile[t * dh + j];
501
+ }
502
+ GPU_BARRIER;
503
+ }
504
+ }
505
+
506
+ for (int j = lid; j < dh; j += #{BLOCK}) out[query * d + head * dh + j] = acc[j] / state[1];
507
+ }
508
+ KERNEL
509
+ end
510
+
511
+ # For backends without batched GEMM (such as :cpu): materializes heads one
512
+ # at a time. Slow, but the result is the same.
513
+ def attention_per_head(q, k, v, heads, dh, n, m, d, scale, bias = nil)
514
+ # head_scatter covers [n, d] across all heads, so a reused pool slot (not
515
+ # zero-filled) leaves no residue.
516
+ out = Ops.allocate_output(:float32, n * d, [n, d])
517
+ heads.times do |h|
518
+ qh = head_slice(q, h, dh, n, d)
519
+ kh = head_slice(k, h, dh, m, d)
520
+ vh = head_slice(v, h, dh, m, d)
521
+ kt = transpose(kh)
522
+ scores = qh.matmul(kt) * scale
523
+ if bias
524
+ # bias is [heads * n, m]; the h-th [n, m] is contiguous by rows, so a
525
+ # view can take it (do not let it outlive its parent)
526
+ biased = scores + bias.view(h * n * m, n * m, shape: [n, m])
527
+ scores.free
528
+ scores = biased
529
+ end
530
+ probs = softmax(scores)
531
+ ctx = probs.matmul(vh)
532
+ head_scatter(ctx, out, h, dh, n, d)
533
+ [qh, kh, vh, kt, scores, probs, ctx].each(&:free)
534
+ end
535
+ out
536
+ rescue StandardError
537
+ [qh, kh, vh, kt, scores, probs, ctx, out].compact.uniq.each { |buffer| buffer.free unless buffer.freed? }
538
+ raise
539
+ end
540
+
541
+ def check_attention_bias!(bias, heads, n, m)
542
+ return if bias.nil?
543
+
544
+ shaped!(bias, "bias")
545
+ return if bias.shape == [heads * n, m]
546
+
547
+ raise ArgumentError,
548
+ "bias must be [heads * n, m] = #{[heads * n, m].inspect}, got #{bias.shape.inspect}"
549
+ end
550
+
551
+ # Extracts the h-th head section of [rows, d] as [rows, dh].
552
+ def head_slice(x, head, dh, rows, _d)
553
+ @head_gather ||= Tilework.kernel("nn_head_gather", out: 1) do |src, o, hd, dhh|
554
+ j = global_id_x
555
+ r = global_id_y
556
+ o[r, j] = src[r, (hd * dhh) + j] if j < dhh && r < src.rows
557
+ end
558
+ @head_gather.call(x, head, dh, over: [dh, rows])
559
+ end
560
+
561
+ # The reverse direction. dst is not out:, so the guard is written by hand.
562
+ def head_scatter(src, dst, head, dh, rows, _d)
563
+ @head_scatter ||= Tilework.kernel("nn_head_scatter") do |s, o, hd, dhh|
564
+ j = global_id_x
565
+ r = global_id_y
566
+ o[r, (hd * dhh) + j] = s[r, j] if j < dhh && r < s.rows
567
+ end
568
+ @head_scatter.call(src, dst, head, dh, over: [dh, rows])
569
+ end
570
+
571
+ def ctype(dtype) = Ops::CTYPE.fetch(dtype)
572
+
573
+ # Half is a storage type, so it is widened to float before entering an expression.
574
+ def read(dtype, expr) = Ops.read_dtype(dtype, expr)
575
+
576
+ # Checks that look only at shape and dtype accept Buffers and Graph::Tensors
577
+ # alike, so recording into a graph goes through **the same checks** —
578
+ # splitting them would let expressions through on one path only.
579
+ def shaped!(x, op)
580
+ return x if x.is_a?(Buffer) || x.is_a?(Graph::Tensor)
581
+
582
+ raise TypeError, "#{op} requires a Tilework::Buffer or Tilework::Graph::Tensor, got #{x.class}"
583
+ end
584
+
585
+ def matrix_dims!(x, op)
586
+ shaped!(x, op)
587
+ unless x.shape.length == 2
588
+ raise ArgumentError, "#{op} requires a 2D buffer, got #{x.shape.inspect}"
589
+ end
590
+
591
+ x.shape
592
+ end
593
+
594
+ # The launch domain for row-wise reductions (softmax / layernorm /
595
+ # rmsnorm): **the last dim is columns and everything before folds into
596
+ # rows**. Kernels only see a flat [rows, cols], so 3D and up pass as is
597
+ # ([batch, tokens, d] or attention's [heads, n, m]); the output keeps the
598
+ # input's shape, so no dims are lost.
599
+ def rowwise_dims!(x, op)
600
+ shaped!(x, op)
601
+ unless x.shape.length >= 2
602
+ raise ArgumentError, "#{op} requires a 2D or higher buffer, got #{x.shape.inspect}"
603
+ end
604
+
605
+ [x.shape[0..-2].inject(:*), x.shape[-1]]
606
+ end
607
+
608
+ def nchw_dims!(x, op)
609
+ shaped!(x, op)
610
+ unless x.shape.length == 4
611
+ raise ArgumentError, "#{op} requires a 4D NCHW buffer, got #{x.shape.inspect}"
612
+ end
613
+
614
+ x.shape
615
+ end
616
+
617
+ def check_affine!(gamma, beta, expected, dtype)
618
+ [["gamma", gamma], ["beta", beta]].each do |name, buf|
619
+ next if buf.nil?
620
+
621
+ shaped!(buf, name)
622
+ unless buf.length == expected
623
+ raise ArgumentError, "#{name} must have #{expected} elements, got #{buf.length}"
624
+ end
625
+ unless buf.dtype == dtype
626
+ raise ArgumentError, "#{name} dtype #{buf.dtype.inspect} does not match the input's #{dtype.inspect}"
627
+ end
628
+ end
629
+ end
630
+
631
+ # --- Shared loss helpers -------------------------------------------------
632
+
633
+ def check_reduction!(reduction)
634
+ return if LOSS_REDUCTIONS.include?(reduction)
635
+
636
+ raise ArgumentError, "reduction must be one of #{LOSS_REDUCTIONS.inspect}, got #{reduction.inspect}"
637
+ end
638
+
639
+ def check_loss_pair!(pred, target)
640
+ shaped!(pred, "mse (pred)")
641
+ shaped!(target, "mse (target)")
642
+ unless pred.shape == target.shape
643
+ raise ArgumentError, "mse requires matching shapes (pred #{pred.shape.inspect}, " \
644
+ "target #{target.shape.inspect})"
645
+ end
646
+ unless pred.dtype == target.dtype
647
+ raise ArgumentError, "mse requires matching dtypes (pred #{pred.dtype.inspect}, " \
648
+ "target #{target.dtype.inspect})"
649
+ end
650
+ return if FLOAT_DTYPES.include?(pred.dtype)
651
+
652
+ raise ArgumentError, "mse requires a float dtype, got #{pred.dtype.inspect}"
653
+ end
654
+
655
+ # Labels are class indices (:int32). Probability-style inputs (one-hot /
656
+ # label smoothing) are floats of the same shape, so a mix-up fails by name,
657
+ # saying what is expected.
658
+ def check_labels!(labels, logits, rows)
659
+ shaped!(labels, "cross_entropy (labels)")
660
+ unless labels.dtype == :int32
661
+ raise ArgumentError, "cross_entropy labels must be :int32 class indices, got " \
662
+ "#{labels.dtype.inspect} (probability targets are not supported yet)"
663
+ end
664
+ expected = logits.shape[0..-2]
665
+ return if labels.shape == expected || labels.length == rows
666
+
667
+ raise ArgumentError, "cross_entropy labels must have shape #{expected.inspect} " \
668
+ "(logits #{logits.shape.inspect}), got #{labels.shape.inspect}"
669
+ end
670
+
671
+ FLOAT_DTYPES = %i[float32 float16 bfloat16].freeze
672
+
673
+ # Losses are always kept in float. Accumulating in half piles up error as
674
+ # the count grows, so even :none (no reduction) widens to float32 (the same
675
+ # rule as Ops' accumulation).
676
+ def loss_dtype(dtype) = Ops::LOW_PRECISION_DTYPES.include?(dtype) ? :float32 : dtype
677
+
678
+ def loss_shape(shape, reduction, rowwise:)
679
+ return [1] unless reduction == :none
680
+
681
+ rowwise ? shape[0..-2] : shape
682
+ end
683
+
684
+ # Receives the reduced value; :none returns it as is.
685
+ def reduce_loss(values, reduction, device:)
686
+ return values if reduction == :none
687
+
688
+ result = if reduction == :mean
689
+ Ops.mean_op(values, device: device)
690
+ else
691
+ Ops.reduce(:sum, values, device: device)
692
+ end
693
+ values.free
694
+ result
695
+ end
696
+
697
+ def squared_error(pred, target)
698
+ dtype = loss_dtype(pred.dtype)
699
+ if Tilework.host_fallback?
700
+ values = pred.to_a.zip(target.to_a).map { |a, b| (a - b)**2 }
701
+ out = Ops.allocate_output(dtype, values.length, pred.shape)
702
+ out.write(values)
703
+ return out
704
+ end
705
+
706
+ out = Ops.allocate_output(dtype, pred.length, pred.shape)
707
+ kernel = Tilework.compile(squared_error_src(pred.dtype, dtype), name: "nn_squared_error")
708
+ Ops.launch_kernel(kernel, grid: (pred.length / 256.0).ceil, block: 256, args: [pred, target, out, pred.length])
709
+ out
710
+ end
711
+
712
+ def squared_error_src(dtype, acc)
713
+ t = ctype(dtype)
714
+ <<~KERNEL
715
+ GPU_KERNEL void nn_squared_error(GPU_ARGS,
716
+ device const #{t}* a,
717
+ device const #{t}* b,
718
+ device #{ctype(acc)}* y,
719
+ constant int& n) {
720
+ int i = GLOBAL_ID_X;
721
+ if (i < n) {
722
+ float d = #{read(dtype, 'a[i]')} - #{read(dtype, 'b[i]')};
723
+ y[i] = d * d;
724
+ }
725
+ }
726
+ KERNEL
727
+ end
728
+
729
+ # One row per block, the same skeleton as softmax (max → sum), but only one
730
+ # loss per row is written, so no third pass is needed.
731
+ def cross_entropy_src(dtype)
732
+ t = ctype(dtype)
733
+ <<~KERNEL
734
+ GPU_KERNEL void nn_cross_entropy(GPU_ARGS,
735
+ device const #{t}* x,
736
+ device const int* labels,
737
+ device #{ctype(loss_dtype(dtype))}* loss,
738
+ constant int& rows,
739
+ constant int& cols) {
740
+ int row = GROUP_ID_X;
741
+ int lid = LOCAL_ID_X;
742
+ if (row >= rows) return;
743
+ GPU_SHARED float sh[#{BLOCK}];
744
+ int base = row * cols;
745
+
746
+ // 1. Row max (stabilizes log-sum-exp)
747
+ float m = #{NEG_MAX};
748
+ for (int i = lid; i < cols; i += #{BLOCK}) m = fmax(m, #{read(dtype, 'x[base + i]')});
749
+ sh[lid] = m;
750
+ #{block_reduce('fmax(sh[lid], sh[lid + s])')}
751
+ m = sh[0];
752
+ GPU_BARRIER; // sh is reused by the next reduction, so wait until everyone has read it
753
+
754
+ // 2. Sum of exp
755
+ float total = 0.0f;
756
+ for (int i = lid; i < cols; i += #{BLOCK}) total += exp(#{read(dtype, 'x[base + i]')} - m);
757
+ sh[lid] = total;
758
+ #{block_reduce('sh[lid] + sh[lid + s]')}
759
+
760
+ // 3. -log p[label] = log(sum exp(x - m)) - (x[label] - m)
761
+ if (lid == 0) {
762
+ int c = labels[row];
763
+ c = c < 0 ? 0 : (c >= cols ? cols - 1 : c); // clamp out-of-range to the ends (as gather does)
764
+ loss[row] = log(sh[0]) - (#{read(dtype, 'x[base + c]')} - m);
765
+ }
766
+ }
767
+ KERNEL
768
+ end
769
+
770
+ def cpu_cross_entropy(logits, labels, rows, cols)
771
+ flat = logits.to_a
772
+ classes = labels.to_a
773
+ values = Array.new(rows) do |row|
774
+ slice = flat[row * cols, cols]
775
+ m = slice.max
776
+ c = classes[row].clamp(0, cols - 1)
777
+ Math.log(slice.sum { |v| Math.exp(v - m) }) - (slice[c] - m)
778
+ end
779
+ out = Ops.allocate_output(loss_dtype(logits.dtype), rows, [rows])
780
+ out.write(values)
781
+ out
782
+ end
783
+
784
+ # Fills in when only one of gamma/beta is passed.
785
+ def ones_like(n, dtype)
786
+ buf = Buffer.new(dtype, n)
787
+ buf.write(Array.new(n, 1.0))
788
+ buf
789
+ end
790
+
791
+ def zeros_like(n, dtype) = Buffer.new(dtype, n) # zero-filled on allocation
792
+
793
+ # In-block reduction: call with values in sh[] and the result lands in
794
+ # sh[0]. To reuse sh afterwards, read it and then put a GPU_BARRIER in between.
795
+ def block_reduce(combine)
796
+ <<~SRC.chomp
797
+ GPU_BARRIER;
798
+ for (int s = #{BLOCK / 2}; s > 0; s >>= 1) {
799
+ if (lid < s) { sh[lid] = #{combine}; }
800
+ GPU_BARRIER;
801
+ }
802
+ SRC
803
+ end
804
+
805
+ def softmax_src(dtype)
806
+ t = ctype(dtype)
807
+ <<~KERNEL
808
+ GPU_KERNEL void nn_softmax(GPU_ARGS,
809
+ device const #{t}* x,
810
+ device #{t}* y,
811
+ constant int& rows,
812
+ constant int& cols) {
813
+ int row = GROUP_ID_X;
814
+ int lid = LOCAL_ID_X;
815
+ if (row >= rows) return;
816
+ GPU_SHARED float sh[#{BLOCK}];
817
+ int base = row * cols;
818
+
819
+ // 1. Row max. Without subtracting it, exp overflows on large logits.
820
+ float m = #{NEG_MAX};
821
+ for (int i = lid; i < cols; i += #{BLOCK}) m = fmax(m, #{read(dtype, 'x[base + i]')});
822
+ sh[lid] = m;
823
+ #{block_reduce('fmax(sh[lid], sh[lid + s])')}
824
+ m = sh[0];
825
+ GPU_BARRIER; // sh is reused by the next reduction, so wait until everyone has read it
826
+
827
+ // 2. Sum of exp
828
+ float total = 0.0f;
829
+ for (int i = lid; i < cols; i += #{BLOCK}) total += exp(#{read(dtype, 'x[base + i]')} - m);
830
+ sh[lid] = total;
831
+ #{block_reduce('sh[lid] + sh[lid + s]')}
832
+ float denom = sh[0];
833
+
834
+ // 3. Write back
835
+ for (int i = lid; i < cols; i += #{BLOCK}) {
836
+ y[base + i] = exp(#{read(dtype, 'x[base + i]')} - m) / denom;
837
+ }
838
+ }
839
+ KERNEL
840
+ end
841
+
842
+ def layernorm_src(dtype, affine)
843
+ t = ctype(dtype)
844
+ params = ["device const #{t}* x", "device #{t}* y"]
845
+ params += ["device const #{t}* gamma", "device const #{t}* beta"] if affine
846
+ params += ["constant int& rows", "constant int& cols", "constant float& eps"]
847
+ scale = affine ? " * #{read(dtype, 'gamma[i]')} + #{read(dtype, 'beta[i]')}" : ""
848
+ <<~KERNEL
849
+ GPU_KERNEL void nn_layernorm(GPU_ARGS,
850
+ #{params.join(",\n ")}) {
851
+ int row = GROUP_ID_X;
852
+ int lid = LOCAL_ID_X;
853
+ if (row >= rows) return;
854
+ GPU_SHARED float sh[#{BLOCK}];
855
+ int base = row * cols;
856
+
857
+ // 1. Mean
858
+ float total = 0.0f;
859
+ for (int i = lid; i < cols; i += #{BLOCK}) total += #{read(dtype, 'x[base + i]')};
860
+ sh[lid] = total;
861
+ #{block_reduce('sh[lid] + sh[lid + s]')}
862
+ float mean = sh[0] / (float)cols;
863
+ GPU_BARRIER;
864
+
865
+ // 2. Variance, recomputing deviations in a second pass (E[x^2] - E[x]^2 cancels catastrophically).
866
+ float sq = 0.0f;
867
+ for (int i = lid; i < cols; i += #{BLOCK}) {
868
+ float d = #{read(dtype, 'x[base + i]')} - mean;
869
+ sq += d * d;
870
+ }
871
+ sh[lid] = sq;
872
+ #{block_reduce('sh[lid] + sh[lid + s]')}
873
+ float inv = 1.0f / sqrt(sh[0] / (float)cols + eps);
874
+
875
+ // 3. Normalize
876
+ for (int i = lid; i < cols; i += #{BLOCK}) {
877
+ y[base + i] = (#{read(dtype, 'x[base + i]')} - mean) * inv#{scale};
878
+ }
879
+ }
880
+ KERNEL
881
+ end
882
+
883
+ # The same one-row-per-block reduction as layernorm, but with no mean
884
+ # subtraction it takes one pass (layernorm scans twice, for mean and variance).
885
+ def rmsnorm_src(dtype, affine)
886
+ t = ctype(dtype)
887
+ params = ["device const #{t}* x", "device #{t}* y"]
888
+ params += ["device const #{t}* gamma"] if affine
889
+ params += ["constant int& rows", "constant int& cols", "constant float& eps"]
890
+ scale = affine ? " * #{read(dtype, 'gamma[i]')}" : ""
891
+ <<~KERNEL
892
+ GPU_KERNEL void nn_rmsnorm(GPU_ARGS,
893
+ #{params.join(",\n ")}) {
894
+ int row = GROUP_ID_X;
895
+ int lid = LOCAL_ID_X;
896
+ if (row >= rows) return;
897
+ GPU_SHARED float sh[#{BLOCK}];
898
+ int base = row * cols;
899
+
900
+ // Sum of squares; always accumulate in float (in half it stops growing midway)
901
+ float sq = 0.0f;
902
+ for (int i = lid; i < cols; i += #{BLOCK}) {
903
+ float v = #{read(dtype, 'x[base + i]')};
904
+ sq += v * v;
905
+ }
906
+ sh[lid] = sq;
907
+ #{block_reduce('sh[lid] + sh[lid + s]')}
908
+ float inv = 1.0f / sqrt(sh[0] / (float)cols + eps);
909
+
910
+ for (int i = lid; i < cols; i += #{BLOCK}) {
911
+ y[base + i] = #{read(dtype, 'x[base + i]')} * inv#{scale};
912
+ }
913
+ }
914
+ KERNEL
915
+ end
916
+
917
+ def groupnorm_src(dtype, affine)
918
+ t = ctype(dtype)
919
+ params = ["device const #{t}* x", "device #{t}* y"]
920
+ params += ["device const #{t}* gamma", "device const #{t}* beta"] if affine
921
+ params += ["constant int& n", "constant int& c", "constant int& hw",
922
+ "constant int& groups", "constant float& eps"]
923
+ scale = affine ? " * #{read(dtype, 'gamma[ch]')} + #{read(dtype, 'beta[ch]')}" : ""
924
+ <<~KERNEL
925
+ GPU_KERNEL void nn_groupnorm(GPU_ARGS,
926
+ #{params.join(",\n ")}) {
927
+ // One block = one (image, group). The reduction spans all of (C/groups) * H * W.
928
+ int blk = GROUP_ID_X;
929
+ int lid = LOCAL_ID_X;
930
+ if (blk >= n * groups) return;
931
+ int img = blk / groups;
932
+ int grp = blk % groups;
933
+ int per = c / groups; // channels per group
934
+ int count = per * hw; // elements per group
935
+ int base = (img * c + grp * per) * hw;
936
+ GPU_SHARED float sh[#{BLOCK}];
937
+
938
+ float total = 0.0f;
939
+ for (int i = lid; i < count; i += #{BLOCK}) total += #{read(dtype, 'x[base + i]')};
940
+ sh[lid] = total;
941
+ #{block_reduce('sh[lid] + sh[lid + s]')}
942
+ float mean = sh[0] / (float)count;
943
+ GPU_BARRIER;
944
+
945
+ float sq = 0.0f;
946
+ for (int i = lid; i < count; i += #{BLOCK}) {
947
+ float d = #{read(dtype, 'x[base + i]')} - mean;
948
+ sq += d * d;
949
+ }
950
+ sh[lid] = sq;
951
+ #{block_reduce('sh[lid] + sh[lid + s]')}
952
+ float inv = 1.0f / sqrt(sh[0] / (float)count + eps);
953
+
954
+ for (int i = lid; i < count; i += #{BLOCK}) {
955
+ int ch = grp * per + i / hw; // gamma/beta are per channel
956
+ y[base + i] = (#{read(dtype, 'x[base + i]')} - mean) * inv#{scale};
957
+ }
958
+ }
959
+ KERNEL
960
+ end
961
+
962
+ # Every activation is one thread per element; only the expression differs.
963
+ ACTIVATIONS = {
964
+ relu: "fmax(v, 0.0f)",
965
+ # Writing 1/(1+exp(-v)) directly overflows exp for very negative v. This
966
+ # form uses only exp(-|v|), so it is safe for either sign.
967
+ sigmoid: "(v >= 0.0f ? 1.0f / (1.0f + exp(-v)) : exp(v) / (1.0f + exp(v)))",
968
+ silu: "v * (v >= 0.0f ? 1.0f / (1.0f + exp(-v)) : exp(v) / (1.0f + exp(v)))",
969
+ gelu_tanh: "0.5f * v * (1.0f + tanh(0.7978845608028654f * (v + 0.044715f * v * v * v)))",
970
+ gelu_exact: "0.5f * v * (1.0f + GPU_ERF(v * 0.7071067811865476f))",
971
+ }.freeze
972
+
973
+ def activation(x, kind)
974
+ shaped!(x, kind.to_s)
975
+ return Graph.record_nn(:activation, [x], args: [kind], shape: x.shape, dtype: x.dtype) if Graph.symbolic?(x)
976
+ return cpu_activation(x, kind) if Tilework.host_fallback?
977
+
978
+ expr = ACTIVATIONS.fetch(kind)
979
+ t = ctype(x.dtype)
980
+ src = <<~KERNEL
981
+ GPU_KERNEL void nn_#{kind}(GPU_ARGS,
982
+ device const #{t}* x,
983
+ device #{t}* y,
984
+ constant int& n) {
985
+ int i = GLOBAL_ID_X;
986
+ if (i < n) {
987
+ float v = #{read(x.dtype, 'x[i]')};
988
+ y[i] = #{expr};
989
+ }
990
+ }
991
+ KERNEL
992
+ out = Ops.allocate_output(x.dtype, x.length, x.shape)
993
+ kernel = Tilework.compile(src, name: "nn_#{kind}")
994
+ Ops.launch_kernel(kernel, grid: (x.length / 256.0).ceil, block: 256, args: [x, out, x.length])
995
+ out
996
+ end
997
+
998
+ def transpose_src(dtype)
999
+ t = ctype(dtype)
1000
+ <<~KERNEL
1001
+ GPU_KERNEL void nn_transpose(GPU_ARGS,
1002
+ device const #{t}* x,
1003
+ device #{t}* y,
1004
+ constant int& rows,
1005
+ constant int& cols) {
1006
+ // Indexed by the input [rows, cols] position; the output is [cols, rows].
1007
+ int r = GLOBAL_ID_X;
1008
+ int c = GLOBAL_ID_Y;
1009
+ if (r < rows && c < cols) y[c * rows + r] = x[r * cols + c];
1010
+ }
1011
+ KERNEL
1012
+ end
1013
+
1014
+ # --- Reference implementations for the :cpu backend ----------------------
1015
+ #
1016
+ # :cpu has no generator for shared memory, so these compute on the host.
1017
+ # Slow, but they also **produce the expected values for tests** (GPU
1018
+ # results are compared against them).
1019
+
1020
+ def rebuild(x, values, shape = x.shape)
1021
+ out = Ops.allocate_output(x.dtype, values.length, shape)
1022
+ out.write(values)
1023
+ out
1024
+ end
1025
+
1026
+ def cpu_softmax(x, rows, cols)
1027
+ values = x.to_a.each_slice(cols).flat_map do |row|
1028
+ m = row.max
1029
+ exps = row.map { |v| Math.exp(v - m) }
1030
+ total = exps.sum
1031
+ exps.map { |e| e / total }
1032
+ end
1033
+ rebuild(x, values)
1034
+ end
1035
+
1036
+ def cpu_layernorm(x, rows, cols, gamma, beta, eps)
1037
+ g = gamma&.to_a
1038
+ b = beta&.to_a
1039
+ values = x.to_a.each_slice(cols).flat_map do |row|
1040
+ mean = row.sum / cols.to_f
1041
+ var = row.sum { |v| (v - mean)**2 } / cols.to_f
1042
+ inv = 1.0 / Math.sqrt(var + eps)
1043
+ row.each_with_index.map do |v, i|
1044
+ ((v - mean) * inv * (g ? g[i] : 1.0)) + (b ? b[i] : 0.0)
1045
+ end
1046
+ end
1047
+ rebuild(x, values)
1048
+ end
1049
+
1050
+ def cpu_rmsnorm(x, rows, cols, gamma, eps)
1051
+ g = gamma&.to_a
1052
+ values = x.to_a.each_slice(cols).flat_map do |row|
1053
+ inv = 1.0 / Math.sqrt((row.sum { |v| v * v } / cols.to_f) + eps)
1054
+ row.each_with_index.map { |v, i| v * inv * (g ? g[i] : 1.0) }
1055
+ end
1056
+ rebuild(x, values)
1057
+ end
1058
+
1059
+ def cpu_groupnorm(x, shape, groups, gamma, beta, eps)
1060
+ n, c, h, w = shape
1061
+ hw = h * w
1062
+ per = c / groups
1063
+ g = gamma&.to_a
1064
+ b = beta&.to_a
1065
+ flat = x.to_a
1066
+ out = Array.new(flat.length)
1067
+ n.times do |img|
1068
+ groups.times do |grp|
1069
+ base = ((img * c) + (grp * per)) * hw
1070
+ slice = flat[base, per * hw]
1071
+ mean = slice.sum / slice.length.to_f
1072
+ var = slice.sum { |v| (v - mean)**2 } / slice.length.to_f
1073
+ inv = 1.0 / Math.sqrt(var + eps)
1074
+ slice.each_with_index do |v, i|
1075
+ ch = (grp * per) + (i / hw)
1076
+ out[base + i] = ((v - mean) * inv * (g ? g[ch] : 1.0)) + (b ? b[ch] : 0.0)
1077
+ end
1078
+ end
1079
+ end
1080
+ rebuild(x, out)
1081
+ end
1082
+
1083
+ def cpu_activation(x, kind)
1084
+ sigmoid = ->(v) { v >= 0 ? 1.0 / (1.0 + Math.exp(-v)) : Math.exp(v) / (1.0 + Math.exp(v)) }
1085
+ fn = case kind
1086
+ when :relu then ->(v) { v.positive? ? v : 0.0 }
1087
+ when :sigmoid then sigmoid
1088
+ when :silu then ->(v) { v * sigmoid.(v) }
1089
+ when :gelu_tanh
1090
+ ->(v) { 0.5 * v * (1.0 + Math.tanh(0.7978845608028654 * (v + (0.044715 * v * v * v)))) }
1091
+ when :gelu_exact then ->(v) { 0.5 * v * (1.0 + Math.erf(v * 0.7071067811865476)) }
1092
+ end
1093
+ rebuild(x, x.to_a.map(&fn))
1094
+ end
1095
+
1096
+ def cpu_transpose(x, rows, cols)
1097
+ flat = x.to_a
1098
+ values = Array.new(rows * cols)
1099
+ cols.times { |c| rows.times { |r| values[(c * rows) + r] = flat[(r * cols) + c] } }
1100
+ rebuild(x, values, [cols, rows])
1101
+ end
1102
+ end
1103
+ end