tilework 0.18.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE +21 -0
  3. data/lib/tilework/backend/cpu/dsl_codegen.rb +509 -0
  4. data/lib/tilework/backend/cpu/ops_codegen.rb +451 -0
  5. data/lib/tilework/backend/cpu.rb +156 -0
  6. data/lib/tilework/buffer.rb +753 -0
  7. data/lib/tilework/capabilities.rb +183 -0
  8. data/lib/tilework/compat/cuda.h +66 -0
  9. data/lib/tilework/compat/hip.h +35 -0
  10. data/lib/tilework/compat/metal.h +45 -0
  11. data/lib/tilework/conformance.rb +2296 -0
  12. data/lib/tilework/core_ext.rb +21 -0
  13. data/lib/tilework/dialect.rb +67 -0
  14. data/lib/tilework/dsl.rb +722 -0
  15. data/lib/tilework/dtype/bfloat.rb +31 -0
  16. data/lib/tilework/dtype/cast.rb +88 -0
  17. data/lib/tilework/dtype/half.rb +98 -0
  18. data/lib/tilework/graph/builder.rb +1073 -0
  19. data/lib/tilework/graph/fusion.rb +520 -0
  20. data/lib/tilework/graph/grad.rb +443 -0
  21. data/lib/tilework/graph/high_level.rb +266 -0
  22. data/lib/tilework/graph/kernels.rb +403 -0
  23. data/lib/tilework/graph/node.rb +27 -0
  24. data/lib/tilework/graph/op.rb +290 -0
  25. data/lib/tilework/graph/optim.rb +183 -0
  26. data/lib/tilework/graph/plan.rb +1086 -0
  27. data/lib/tilework/graph/pool.rb +178 -0
  28. data/lib/tilework/graph/region.rb +210 -0
  29. data/lib/tilework/graph/residency.rb +449 -0
  30. data/lib/tilework/graph/state.rb +64 -0
  31. data/lib/tilework/graph/tensor.rb +124 -0
  32. data/lib/tilework/graph.rb +213 -0
  33. data/lib/tilework/kernel.rb +163 -0
  34. data/lib/tilework/math/conv.rb +270 -0
  35. data/lib/tilework/math/fft.rb +172 -0
  36. data/lib/tilework/math/nn.rb +1103 -0
  37. data/lib/tilework/math/philox.rb +194 -0
  38. data/lib/tilework/math/random.rb +232 -0
  39. data/lib/tilework/math/tensor_core.rb +269 -0
  40. data/lib/tilework/ops.rb +682 -0
  41. data/lib/tilework/runtime/offload.rb +498 -0
  42. data/lib/tilework/runtime/submission.rb +163 -0
  43. data/lib/tilework/runtime/telemetry.rb +112 -0
  44. data/lib/tilework/scope.rb +33 -0
  45. data/lib/tilework/version.rb +5 -0
  46. data/lib/tilework.rb +345 -0
  47. metadata +92 -0
@@ -0,0 +1,682 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Tilework
4
+ # Elementwise and reduction operations mixed into {Buffer}.
5
+ # Generates and launches CUDA-flavored dialect kernels. The generated source
6
+ # rides Kernel's PSO cache, so repeating an operation does not recompile.
7
+ module Ops
8
+ # float16 / bfloat16 appear **only as storage types**. Arithmetic always
9
+ # widens to float, so they never become a compute dtype (see result_dtypes).
10
+ CTYPE = { float32: "float", int32: "int", float16: "GPU_HALF", bfloat16: "GPU_BFLOAT" }.freeze
11
+ LOW_PRECISION_DTYPES = %i[float16 bfloat16].freeze
12
+ ZERO = { float32: "0.0f", int32: "0" }.freeze
13
+ BLOCK = 256
14
+ OPERATORS = %i[+ - * /].freeze
15
+ COMPARISONS = %i[> < >= <=].freeze
16
+ # Unary math functions. int32 input is promoted to float32
17
+ UNARY_MATH = %i[exp log sqrt sin cos tanh floor ceil round].freeze
18
+
19
+ # Reduction combine expressions and identities. The identity is the value
20
+ # padding elements take so they do not affect the result
21
+ REDUCTIONS = {
22
+ sum: {
23
+ combine: "sh[lid] += sh[lid + s];",
24
+ identity: ZERO,
25
+ },
26
+ min: {
27
+ combine: "sh[lid] = min(sh[lid], sh[lid + s]);",
28
+ identity: { float32: "INFINITY", int32: "2147483647" },
29
+ },
30
+ max: {
31
+ combine: "sh[lid] = max(sh[lid], sh[lid + s]);",
32
+ identity: { float32: "-INFINITY", int32: "(-2147483647 - 1)" },
33
+ },
34
+ }.freeze
35
+
36
+ OPERATORS.each do |op|
37
+ define_method(op) { |other| Ops.arithmetic(op, self, other) }
38
+ end
39
+
40
+ COMPARISONS.each do |op|
41
+ define_method(op) { |other| Ops.compare(op, self, other) }
42
+ end
43
+
44
+ UNARY_MATH.each do |fn|
45
+ define_method(fn) { Ops.unary_math(fn, self) }
46
+ end
47
+
48
+ # Compares elements for equality and returns an int32 mask.
49
+ # @param other [Buffer, Numeric]
50
+ # @return [Buffer]
51
+ def eq(other) = Ops.compare(:==, self, other)
52
+ def ne(other) = Ops.compare(:!=, self, other)
53
+
54
+ # Returns elementwise absolute values.
55
+ # @return [Buffer]
56
+ def abs = Ops.unary_abs(self)
57
+ def clamp(lo, hi) = Ops.clamp_op(self, lo, hi)
58
+ def maximum(other) = Ops.binary_fn("max", self, other)
59
+ def minimum(other) = Ops.binary_fn("min", self, other)
60
+
61
+ # Multiplies two two-dimensional matrices.
62
+ # @param other [Buffer]
63
+ # @param out_dtype [Symbol, nil] optional low-precision output storage dtype
64
+ # @return [Buffer] `[rows, other.cols]` matrix; float inputs accumulate in float32
65
+ def matmul(other, out_dtype: nil) = Ops.matmul_op(self, other, out_dtype: out_dtype)
66
+
67
+ # Makes scalar-on-the-left expressions like 2.0 * buffer work
68
+ def coerce(num)
69
+ [Scalar.new(num), self]
70
+ end
71
+
72
+ # Reduces all elements to a sum.
73
+ # @param device [Boolean] return a one-element buffer instead of a host scalar
74
+ # @return [Numeric, Buffer]
75
+ #
76
+ # Returns a host Ruby number by default. **device: true returns a
77
+ # one-element Buffer holding the result** (see Ops.reduce).
78
+ def sum(device: false) = Ops.reduce(:sum, self, device: device)
79
+ def min(device: false) = Ops.reduce(:min, self, device: device)
80
+ def max(device: false) = Ops.reduce(:max, self, device: device)
81
+ def mean(device: false) = Ops.mean_op(self, device: device)
82
+ def argmax(device: false) = Ops.arg_reduce(:max, self, device: device)
83
+ def argmin(device: false) = Ops.arg_reduce(:min, self, device: device)
84
+
85
+ # The scalar-side proxy returned by coerce. Non-commutative ops (- /)
86
+ # generate a kernel with swapped operands; comparisons flip the operator
87
+ # (s > a[i] ≡ a[i] < s).
88
+ class Scalar
89
+ def initialize(value)
90
+ @value = value
91
+ end
92
+
93
+ def +(other) = Ops.arithmetic(:+, other, @value)
94
+ def *(other) = Ops.arithmetic(:*, other, @value)
95
+ def -(other) = Ops.arithmetic(:-, other, @value, reversed: true)
96
+ def /(other) = Ops.arithmetic(:/, other, @value, reversed: true)
97
+ def >(other) = Ops.compare(:<, other, @value)
98
+ def <(other) = Ops.compare(:>, other, @value)
99
+ def >=(other) = Ops.compare(:<=, other, @value)
100
+ def <=(other) = Ops.compare(:>=, other, @value)
101
+ end
102
+
103
+ # The execution context an upper layer (the graph's Plan) plugs in while
104
+ # running. **Without one (= eager use) behavior is unchanged.**
105
+ #
106
+ # #allocate(dtype, length, shape) — takes over output allocation; a
107
+ # caller that knows lifetimes can hand
108
+ # out pooled buffers instead
109
+ # #async? — true makes launches launch_async
110
+ #
111
+ # It is a thread-local context rather than an argument to keep the branch
112
+ # out of eager code (threading out:/async: through every op signature would
113
+ # make everyone carry arguments no caller uses).
114
+ CONTEXT_KEY = :tilework_ops_context
115
+
116
+ class << self
117
+ def context = Thread.current[CONTEXT_KEY]
118
+
119
+ def with_context(context)
120
+ previous = Thread.current[CONTEXT_KEY]
121
+ Thread.current[CONTEXT_KEY] = context
122
+ yield
123
+ ensure
124
+ Thread.current[CONTEXT_KEY] = previous
125
+ end
126
+ end
127
+
128
+ module_function
129
+
130
+ def allocate_output(dtype, length, shape)
131
+ context = Ops.context
132
+ return Buffer.new(dtype, length, shape: shape) unless context
133
+
134
+ context.allocate(dtype, length, shape)
135
+ end
136
+
137
+ # Called from `Buffer#free`: the third part of the context contract (after
138
+ # where to allocate and whether to go async — who decides freeing).
139
+ # Releasing memory a running kernel reads is undefined behavior on
140
+ # CUDA/HIP, so with a context the free is handed to it. With no context
141
+ # (eager) it returns false and nothing changes.
142
+ def defer_free(buffer)
143
+ context = Ops.context
144
+ return false unless context.respond_to?(:defer_free)
145
+
146
+ # The context decides whether it took the buffer (things with no point in
147
+ # deferring, like views, return false and are freed on the spot).
148
+ !!context.defer_free(buffer)
149
+ end
150
+
151
+ def launch_kernel(kernel, grid:, block:, args:)
152
+ return kernel.launch(grid: grid, block: block, args: args) unless Ops.context&.async?
153
+
154
+ kernel.launch_async(grid: grid, block: block, args: args)
155
+ end
156
+
157
+ # The source generator for backends that cannot use the dialect
158
+ # (capabilities[:dialect] == false). The bridge declares it itself (the same
159
+ # pattern as Tilework.host_fallback?), so no backend name is written here.
160
+ def codegen
161
+ Tilework.bridge.ops_codegen
162
+ end
163
+
164
+ def require_computable!(*buffers)
165
+ caps = Tilework.capabilities
166
+ bad = buffers.compact.select do |buffer|
167
+ next false unless buffer.is_a?(Buffer)
168
+
169
+ (buffer.dtype == :float16 && !caps.fp16_storage) ||
170
+ (buffer.dtype == :bfloat16 && !caps.bf16_storage)
171
+ end
172
+ return if bad.empty?
173
+
174
+ raise Error, "#{bad.map(&:dtype).uniq.join(' / ')} is not computable on the #{Tilework.backend.inspect} backend; " \
175
+ "use #cast(:float32) to compute with them"
176
+ end
177
+
178
+ # The compute dtype. **float16 is never a compute dtype** — accumulating in
179
+ # half precision piles up error, so it always widens to float.
180
+ def promoted_dtype(buffer, operand)
181
+ other_int = operand.is_a?(Buffer) ? operand.dtype == :int32 : operand.is_a?(Integer)
182
+ buffer.dtype == :int32 && other_int ? :int32 : :float32
183
+ end
184
+
185
+ # Returns [compute dtype, output dtype]. half with half **keeps a float16
186
+ # output**: half precision exists to save memory bandwidth, and widening to
187
+ # float32 here would defeat it. Rounding happens once on write-back, so the
188
+ # precision is the same as "compute in float, round at the end".
189
+ # Half is a storage type, so it is widened to float before entering an expression.
190
+ def read(buffer, expr)
191
+ buffer.is_a?(Buffer) ? read_dtype(buffer.dtype, expr) : expr
192
+ end
193
+
194
+ def read_dtype(dtype, expr)
195
+ case dtype
196
+ when :float16 then "(float)#{expr}"
197
+ when :bfloat16 then "GPU_BFLOAT_TO_FLOAT(#{expr})"
198
+ else expr
199
+ end
200
+ end
201
+
202
+ def result_dtypes(buffer, operand)
203
+ compute = promoted_dtype(buffer, operand)
204
+ same_storage = operand.is_a?(Buffer) ? operand.dtype == buffer.dtype : operand.is_a?(Numeric)
205
+ out = LOW_PRECISION_DTYPES.include?(buffer.dtype) && same_storage ? buffer.dtype : compute
206
+ [compute, out]
207
+ end
208
+
209
+ # cpu_expr is passed as a thunk (a zero-argument lambda) so codegen
210
+ # (= Tilework.bridge.ops_codegen) need not be called. When compiled takes the
211
+ # dialect branch (always, on dialect backends) it is never called, so
212
+ # bridges without ops_codegen (MockBridge, Metal, CUDA) do not fail here.
213
+ def arithmetic(op, buffer, operand, reversed: false)
214
+ compute, out = result_dtypes(buffer, operand)
215
+ binary(buffer, operand, compute_dtype: compute, out_dtype: out, reversed: reversed,
216
+ cpu_expr: -> { codegen.arithmetic_expr(op) }) do |lhs, rhs|
217
+ "#{lhs} #{op} #{rhs}"
218
+ end
219
+ end
220
+
221
+ # Comparisons evaluate in the promoted type and return an int32 0/1 mask
222
+ def compare(op, buffer, operand)
223
+ binary(buffer, operand, compute_dtype: promoted_dtype(buffer, operand), out_dtype: :int32,
224
+ cpu_expr: -> { codegen.compare_expr(op) }) do |lhs, rhs|
225
+ "#{lhs} #{op} #{rhs} ? 1 : 0"
226
+ end
227
+ end
228
+
229
+ def binary_fn(fn, buffer, operand)
230
+ compute, out = result_dtypes(buffer, operand)
231
+ binary(buffer, operand, compute_dtype: compute, out_dtype: out,
232
+ cpu_expr: -> { codegen.binary_fn_expr(fn) }) do |lhs, rhs|
233
+ "#{fn}(#{lhs}, #{rhs})"
234
+ end
235
+ end
236
+
237
+ def binary(buffer, operand, compute_dtype:, out_dtype:, cpu_expr:, reversed: false, &expr)
238
+ require_computable!(buffer, operand)
239
+ case operand
240
+ when Buffer then buffer_binary(buffer, operand, compute_dtype, out_dtype, cpu_expr, &expr)
241
+ when Numeric then scalar_binary(buffer, operand, compute_dtype, out_dtype, reversed, cpu_expr, &expr)
242
+ else
243
+ raise TypeError, "cannot combine Tilework::Buffer with #{operand.class}"
244
+ end
245
+ end
246
+
247
+ def buffer_binary(a, b, compute_dtype, out_dtype, cpu_expr, &expr)
248
+ # If either side has a 2D-or-higher shape, require equal shapes (to catch
249
+ # mix-ups); two 1D sides compare by length only.
250
+ if a.shape.length > 1 || b.shape.length > 1
251
+ raise ArgumentError, "buffer shape mismatch: #{a.shape.inspect} vs #{b.shape.inspect}" unless a.shape == b.shape
252
+ elsif a.length != b.length
253
+ raise ArgumentError, "buffer length mismatch: #{a.length} vs #{b.length}"
254
+ end
255
+
256
+ ct = CTYPE.fetch(compute_dtype)
257
+ src = <<~KERNEL
258
+ GPU_KERNEL void ew(GPU_ARGS,
259
+ device const #{CTYPE.fetch(a.dtype)}* a,
260
+ device const #{CTYPE.fetch(b.dtype)}* b,
261
+ device #{CTYPE.fetch(out_dtype)}* c,
262
+ constant int& n) {
263
+ int i = GLOBAL_ID_X;
264
+ if (i < n) c[i] = #{expr.call("(#{ct})#{read_dtype(a.dtype, 'a[i]')}", "(#{ct})#{read_dtype(b.dtype, 'b[i]')}")};
265
+ }
266
+ KERNEL
267
+
268
+ kernel = compiled(
269
+ dialect_name: "ew", dialect_src: src,
270
+ cpu_name: "ew",
271
+ cpu_src: -> { codegen.buffer_binary("ew", a_dtype: a.dtype, b_dtype: b.dtype, out_dtype: out_dtype,
272
+ compute_dtype: compute_dtype, &cpu_expr.call) },
273
+ )
274
+ launch_into(kernel, a.length, [a, b], out_dtype: out_dtype, shape: a.shape)
275
+ end
276
+
277
+ def scalar_binary(a, scalar, compute_dtype, out_dtype, reversed, cpu_expr, &expr)
278
+ ct = CTYPE.fetch(compute_dtype)
279
+ lhs, rhs = "(#{ct})#{read_dtype(a.dtype, 'a[i]')}", "s"
280
+ lhs, rhs = rhs, lhs if reversed
281
+ src = <<~KERNEL
282
+ GPU_KERNEL void ew(GPU_ARGS,
283
+ device const #{CTYPE.fetch(a.dtype)}* a,
284
+ constant #{ct}& s,
285
+ device #{CTYPE.fetch(out_dtype)}* c,
286
+ constant int& n) {
287
+ int i = GLOBAL_ID_X;
288
+ if (i < n) c[i] = #{expr.call(lhs, rhs)};
289
+ }
290
+ KERNEL
291
+
292
+ kernel = compiled(
293
+ dialect_name: "ew", dialect_src: src,
294
+ cpu_name: "ew",
295
+ cpu_src: -> { codegen.scalar_binary("ew", a_dtype: a.dtype, out_dtype: out_dtype,
296
+ compute_dtype: compute_dtype, reversed: reversed, &cpu_expr.call) },
297
+ )
298
+ converted = compute_dtype == :float32 ? scalar.to_f : scalar
299
+ launch_into(kernel, a.length, [a, converted], out_dtype: out_dtype, shape: a.shape)
300
+ end
301
+
302
+ def unary(a, out_dtype:, cpu_name:, cpu_src:, &expr)
303
+ require_computable!(a)
304
+ src = <<~KERNEL
305
+ GPU_KERNEL void ew(GPU_ARGS,
306
+ device const #{CTYPE.fetch(a.dtype)}* a,
307
+ device #{CTYPE.fetch(out_dtype)}* c,
308
+ constant int& n) {
309
+ int i = GLOBAL_ID_X;
310
+ if (i < n) c[i] = #{expr.call(read(a, 'a[i]'))};
311
+ }
312
+ KERNEL
313
+
314
+ kernel = compiled(dialect_name: "ew", dialect_src: src, cpu_name: cpu_name, cpu_src: cpu_src)
315
+ launch_into(kernel, a.length, [a], out_dtype: out_dtype, shape: a.shape)
316
+ end
317
+
318
+ def unary_math(fn, a)
319
+ unary(a, out_dtype: LOW_PRECISION_DTYPES.include?(a.dtype) ? a.dtype : :float32,
320
+ cpu_name: "ew_#{fn}",
321
+ cpu_src: -> { codegen.unary_math(fn, a_dtype: a.dtype) }) { |x| "#{fn}((float)#{x})" }
322
+ end
323
+
324
+ def unary_abs(a)
325
+ fn = a.dtype == :int32 ? "abs" : "fabs"
326
+ unary(a, out_dtype: a.dtype, cpu_name: "abs_#{a.dtype}",
327
+ cpu_src: -> { codegen.unary_abs(a_dtype: a.dtype) }) { |x| "#{fn}(#{x})" }
328
+ end
329
+
330
+ def clamp_op(a, lo, hi)
331
+ require_computable!(a)
332
+ unless lo.is_a?(Numeric) && hi.is_a?(Numeric)
333
+ raise TypeError, "clamp bounds must be Numeric"
334
+ end
335
+
336
+ dtype = a.dtype == :int32 && lo.is_a?(Integer) && hi.is_a?(Integer) ? :int32 : :float32
337
+ out_dtype = LOW_PRECISION_DTYPES.include?(a.dtype) ? a.dtype : dtype
338
+ ct = CTYPE.fetch(dtype)
339
+ src = <<~KERNEL
340
+ GPU_KERNEL void ew(GPU_ARGS,
341
+ device const #{CTYPE.fetch(a.dtype)}* a,
342
+ constant #{ct}& lo,
343
+ constant #{ct}& hi,
344
+ device #{CTYPE.fetch(out_dtype)}* c,
345
+ constant int& n) {
346
+ int i = GLOBAL_ID_X;
347
+ if (i < n) c[i] = clamp((#{ct})#{read_dtype(a.dtype, 'a[i]')}, lo, hi);
348
+ }
349
+ KERNEL
350
+
351
+ kernel = compiled(
352
+ dialect_name: "ew", dialect_src: src,
353
+ cpu_name: "clamp_ew", cpu_src: -> { codegen.clamp(a_dtype: a.dtype, dtype: dtype) },
354
+ )
355
+ convert = ->(v) { dtype == :float32 ? v.to_f : v }
356
+ launch_into(kernel, a.length, [a, convert.(lo), convert.(hi)], out_dtype: out_dtype, shape: a.shape)
357
+ end
358
+
359
+ def launch_into(kernel, n, inputs, out_dtype:, shape: [n])
360
+ out = allocate_output(out_dtype, n, shape)
361
+ launch_kernel(kernel, grid: (n / BLOCK.to_f).ceil, block: BLOCK, args: [*inputs, out, n])
362
+ out
363
+ rescue StandardError
364
+ out&.free
365
+ raise
366
+ end
367
+
368
+ # Backends that cannot use the dialect use the bridge's own ops_codegen
369
+ # (lib/tilework/backend/cpu/ops_codegen.rb on :cpu), passed with raw: true.
370
+ # The dialect backends' path (dialect_src straight into Tilework.compile) is
371
+ # unchanged (the same pattern as DSLKernel in lib/tilework/dsl.rb).
372
+ def compiled(dialect_name:, dialect_src:, cpu_name:, cpu_src:)
373
+ return Tilework.compile(dialect_src, name: dialect_name) if Tilework.capabilities[:dialect]
374
+
375
+ unless Tilework.bridge.respond_to?(:ops_codegen)
376
+ raise Error, "the #{Tilework.backend.inspect} backend doesn't support dialect kernel strings " \
377
+ "and doesn't implement ops_codegen either"
378
+ end
379
+ Tilework.compile(cpu_src.call, name: cpu_name, raw: true)
380
+ end
381
+
382
+ # Matrix multiply. Requires 2D shapes a: [m, k] and b: [k, n]; the output is
383
+ # [m, n]. A naive implementation where each thread on a 1D grid computes one
384
+ # output element's dot product (no tiling or shared-memory optimization).
385
+ def matmul_op(a, b, out_dtype: nil)
386
+ raise TypeError, "matmul requires a Tilework::Buffer, got #{b.class}" unless b.is_a?(Buffer)
387
+ unless a.matrix? && b.matrix?
388
+ raise ArgumentError, "matmul requires 2D buffers (shape), got #{a.shape.inspect} and #{b.shape.inspect}"
389
+ end
390
+ unless a.cols == b.rows
391
+ raise ArgumentError, "matmul shape mismatch: #{a.shape.inspect} x #{b.shape.inspect}"
392
+ end
393
+
394
+ m, k, n = a.rows, a.cols, b.cols
395
+ dtype = promoted_dtype(a, b)
396
+ out_dtype ||= dtype
397
+ allowed = [dtype]
398
+ allowed << a.dtype if a.dtype == b.dtype && LOW_PRECISION_DTYPES.include?(a.dtype)
399
+ unless allowed.include?(out_dtype)
400
+ raise ArgumentError, "matmul out_dtype must be #{allowed.map(&:inspect).join(' or ')} " \
401
+ "for #{a.dtype.inspect} x #{b.dtype.inspect} inputs"
402
+ end
403
+
404
+ require_computable!(a, b)
405
+
406
+ # Fast GEMM paths. This is the only branch point; anything that takes
407
+ # none of them falls through to the SIMT kernel below.
408
+ # 1. a verified bridge-specific matrix-core kernel (tile-aligned shapes)
409
+ # 2. a GEMM library (no shape constraints)
410
+ # 3. CUDA's WMMA kernel
411
+ # 4. otherwise SIMT
412
+ result = if TensorCore.matrix_core_eligible?(a, b, dtype)
413
+ TensorCore.matrix_core_gemm(a, b, m, k, n)
414
+ elsif TensorCore.gemm_eligible?(a, b, dtype)
415
+ TensorCore.gemm(a, b, m, k, n)
416
+ elsif TensorCore.eligible?(a, b, dtype)
417
+ TensorCore.matmul(a, b, m, k, n)
418
+ end
419
+ return cast_matmul_output(result, out_dtype) if result
420
+
421
+ ct = CTYPE.fetch(dtype)
422
+ src = <<~KERNEL
423
+ GPU_KERNEL void matmul(GPU_ARGS,
424
+ device const #{CTYPE.fetch(a.dtype)}* a,
425
+ device const #{CTYPE.fetch(b.dtype)}* b,
426
+ device #{ct}* c,
427
+ constant int& m,
428
+ constant int& k,
429
+ constant int& n) {
430
+ int i = GLOBAL_ID_X;
431
+ if (i < m * n) {
432
+ int row = i / n;
433
+ int col = i % n;
434
+ #{ct} acc = #{ZERO.fetch(dtype)};
435
+ for (int t = 0; t < k; t++) {
436
+ acc += (#{ct})#{read_dtype(a.dtype, 'a[row * k + t]')} * (#{ct})#{read_dtype(b.dtype, 'b[t * n + col]')};
437
+ }
438
+ c[i] = acc;
439
+ }
440
+ }
441
+ KERNEL
442
+
443
+ kernel = compiled(
444
+ dialect_name: "matmul", dialect_src: src,
445
+ cpu_name: "matmul", cpu_src: -> { codegen.matmul(a_dtype: a.dtype, b_dtype: b.dtype, dtype: dtype) },
446
+ )
447
+ out = allocate_output(dtype, m * n, [m, n])
448
+ launch_kernel(kernel, grid: (m * n / BLOCK.to_f).ceil, block: BLOCK, args: [a, b, out, m, k, n])
449
+ cast_matmul_output(out, out_dtype)
450
+ end
451
+
452
+ def cast_matmul_output(out, out_dtype)
453
+ return out if out.dtype == out_dtype
454
+
455
+ converted = out.cast(out_dtype)
456
+ out.free
457
+ converted
458
+ rescue StandardError
459
+ out.free
460
+ raise
461
+ end
462
+
463
+ # Builds per-threadgroup partial reductions on the GPU.
464
+ #
465
+ # There are two ways to receive the result:
466
+ #
467
+ # buffer.sum # => Float. Runs one pass and folds the
468
+ # # partials on the host
469
+ # buffer.sum(device: true) # => a one-element Buffer, folded all the
470
+ # # way on the device
471
+ #
472
+ # **Receiving on the host drains the queue** — Buffer#to_a waits for
473
+ # pending kernels, so every reduction becomes a full synchronization.
474
+ # Reductions come at the end of loops (loss, sampling, normalizer
475
+ # denominators), so dropping to the host each time rules out "queue the
476
+ # launches and wait once at the end". Use device: true to feed the result
477
+ # straight into the next operation.
478
+ #
479
+ # device: true reapplies the kernel until one partial is left (folding 256
480
+ # at a time, so even a million elements take three passes).
481
+ #
482
+ # Backends with Tilework.host_fallback? (:cpu) bypass kernel generation and
483
+ # compute directly in Ruby. GPU_SHARED/GPU_BARRIER assume every thread in a
484
+ # group runs the body at once and synchronizes at barriers; reproducing that
485
+ # correctly on a CPU would need an interpreter emulating barrier phases.
486
+ # Only these two functions (reduce/arg_reduce) use GPU_SHARED/GPU_BARRIER,
487
+ # internally to Ops (not exposed to user Tilework.kernel {}), so a dedicated
488
+ # interpreter is not worth it (see the top of
489
+ # lib/tilework/backend/cpu/ops_codegen.rb).
490
+ def reduce(kind, buffer, device: false)
491
+ require_computable!(buffer)
492
+ if Tilework.host_fallback?
493
+ value = buffer.to_a.public_send(kind)
494
+ return device ? scalar_buffer(value, accumulator_dtype(buffer)) : value
495
+ end
496
+
497
+ partial = reduce_tree(kind, buffer, device: device)
498
+ return partial if device
499
+
500
+ value = partial.to_a.public_send(kind)
501
+ partial.free
502
+ value
503
+ end
504
+
505
+ # With device: false it runs one pass and returns the partials (the caller
506
+ # folds them); with device: true it reapplies the kernel until one is left.
507
+ def reduce_tree(kind, buffer, device:)
508
+ acc_dtype = accumulator_dtype(buffer)
509
+ source = buffer
510
+ loop do
511
+ groups = (source.length / BLOCK.to_f).ceil
512
+ # Take the partials and the final value from the execution context
513
+ # (borrow and return symmetrically). The context takes over frees, so a
514
+ # Buffer.new temporary would become a slot that is returned but never
515
+ # borrowed, growing the pool every plan run.
516
+ partial = allocate_output(acc_dtype, groups, [groups])
517
+ launch_kernel(reduce_kernel(kind, source.dtype, acc_dtype),
518
+ grid: groups, block: BLOCK, args: [source, partial, source.length])
519
+ source.free unless source.equal?(buffer) # intermediate partials are ours to free
520
+ source = partial
521
+ break if !device || groups == 1
522
+ end
523
+ source
524
+ rescue StandardError
525
+ partial&.free unless partial&.equal?(source)
526
+ source&.free unless source&.equal?(buffer)
527
+ raise
528
+ end
529
+
530
+ # Accumulating in half precision piles up error as the count grows, so
531
+ # reduce in float and return float32 (accumulation is always FP32).
532
+ def accumulator_dtype(buffer) = LOW_PRECISION_DTYPES.include?(buffer.dtype) ? :float32 : buffer.dtype
533
+
534
+ def scalar_buffer(value, dtype)
535
+ allocate_output(dtype, 1, [1]).tap { |buffer| buffer.write([value]) }
536
+ end
537
+
538
+ # From the second pass the input is partials (in the accumulator type), so
539
+ # the input type is a parameter.
540
+ def reduce_kernel(kind, in_dtype, acc_dtype)
541
+ spec = REDUCTIONS.fetch(kind)
542
+ ctype = CTYPE.fetch(acc_dtype)
543
+ element = read_dtype(in_dtype, "a[gid]")
544
+ src = <<~KERNEL
545
+ GPU_KERNEL void reduce_#{kind}(GPU_ARGS,
546
+ device const #{CTYPE.fetch(in_dtype)}* a,
547
+ device #{ctype}* partial,
548
+ constant int& n) {
549
+ GPU_SHARED #{ctype} sh[#{BLOCK}];
550
+ int gid = GLOBAL_ID_X;
551
+ int lid = LOCAL_ID_X;
552
+ sh[lid] = gid < n ? #{element} : #{spec[:identity].fetch(acc_dtype)};
553
+ GPU_BARRIER;
554
+ for (int s = #{BLOCK / 2}; s > 0; s >>= 1) {
555
+ if (lid < s) #{spec[:combine]}
556
+ GPU_BARRIER;
557
+ }
558
+ if (lid == 0) partial[GROUP_ID_X] = sh[0];
559
+ }
560
+ KERNEL
561
+
562
+ compiled(
563
+ dialect_name: "reduce_#{kind}", dialect_src: src,
564
+ cpu_name: "reduce_#{kind}", cpu_src: -> { raise NotImplementedError },
565
+ )
566
+ end
567
+
568
+ # Mean. With device: true both the sum and the division finish on the device.
569
+ def mean_op(buffer, device: false)
570
+ return reduce(:sum, buffer).fdiv(buffer.length) unless device
571
+
572
+ total = reduce(:sum, buffer, device: true)
573
+ mean = arithmetic(:/, total, buffer.length.to_f)
574
+ total.free
575
+ mean
576
+ rescue StandardError
577
+ total&.free
578
+ raise
579
+ end
580
+
581
+ # Reduction over (value, index) pairs. Ties keep the smaller index (strict
582
+ # comparison, left-biased). Later passes keep indices ascending, so the same
583
+ # strict comparison picks the same winners — device: true and host folding
584
+ # agree. See #reduce about the early return on :cpu.
585
+ def arg_reduce(kind, buffer, device: false)
586
+ require_computable!(buffer)
587
+ if Tilework.host_fallback?
588
+ index = host_arg_reduce(kind, buffer.to_a)
589
+ return device ? scalar_buffer(index, :int32) : index
590
+ end
591
+
592
+ values, indices = arg_reduce_pass(kind, buffer, nil)
593
+ unless device
594
+ index = host_arg_reduce(kind, values.to_a, indices.to_a)
595
+ values.free
596
+ indices.free
597
+ return index
598
+ end
599
+
600
+ until values.length == 1
601
+ next_values, next_indices = arg_reduce_pass(kind, values, indices)
602
+ values.free
603
+ indices.free
604
+ values, indices = next_values, next_indices
605
+ end
606
+ values.free
607
+ indices # only the indices are the answer
608
+ end
609
+
610
+ # One pass. With indices nil the indices come from thread ids (the first
611
+ # pass); otherwise the indices chosen by the previous pass are carried along.
612
+ def arg_reduce_pass(kind, source, indices)
613
+ groups = (source.length / BLOCK.to_f).ceil
614
+ # Borrow and return symmetrically (the same reason as reduce_tree). pval
615
+ # is always a temporary freed within this operation; only the final pidx
616
+ # goes to the caller and becomes the Plan's.
617
+ pval = allocate_output(source.dtype, groups, [groups])
618
+ pidx = allocate_output(:int32, groups, [groups])
619
+ args = indices ? [source, indices, pval, pidx] : [source, pval, pidx]
620
+ launch_kernel(arg_reduce_kernel(kind, source.dtype, carry: !indices.nil?),
621
+ grid: groups, block: BLOCK, args: [*args, source.length])
622
+ [pval, pidx]
623
+ rescue StandardError
624
+ pval&.free
625
+ pidx&.free
626
+ raise
627
+ end
628
+
629
+ def arg_reduce_kernel(kind, dtype, carry:)
630
+ cmp = kind == :max ? ">" : "<"
631
+ ctype = CTYPE.fetch(dtype)
632
+ identity = REDUCTIONS.fetch(kind)[:identity].fetch(dtype)
633
+ name = carry ? "arg_#{kind}_pairs" : "arg_#{kind}"
634
+ params = ["device const #{ctype}* a"]
635
+ params << "device const int* idx" if carry
636
+ params += ["device #{ctype}* pval", "device int* pidx", "constant int& n"]
637
+ src = <<~KERNEL
638
+ GPU_KERNEL void #{name}(GPU_ARGS, #{params.join(', ')}) {
639
+ GPU_SHARED #{ctype} sv[#{BLOCK}];
640
+ GPU_SHARED int si[#{BLOCK}];
641
+ int gid = GLOBAL_ID_X;
642
+ int lid = LOCAL_ID_X;
643
+ sv[lid] = gid < n ? a[gid] : #{identity};
644
+ si[lid] = #{carry ? 'gid < n ? idx[gid] : 0' : 'gid'};
645
+ GPU_BARRIER;
646
+ for (int s = #{BLOCK / 2}; s > 0; s >>= 1) {
647
+ if (lid < s && sv[lid + s] #{cmp} sv[lid]) {
648
+ sv[lid] = sv[lid + s];
649
+ si[lid] = si[lid + s];
650
+ }
651
+ GPU_BARRIER;
652
+ }
653
+ if (lid == 0) {
654
+ pval[GROUP_ID_X] = sv[0];
655
+ pidx[GROUP_ID_X] = si[0];
656
+ }
657
+ }
658
+ KERNEL
659
+
660
+ compiled(
661
+ dialect_name: name, dialect_src: src,
662
+ cpu_name: name, cpu_src: -> { raise NotImplementedError },
663
+ )
664
+ end
665
+
666
+ # Host side of index reductions. Without indices, indices count from 0.
667
+ def host_arg_reduce(kind, values, indices = nil)
668
+ best_value = best_index = nil
669
+ values.each_with_index do |value, i|
670
+ next unless best_value.nil? || (kind == :max ? value > best_value : value < best_value)
671
+
672
+ best_value = value
673
+ best_index = indices ? indices[i] : i
674
+ end
675
+ best_index
676
+ end
677
+ end
678
+
679
+ class Buffer
680
+ include Ops
681
+ end
682
+ end