tilework 0.18.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE +21 -0
- data/lib/tilework/backend/cpu/dsl_codegen.rb +509 -0
- data/lib/tilework/backend/cpu/ops_codegen.rb +451 -0
- data/lib/tilework/backend/cpu.rb +156 -0
- data/lib/tilework/buffer.rb +753 -0
- data/lib/tilework/capabilities.rb +183 -0
- data/lib/tilework/compat/cuda.h +66 -0
- data/lib/tilework/compat/hip.h +35 -0
- data/lib/tilework/compat/metal.h +45 -0
- data/lib/tilework/conformance.rb +2296 -0
- data/lib/tilework/core_ext.rb +21 -0
- data/lib/tilework/dialect.rb +67 -0
- data/lib/tilework/dsl.rb +722 -0
- data/lib/tilework/dtype/bfloat.rb +31 -0
- data/lib/tilework/dtype/cast.rb +88 -0
- data/lib/tilework/dtype/half.rb +98 -0
- data/lib/tilework/graph/builder.rb +1073 -0
- data/lib/tilework/graph/fusion.rb +520 -0
- data/lib/tilework/graph/grad.rb +443 -0
- data/lib/tilework/graph/high_level.rb +266 -0
- data/lib/tilework/graph/kernels.rb +403 -0
- data/lib/tilework/graph/node.rb +27 -0
- data/lib/tilework/graph/op.rb +290 -0
- data/lib/tilework/graph/optim.rb +183 -0
- data/lib/tilework/graph/plan.rb +1086 -0
- data/lib/tilework/graph/pool.rb +178 -0
- data/lib/tilework/graph/region.rb +210 -0
- data/lib/tilework/graph/residency.rb +449 -0
- data/lib/tilework/graph/state.rb +64 -0
- data/lib/tilework/graph/tensor.rb +124 -0
- data/lib/tilework/graph.rb +213 -0
- data/lib/tilework/kernel.rb +163 -0
- data/lib/tilework/math/conv.rb +270 -0
- data/lib/tilework/math/fft.rb +172 -0
- data/lib/tilework/math/nn.rb +1103 -0
- data/lib/tilework/math/philox.rb +194 -0
- data/lib/tilework/math/random.rb +232 -0
- data/lib/tilework/math/tensor_core.rb +269 -0
- data/lib/tilework/ops.rb +682 -0
- data/lib/tilework/runtime/offload.rb +498 -0
- data/lib/tilework/runtime/submission.rb +163 -0
- data/lib/tilework/runtime/telemetry.rb +112 -0
- data/lib/tilework/scope.rb +33 -0
- data/lib/tilework/version.rb +5 -0
- data/lib/tilework.rb +345 -0
- metadata +92 -0
data/lib/tilework/ops.rb
ADDED
|
@@ -0,0 +1,682 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Tilework
|
|
4
|
+
# Elementwise and reduction operations mixed into {Buffer}.
|
|
5
|
+
# Generates and launches CUDA-flavored dialect kernels. The generated source
|
|
6
|
+
# rides Kernel's PSO cache, so repeating an operation does not recompile.
|
|
7
|
+
module Ops
|
|
8
|
+
# float16 / bfloat16 appear **only as storage types**. Arithmetic always
|
|
9
|
+
# widens to float, so they never become a compute dtype (see result_dtypes).
|
|
10
|
+
CTYPE = { float32: "float", int32: "int", float16: "GPU_HALF", bfloat16: "GPU_BFLOAT" }.freeze
|
|
11
|
+
LOW_PRECISION_DTYPES = %i[float16 bfloat16].freeze
|
|
12
|
+
ZERO = { float32: "0.0f", int32: "0" }.freeze
|
|
13
|
+
BLOCK = 256
|
|
14
|
+
OPERATORS = %i[+ - * /].freeze
|
|
15
|
+
COMPARISONS = %i[> < >= <=].freeze
|
|
16
|
+
# Unary math functions. int32 input is promoted to float32
|
|
17
|
+
UNARY_MATH = %i[exp log sqrt sin cos tanh floor ceil round].freeze
|
|
18
|
+
|
|
19
|
+
# Reduction combine expressions and identities. The identity is the value
|
|
20
|
+
# padding elements take so they do not affect the result
|
|
21
|
+
REDUCTIONS = {
|
|
22
|
+
sum: {
|
|
23
|
+
combine: "sh[lid] += sh[lid + s];",
|
|
24
|
+
identity: ZERO,
|
|
25
|
+
},
|
|
26
|
+
min: {
|
|
27
|
+
combine: "sh[lid] = min(sh[lid], sh[lid + s]);",
|
|
28
|
+
identity: { float32: "INFINITY", int32: "2147483647" },
|
|
29
|
+
},
|
|
30
|
+
max: {
|
|
31
|
+
combine: "sh[lid] = max(sh[lid], sh[lid + s]);",
|
|
32
|
+
identity: { float32: "-INFINITY", int32: "(-2147483647 - 1)" },
|
|
33
|
+
},
|
|
34
|
+
}.freeze
|
|
35
|
+
|
|
36
|
+
OPERATORS.each do |op|
|
|
37
|
+
define_method(op) { |other| Ops.arithmetic(op, self, other) }
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
COMPARISONS.each do |op|
|
|
41
|
+
define_method(op) { |other| Ops.compare(op, self, other) }
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
UNARY_MATH.each do |fn|
|
|
45
|
+
define_method(fn) { Ops.unary_math(fn, self) }
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
# Compares elements for equality and returns an int32 mask.
|
|
49
|
+
# @param other [Buffer, Numeric]
|
|
50
|
+
# @return [Buffer]
|
|
51
|
+
def eq(other) = Ops.compare(:==, self, other)
|
|
52
|
+
def ne(other) = Ops.compare(:!=, self, other)
|
|
53
|
+
|
|
54
|
+
# Returns elementwise absolute values.
|
|
55
|
+
# @return [Buffer]
|
|
56
|
+
def abs = Ops.unary_abs(self)
|
|
57
|
+
def clamp(lo, hi) = Ops.clamp_op(self, lo, hi)
|
|
58
|
+
def maximum(other) = Ops.binary_fn("max", self, other)
|
|
59
|
+
def minimum(other) = Ops.binary_fn("min", self, other)
|
|
60
|
+
|
|
61
|
+
# Multiplies two two-dimensional matrices.
|
|
62
|
+
# @param other [Buffer]
|
|
63
|
+
# @param out_dtype [Symbol, nil] optional low-precision output storage dtype
|
|
64
|
+
# @return [Buffer] `[rows, other.cols]` matrix; float inputs accumulate in float32
|
|
65
|
+
def matmul(other, out_dtype: nil) = Ops.matmul_op(self, other, out_dtype: out_dtype)
|
|
66
|
+
|
|
67
|
+
# Makes scalar-on-the-left expressions like 2.0 * buffer work
|
|
68
|
+
def coerce(num)
|
|
69
|
+
[Scalar.new(num), self]
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
# Reduces all elements to a sum.
|
|
73
|
+
# @param device [Boolean] return a one-element buffer instead of a host scalar
|
|
74
|
+
# @return [Numeric, Buffer]
|
|
75
|
+
#
|
|
76
|
+
# Returns a host Ruby number by default. **device: true returns a
|
|
77
|
+
# one-element Buffer holding the result** (see Ops.reduce).
|
|
78
|
+
def sum(device: false) = Ops.reduce(:sum, self, device: device)
|
|
79
|
+
def min(device: false) = Ops.reduce(:min, self, device: device)
|
|
80
|
+
def max(device: false) = Ops.reduce(:max, self, device: device)
|
|
81
|
+
def mean(device: false) = Ops.mean_op(self, device: device)
|
|
82
|
+
def argmax(device: false) = Ops.arg_reduce(:max, self, device: device)
|
|
83
|
+
def argmin(device: false) = Ops.arg_reduce(:min, self, device: device)
|
|
84
|
+
|
|
85
|
+
# The scalar-side proxy returned by coerce. Non-commutative ops (- /)
|
|
86
|
+
# generate a kernel with swapped operands; comparisons flip the operator
|
|
87
|
+
# (s > a[i] ≡ a[i] < s).
|
|
88
|
+
class Scalar
|
|
89
|
+
def initialize(value)
|
|
90
|
+
@value = value
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
def +(other) = Ops.arithmetic(:+, other, @value)
|
|
94
|
+
def *(other) = Ops.arithmetic(:*, other, @value)
|
|
95
|
+
def -(other) = Ops.arithmetic(:-, other, @value, reversed: true)
|
|
96
|
+
def /(other) = Ops.arithmetic(:/, other, @value, reversed: true)
|
|
97
|
+
def >(other) = Ops.compare(:<, other, @value)
|
|
98
|
+
def <(other) = Ops.compare(:>, other, @value)
|
|
99
|
+
def >=(other) = Ops.compare(:<=, other, @value)
|
|
100
|
+
def <=(other) = Ops.compare(:>=, other, @value)
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
# The execution context an upper layer (the graph's Plan) plugs in while
|
|
104
|
+
# running. **Without one (= eager use) behavior is unchanged.**
|
|
105
|
+
#
|
|
106
|
+
# #allocate(dtype, length, shape) — takes over output allocation; a
|
|
107
|
+
# caller that knows lifetimes can hand
|
|
108
|
+
# out pooled buffers instead
|
|
109
|
+
# #async? — true makes launches launch_async
|
|
110
|
+
#
|
|
111
|
+
# It is a thread-local context rather than an argument to keep the branch
|
|
112
|
+
# out of eager code (threading out:/async: through every op signature would
|
|
113
|
+
# make everyone carry arguments no caller uses).
|
|
114
|
+
CONTEXT_KEY = :tilework_ops_context
|
|
115
|
+
|
|
116
|
+
class << self
|
|
117
|
+
def context = Thread.current[CONTEXT_KEY]
|
|
118
|
+
|
|
119
|
+
def with_context(context)
|
|
120
|
+
previous = Thread.current[CONTEXT_KEY]
|
|
121
|
+
Thread.current[CONTEXT_KEY] = context
|
|
122
|
+
yield
|
|
123
|
+
ensure
|
|
124
|
+
Thread.current[CONTEXT_KEY] = previous
|
|
125
|
+
end
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
module_function
|
|
129
|
+
|
|
130
|
+
def allocate_output(dtype, length, shape)
|
|
131
|
+
context = Ops.context
|
|
132
|
+
return Buffer.new(dtype, length, shape: shape) unless context
|
|
133
|
+
|
|
134
|
+
context.allocate(dtype, length, shape)
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
# Called from `Buffer#free`: the third part of the context contract (after
|
|
138
|
+
# where to allocate and whether to go async — who decides freeing).
|
|
139
|
+
# Releasing memory a running kernel reads is undefined behavior on
|
|
140
|
+
# CUDA/HIP, so with a context the free is handed to it. With no context
|
|
141
|
+
# (eager) it returns false and nothing changes.
|
|
142
|
+
def defer_free(buffer)
|
|
143
|
+
context = Ops.context
|
|
144
|
+
return false unless context.respond_to?(:defer_free)
|
|
145
|
+
|
|
146
|
+
# The context decides whether it took the buffer (things with no point in
|
|
147
|
+
# deferring, like views, return false and are freed on the spot).
|
|
148
|
+
!!context.defer_free(buffer)
|
|
149
|
+
end
|
|
150
|
+
|
|
151
|
+
def launch_kernel(kernel, grid:, block:, args:)
|
|
152
|
+
return kernel.launch(grid: grid, block: block, args: args) unless Ops.context&.async?
|
|
153
|
+
|
|
154
|
+
kernel.launch_async(grid: grid, block: block, args: args)
|
|
155
|
+
end
|
|
156
|
+
|
|
157
|
+
# The source generator for backends that cannot use the dialect
|
|
158
|
+
# (capabilities[:dialect] == false). The bridge declares it itself (the same
|
|
159
|
+
# pattern as Tilework.host_fallback?), so no backend name is written here.
|
|
160
|
+
def codegen
|
|
161
|
+
Tilework.bridge.ops_codegen
|
|
162
|
+
end
|
|
163
|
+
|
|
164
|
+
def require_computable!(*buffers)
|
|
165
|
+
caps = Tilework.capabilities
|
|
166
|
+
bad = buffers.compact.select do |buffer|
|
|
167
|
+
next false unless buffer.is_a?(Buffer)
|
|
168
|
+
|
|
169
|
+
(buffer.dtype == :float16 && !caps.fp16_storage) ||
|
|
170
|
+
(buffer.dtype == :bfloat16 && !caps.bf16_storage)
|
|
171
|
+
end
|
|
172
|
+
return if bad.empty?
|
|
173
|
+
|
|
174
|
+
raise Error, "#{bad.map(&:dtype).uniq.join(' / ')} is not computable on the #{Tilework.backend.inspect} backend; " \
|
|
175
|
+
"use #cast(:float32) to compute with them"
|
|
176
|
+
end
|
|
177
|
+
|
|
178
|
+
# The compute dtype. **float16 is never a compute dtype** — accumulating in
|
|
179
|
+
# half precision piles up error, so it always widens to float.
|
|
180
|
+
def promoted_dtype(buffer, operand)
|
|
181
|
+
other_int = operand.is_a?(Buffer) ? operand.dtype == :int32 : operand.is_a?(Integer)
|
|
182
|
+
buffer.dtype == :int32 && other_int ? :int32 : :float32
|
|
183
|
+
end
|
|
184
|
+
|
|
185
|
+
# Returns [compute dtype, output dtype]. half with half **keeps a float16
|
|
186
|
+
# output**: half precision exists to save memory bandwidth, and widening to
|
|
187
|
+
# float32 here would defeat it. Rounding happens once on write-back, so the
|
|
188
|
+
# precision is the same as "compute in float, round at the end".
|
|
189
|
+
# Half is a storage type, so it is widened to float before entering an expression.
|
|
190
|
+
def read(buffer, expr)
|
|
191
|
+
buffer.is_a?(Buffer) ? read_dtype(buffer.dtype, expr) : expr
|
|
192
|
+
end
|
|
193
|
+
|
|
194
|
+
def read_dtype(dtype, expr)
|
|
195
|
+
case dtype
|
|
196
|
+
when :float16 then "(float)#{expr}"
|
|
197
|
+
when :bfloat16 then "GPU_BFLOAT_TO_FLOAT(#{expr})"
|
|
198
|
+
else expr
|
|
199
|
+
end
|
|
200
|
+
end
|
|
201
|
+
|
|
202
|
+
def result_dtypes(buffer, operand)
|
|
203
|
+
compute = promoted_dtype(buffer, operand)
|
|
204
|
+
same_storage = operand.is_a?(Buffer) ? operand.dtype == buffer.dtype : operand.is_a?(Numeric)
|
|
205
|
+
out = LOW_PRECISION_DTYPES.include?(buffer.dtype) && same_storage ? buffer.dtype : compute
|
|
206
|
+
[compute, out]
|
|
207
|
+
end
|
|
208
|
+
|
|
209
|
+
# cpu_expr is passed as a thunk (a zero-argument lambda) so codegen
|
|
210
|
+
# (= Tilework.bridge.ops_codegen) need not be called. When compiled takes the
|
|
211
|
+
# dialect branch (always, on dialect backends) it is never called, so
|
|
212
|
+
# bridges without ops_codegen (MockBridge, Metal, CUDA) do not fail here.
|
|
213
|
+
def arithmetic(op, buffer, operand, reversed: false)
|
|
214
|
+
compute, out = result_dtypes(buffer, operand)
|
|
215
|
+
binary(buffer, operand, compute_dtype: compute, out_dtype: out, reversed: reversed,
|
|
216
|
+
cpu_expr: -> { codegen.arithmetic_expr(op) }) do |lhs, rhs|
|
|
217
|
+
"#{lhs} #{op} #{rhs}"
|
|
218
|
+
end
|
|
219
|
+
end
|
|
220
|
+
|
|
221
|
+
# Comparisons evaluate in the promoted type and return an int32 0/1 mask
|
|
222
|
+
def compare(op, buffer, operand)
|
|
223
|
+
binary(buffer, operand, compute_dtype: promoted_dtype(buffer, operand), out_dtype: :int32,
|
|
224
|
+
cpu_expr: -> { codegen.compare_expr(op) }) do |lhs, rhs|
|
|
225
|
+
"#{lhs} #{op} #{rhs} ? 1 : 0"
|
|
226
|
+
end
|
|
227
|
+
end
|
|
228
|
+
|
|
229
|
+
def binary_fn(fn, buffer, operand)
|
|
230
|
+
compute, out = result_dtypes(buffer, operand)
|
|
231
|
+
binary(buffer, operand, compute_dtype: compute, out_dtype: out,
|
|
232
|
+
cpu_expr: -> { codegen.binary_fn_expr(fn) }) do |lhs, rhs|
|
|
233
|
+
"#{fn}(#{lhs}, #{rhs})"
|
|
234
|
+
end
|
|
235
|
+
end
|
|
236
|
+
|
|
237
|
+
def binary(buffer, operand, compute_dtype:, out_dtype:, cpu_expr:, reversed: false, &expr)
|
|
238
|
+
require_computable!(buffer, operand)
|
|
239
|
+
case operand
|
|
240
|
+
when Buffer then buffer_binary(buffer, operand, compute_dtype, out_dtype, cpu_expr, &expr)
|
|
241
|
+
when Numeric then scalar_binary(buffer, operand, compute_dtype, out_dtype, reversed, cpu_expr, &expr)
|
|
242
|
+
else
|
|
243
|
+
raise TypeError, "cannot combine Tilework::Buffer with #{operand.class}"
|
|
244
|
+
end
|
|
245
|
+
end
|
|
246
|
+
|
|
247
|
+
def buffer_binary(a, b, compute_dtype, out_dtype, cpu_expr, &expr)
|
|
248
|
+
# If either side has a 2D-or-higher shape, require equal shapes (to catch
|
|
249
|
+
# mix-ups); two 1D sides compare by length only.
|
|
250
|
+
if a.shape.length > 1 || b.shape.length > 1
|
|
251
|
+
raise ArgumentError, "buffer shape mismatch: #{a.shape.inspect} vs #{b.shape.inspect}" unless a.shape == b.shape
|
|
252
|
+
elsif a.length != b.length
|
|
253
|
+
raise ArgumentError, "buffer length mismatch: #{a.length} vs #{b.length}"
|
|
254
|
+
end
|
|
255
|
+
|
|
256
|
+
ct = CTYPE.fetch(compute_dtype)
|
|
257
|
+
src = <<~KERNEL
|
|
258
|
+
GPU_KERNEL void ew(GPU_ARGS,
|
|
259
|
+
device const #{CTYPE.fetch(a.dtype)}* a,
|
|
260
|
+
device const #{CTYPE.fetch(b.dtype)}* b,
|
|
261
|
+
device #{CTYPE.fetch(out_dtype)}* c,
|
|
262
|
+
constant int& n) {
|
|
263
|
+
int i = GLOBAL_ID_X;
|
|
264
|
+
if (i < n) c[i] = #{expr.call("(#{ct})#{read_dtype(a.dtype, 'a[i]')}", "(#{ct})#{read_dtype(b.dtype, 'b[i]')}")};
|
|
265
|
+
}
|
|
266
|
+
KERNEL
|
|
267
|
+
|
|
268
|
+
kernel = compiled(
|
|
269
|
+
dialect_name: "ew", dialect_src: src,
|
|
270
|
+
cpu_name: "ew",
|
|
271
|
+
cpu_src: -> { codegen.buffer_binary("ew", a_dtype: a.dtype, b_dtype: b.dtype, out_dtype: out_dtype,
|
|
272
|
+
compute_dtype: compute_dtype, &cpu_expr.call) },
|
|
273
|
+
)
|
|
274
|
+
launch_into(kernel, a.length, [a, b], out_dtype: out_dtype, shape: a.shape)
|
|
275
|
+
end
|
|
276
|
+
|
|
277
|
+
def scalar_binary(a, scalar, compute_dtype, out_dtype, reversed, cpu_expr, &expr)
|
|
278
|
+
ct = CTYPE.fetch(compute_dtype)
|
|
279
|
+
lhs, rhs = "(#{ct})#{read_dtype(a.dtype, 'a[i]')}", "s"
|
|
280
|
+
lhs, rhs = rhs, lhs if reversed
|
|
281
|
+
src = <<~KERNEL
|
|
282
|
+
GPU_KERNEL void ew(GPU_ARGS,
|
|
283
|
+
device const #{CTYPE.fetch(a.dtype)}* a,
|
|
284
|
+
constant #{ct}& s,
|
|
285
|
+
device #{CTYPE.fetch(out_dtype)}* c,
|
|
286
|
+
constant int& n) {
|
|
287
|
+
int i = GLOBAL_ID_X;
|
|
288
|
+
if (i < n) c[i] = #{expr.call(lhs, rhs)};
|
|
289
|
+
}
|
|
290
|
+
KERNEL
|
|
291
|
+
|
|
292
|
+
kernel = compiled(
|
|
293
|
+
dialect_name: "ew", dialect_src: src,
|
|
294
|
+
cpu_name: "ew",
|
|
295
|
+
cpu_src: -> { codegen.scalar_binary("ew", a_dtype: a.dtype, out_dtype: out_dtype,
|
|
296
|
+
compute_dtype: compute_dtype, reversed: reversed, &cpu_expr.call) },
|
|
297
|
+
)
|
|
298
|
+
converted = compute_dtype == :float32 ? scalar.to_f : scalar
|
|
299
|
+
launch_into(kernel, a.length, [a, converted], out_dtype: out_dtype, shape: a.shape)
|
|
300
|
+
end
|
|
301
|
+
|
|
302
|
+
def unary(a, out_dtype:, cpu_name:, cpu_src:, &expr)
|
|
303
|
+
require_computable!(a)
|
|
304
|
+
src = <<~KERNEL
|
|
305
|
+
GPU_KERNEL void ew(GPU_ARGS,
|
|
306
|
+
device const #{CTYPE.fetch(a.dtype)}* a,
|
|
307
|
+
device #{CTYPE.fetch(out_dtype)}* c,
|
|
308
|
+
constant int& n) {
|
|
309
|
+
int i = GLOBAL_ID_X;
|
|
310
|
+
if (i < n) c[i] = #{expr.call(read(a, 'a[i]'))};
|
|
311
|
+
}
|
|
312
|
+
KERNEL
|
|
313
|
+
|
|
314
|
+
kernel = compiled(dialect_name: "ew", dialect_src: src, cpu_name: cpu_name, cpu_src: cpu_src)
|
|
315
|
+
launch_into(kernel, a.length, [a], out_dtype: out_dtype, shape: a.shape)
|
|
316
|
+
end
|
|
317
|
+
|
|
318
|
+
def unary_math(fn, a)
|
|
319
|
+
unary(a, out_dtype: LOW_PRECISION_DTYPES.include?(a.dtype) ? a.dtype : :float32,
|
|
320
|
+
cpu_name: "ew_#{fn}",
|
|
321
|
+
cpu_src: -> { codegen.unary_math(fn, a_dtype: a.dtype) }) { |x| "#{fn}((float)#{x})" }
|
|
322
|
+
end
|
|
323
|
+
|
|
324
|
+
def unary_abs(a)
|
|
325
|
+
fn = a.dtype == :int32 ? "abs" : "fabs"
|
|
326
|
+
unary(a, out_dtype: a.dtype, cpu_name: "abs_#{a.dtype}",
|
|
327
|
+
cpu_src: -> { codegen.unary_abs(a_dtype: a.dtype) }) { |x| "#{fn}(#{x})" }
|
|
328
|
+
end
|
|
329
|
+
|
|
330
|
+
def clamp_op(a, lo, hi)
|
|
331
|
+
require_computable!(a)
|
|
332
|
+
unless lo.is_a?(Numeric) && hi.is_a?(Numeric)
|
|
333
|
+
raise TypeError, "clamp bounds must be Numeric"
|
|
334
|
+
end
|
|
335
|
+
|
|
336
|
+
dtype = a.dtype == :int32 && lo.is_a?(Integer) && hi.is_a?(Integer) ? :int32 : :float32
|
|
337
|
+
out_dtype = LOW_PRECISION_DTYPES.include?(a.dtype) ? a.dtype : dtype
|
|
338
|
+
ct = CTYPE.fetch(dtype)
|
|
339
|
+
src = <<~KERNEL
|
|
340
|
+
GPU_KERNEL void ew(GPU_ARGS,
|
|
341
|
+
device const #{CTYPE.fetch(a.dtype)}* a,
|
|
342
|
+
constant #{ct}& lo,
|
|
343
|
+
constant #{ct}& hi,
|
|
344
|
+
device #{CTYPE.fetch(out_dtype)}* c,
|
|
345
|
+
constant int& n) {
|
|
346
|
+
int i = GLOBAL_ID_X;
|
|
347
|
+
if (i < n) c[i] = clamp((#{ct})#{read_dtype(a.dtype, 'a[i]')}, lo, hi);
|
|
348
|
+
}
|
|
349
|
+
KERNEL
|
|
350
|
+
|
|
351
|
+
kernel = compiled(
|
|
352
|
+
dialect_name: "ew", dialect_src: src,
|
|
353
|
+
cpu_name: "clamp_ew", cpu_src: -> { codegen.clamp(a_dtype: a.dtype, dtype: dtype) },
|
|
354
|
+
)
|
|
355
|
+
convert = ->(v) { dtype == :float32 ? v.to_f : v }
|
|
356
|
+
launch_into(kernel, a.length, [a, convert.(lo), convert.(hi)], out_dtype: out_dtype, shape: a.shape)
|
|
357
|
+
end
|
|
358
|
+
|
|
359
|
+
def launch_into(kernel, n, inputs, out_dtype:, shape: [n])
|
|
360
|
+
out = allocate_output(out_dtype, n, shape)
|
|
361
|
+
launch_kernel(kernel, grid: (n / BLOCK.to_f).ceil, block: BLOCK, args: [*inputs, out, n])
|
|
362
|
+
out
|
|
363
|
+
rescue StandardError
|
|
364
|
+
out&.free
|
|
365
|
+
raise
|
|
366
|
+
end
|
|
367
|
+
|
|
368
|
+
# Backends that cannot use the dialect use the bridge's own ops_codegen
|
|
369
|
+
# (lib/tilework/backend/cpu/ops_codegen.rb on :cpu), passed with raw: true.
|
|
370
|
+
# The dialect backends' path (dialect_src straight into Tilework.compile) is
|
|
371
|
+
# unchanged (the same pattern as DSLKernel in lib/tilework/dsl.rb).
|
|
372
|
+
def compiled(dialect_name:, dialect_src:, cpu_name:, cpu_src:)
|
|
373
|
+
return Tilework.compile(dialect_src, name: dialect_name) if Tilework.capabilities[:dialect]
|
|
374
|
+
|
|
375
|
+
unless Tilework.bridge.respond_to?(:ops_codegen)
|
|
376
|
+
raise Error, "the #{Tilework.backend.inspect} backend doesn't support dialect kernel strings " \
|
|
377
|
+
"and doesn't implement ops_codegen either"
|
|
378
|
+
end
|
|
379
|
+
Tilework.compile(cpu_src.call, name: cpu_name, raw: true)
|
|
380
|
+
end
|
|
381
|
+
|
|
382
|
+
# Matrix multiply. Requires 2D shapes a: [m, k] and b: [k, n]; the output is
|
|
383
|
+
# [m, n]. A naive implementation where each thread on a 1D grid computes one
|
|
384
|
+
# output element's dot product (no tiling or shared-memory optimization).
|
|
385
|
+
def matmul_op(a, b, out_dtype: nil)
|
|
386
|
+
raise TypeError, "matmul requires a Tilework::Buffer, got #{b.class}" unless b.is_a?(Buffer)
|
|
387
|
+
unless a.matrix? && b.matrix?
|
|
388
|
+
raise ArgumentError, "matmul requires 2D buffers (shape), got #{a.shape.inspect} and #{b.shape.inspect}"
|
|
389
|
+
end
|
|
390
|
+
unless a.cols == b.rows
|
|
391
|
+
raise ArgumentError, "matmul shape mismatch: #{a.shape.inspect} x #{b.shape.inspect}"
|
|
392
|
+
end
|
|
393
|
+
|
|
394
|
+
m, k, n = a.rows, a.cols, b.cols
|
|
395
|
+
dtype = promoted_dtype(a, b)
|
|
396
|
+
out_dtype ||= dtype
|
|
397
|
+
allowed = [dtype]
|
|
398
|
+
allowed << a.dtype if a.dtype == b.dtype && LOW_PRECISION_DTYPES.include?(a.dtype)
|
|
399
|
+
unless allowed.include?(out_dtype)
|
|
400
|
+
raise ArgumentError, "matmul out_dtype must be #{allowed.map(&:inspect).join(' or ')} " \
|
|
401
|
+
"for #{a.dtype.inspect} x #{b.dtype.inspect} inputs"
|
|
402
|
+
end
|
|
403
|
+
|
|
404
|
+
require_computable!(a, b)
|
|
405
|
+
|
|
406
|
+
# Fast GEMM paths. This is the only branch point; anything that takes
|
|
407
|
+
# none of them falls through to the SIMT kernel below.
|
|
408
|
+
# 1. a verified bridge-specific matrix-core kernel (tile-aligned shapes)
|
|
409
|
+
# 2. a GEMM library (no shape constraints)
|
|
410
|
+
# 3. CUDA's WMMA kernel
|
|
411
|
+
# 4. otherwise SIMT
|
|
412
|
+
result = if TensorCore.matrix_core_eligible?(a, b, dtype)
|
|
413
|
+
TensorCore.matrix_core_gemm(a, b, m, k, n)
|
|
414
|
+
elsif TensorCore.gemm_eligible?(a, b, dtype)
|
|
415
|
+
TensorCore.gemm(a, b, m, k, n)
|
|
416
|
+
elsif TensorCore.eligible?(a, b, dtype)
|
|
417
|
+
TensorCore.matmul(a, b, m, k, n)
|
|
418
|
+
end
|
|
419
|
+
return cast_matmul_output(result, out_dtype) if result
|
|
420
|
+
|
|
421
|
+
ct = CTYPE.fetch(dtype)
|
|
422
|
+
src = <<~KERNEL
|
|
423
|
+
GPU_KERNEL void matmul(GPU_ARGS,
|
|
424
|
+
device const #{CTYPE.fetch(a.dtype)}* a,
|
|
425
|
+
device const #{CTYPE.fetch(b.dtype)}* b,
|
|
426
|
+
device #{ct}* c,
|
|
427
|
+
constant int& m,
|
|
428
|
+
constant int& k,
|
|
429
|
+
constant int& n) {
|
|
430
|
+
int i = GLOBAL_ID_X;
|
|
431
|
+
if (i < m * n) {
|
|
432
|
+
int row = i / n;
|
|
433
|
+
int col = i % n;
|
|
434
|
+
#{ct} acc = #{ZERO.fetch(dtype)};
|
|
435
|
+
for (int t = 0; t < k; t++) {
|
|
436
|
+
acc += (#{ct})#{read_dtype(a.dtype, 'a[row * k + t]')} * (#{ct})#{read_dtype(b.dtype, 'b[t * n + col]')};
|
|
437
|
+
}
|
|
438
|
+
c[i] = acc;
|
|
439
|
+
}
|
|
440
|
+
}
|
|
441
|
+
KERNEL
|
|
442
|
+
|
|
443
|
+
kernel = compiled(
|
|
444
|
+
dialect_name: "matmul", dialect_src: src,
|
|
445
|
+
cpu_name: "matmul", cpu_src: -> { codegen.matmul(a_dtype: a.dtype, b_dtype: b.dtype, dtype: dtype) },
|
|
446
|
+
)
|
|
447
|
+
out = allocate_output(dtype, m * n, [m, n])
|
|
448
|
+
launch_kernel(kernel, grid: (m * n / BLOCK.to_f).ceil, block: BLOCK, args: [a, b, out, m, k, n])
|
|
449
|
+
cast_matmul_output(out, out_dtype)
|
|
450
|
+
end
|
|
451
|
+
|
|
452
|
+
def cast_matmul_output(out, out_dtype)
|
|
453
|
+
return out if out.dtype == out_dtype
|
|
454
|
+
|
|
455
|
+
converted = out.cast(out_dtype)
|
|
456
|
+
out.free
|
|
457
|
+
converted
|
|
458
|
+
rescue StandardError
|
|
459
|
+
out.free
|
|
460
|
+
raise
|
|
461
|
+
end
|
|
462
|
+
|
|
463
|
+
# Builds per-threadgroup partial reductions on the GPU.
|
|
464
|
+
#
|
|
465
|
+
# There are two ways to receive the result:
|
|
466
|
+
#
|
|
467
|
+
# buffer.sum # => Float. Runs one pass and folds the
|
|
468
|
+
# # partials on the host
|
|
469
|
+
# buffer.sum(device: true) # => a one-element Buffer, folded all the
|
|
470
|
+
# # way on the device
|
|
471
|
+
#
|
|
472
|
+
# **Receiving on the host drains the queue** — Buffer#to_a waits for
|
|
473
|
+
# pending kernels, so every reduction becomes a full synchronization.
|
|
474
|
+
# Reductions come at the end of loops (loss, sampling, normalizer
|
|
475
|
+
# denominators), so dropping to the host each time rules out "queue the
|
|
476
|
+
# launches and wait once at the end". Use device: true to feed the result
|
|
477
|
+
# straight into the next operation.
|
|
478
|
+
#
|
|
479
|
+
# device: true reapplies the kernel until one partial is left (folding 256
|
|
480
|
+
# at a time, so even a million elements take three passes).
|
|
481
|
+
#
|
|
482
|
+
# Backends with Tilework.host_fallback? (:cpu) bypass kernel generation and
|
|
483
|
+
# compute directly in Ruby. GPU_SHARED/GPU_BARRIER assume every thread in a
|
|
484
|
+
# group runs the body at once and synchronizes at barriers; reproducing that
|
|
485
|
+
# correctly on a CPU would need an interpreter emulating barrier phases.
|
|
486
|
+
# Only these two functions (reduce/arg_reduce) use GPU_SHARED/GPU_BARRIER,
|
|
487
|
+
# internally to Ops (not exposed to user Tilework.kernel {}), so a dedicated
|
|
488
|
+
# interpreter is not worth it (see the top of
|
|
489
|
+
# lib/tilework/backend/cpu/ops_codegen.rb).
|
|
490
|
+
def reduce(kind, buffer, device: false)
|
|
491
|
+
require_computable!(buffer)
|
|
492
|
+
if Tilework.host_fallback?
|
|
493
|
+
value = buffer.to_a.public_send(kind)
|
|
494
|
+
return device ? scalar_buffer(value, accumulator_dtype(buffer)) : value
|
|
495
|
+
end
|
|
496
|
+
|
|
497
|
+
partial = reduce_tree(kind, buffer, device: device)
|
|
498
|
+
return partial if device
|
|
499
|
+
|
|
500
|
+
value = partial.to_a.public_send(kind)
|
|
501
|
+
partial.free
|
|
502
|
+
value
|
|
503
|
+
end
|
|
504
|
+
|
|
505
|
+
# With device: false it runs one pass and returns the partials (the caller
|
|
506
|
+
# folds them); with device: true it reapplies the kernel until one is left.
|
|
507
|
+
def reduce_tree(kind, buffer, device:)
|
|
508
|
+
acc_dtype = accumulator_dtype(buffer)
|
|
509
|
+
source = buffer
|
|
510
|
+
loop do
|
|
511
|
+
groups = (source.length / BLOCK.to_f).ceil
|
|
512
|
+
# Take the partials and the final value from the execution context
|
|
513
|
+
# (borrow and return symmetrically). The context takes over frees, so a
|
|
514
|
+
# Buffer.new temporary would become a slot that is returned but never
|
|
515
|
+
# borrowed, growing the pool every plan run.
|
|
516
|
+
partial = allocate_output(acc_dtype, groups, [groups])
|
|
517
|
+
launch_kernel(reduce_kernel(kind, source.dtype, acc_dtype),
|
|
518
|
+
grid: groups, block: BLOCK, args: [source, partial, source.length])
|
|
519
|
+
source.free unless source.equal?(buffer) # intermediate partials are ours to free
|
|
520
|
+
source = partial
|
|
521
|
+
break if !device || groups == 1
|
|
522
|
+
end
|
|
523
|
+
source
|
|
524
|
+
rescue StandardError
|
|
525
|
+
partial&.free unless partial&.equal?(source)
|
|
526
|
+
source&.free unless source&.equal?(buffer)
|
|
527
|
+
raise
|
|
528
|
+
end
|
|
529
|
+
|
|
530
|
+
# Accumulating in half precision piles up error as the count grows, so
|
|
531
|
+
# reduce in float and return float32 (accumulation is always FP32).
|
|
532
|
+
def accumulator_dtype(buffer) = LOW_PRECISION_DTYPES.include?(buffer.dtype) ? :float32 : buffer.dtype
|
|
533
|
+
|
|
534
|
+
def scalar_buffer(value, dtype)
|
|
535
|
+
allocate_output(dtype, 1, [1]).tap { |buffer| buffer.write([value]) }
|
|
536
|
+
end
|
|
537
|
+
|
|
538
|
+
# From the second pass the input is partials (in the accumulator type), so
|
|
539
|
+
# the input type is a parameter.
|
|
540
|
+
def reduce_kernel(kind, in_dtype, acc_dtype)
|
|
541
|
+
spec = REDUCTIONS.fetch(kind)
|
|
542
|
+
ctype = CTYPE.fetch(acc_dtype)
|
|
543
|
+
element = read_dtype(in_dtype, "a[gid]")
|
|
544
|
+
src = <<~KERNEL
|
|
545
|
+
GPU_KERNEL void reduce_#{kind}(GPU_ARGS,
|
|
546
|
+
device const #{CTYPE.fetch(in_dtype)}* a,
|
|
547
|
+
device #{ctype}* partial,
|
|
548
|
+
constant int& n) {
|
|
549
|
+
GPU_SHARED #{ctype} sh[#{BLOCK}];
|
|
550
|
+
int gid = GLOBAL_ID_X;
|
|
551
|
+
int lid = LOCAL_ID_X;
|
|
552
|
+
sh[lid] = gid < n ? #{element} : #{spec[:identity].fetch(acc_dtype)};
|
|
553
|
+
GPU_BARRIER;
|
|
554
|
+
for (int s = #{BLOCK / 2}; s > 0; s >>= 1) {
|
|
555
|
+
if (lid < s) #{spec[:combine]}
|
|
556
|
+
GPU_BARRIER;
|
|
557
|
+
}
|
|
558
|
+
if (lid == 0) partial[GROUP_ID_X] = sh[0];
|
|
559
|
+
}
|
|
560
|
+
KERNEL
|
|
561
|
+
|
|
562
|
+
compiled(
|
|
563
|
+
dialect_name: "reduce_#{kind}", dialect_src: src,
|
|
564
|
+
cpu_name: "reduce_#{kind}", cpu_src: -> { raise NotImplementedError },
|
|
565
|
+
)
|
|
566
|
+
end
|
|
567
|
+
|
|
568
|
+
# Mean. With device: true both the sum and the division finish on the device.
|
|
569
|
+
def mean_op(buffer, device: false)
|
|
570
|
+
return reduce(:sum, buffer).fdiv(buffer.length) unless device
|
|
571
|
+
|
|
572
|
+
total = reduce(:sum, buffer, device: true)
|
|
573
|
+
mean = arithmetic(:/, total, buffer.length.to_f)
|
|
574
|
+
total.free
|
|
575
|
+
mean
|
|
576
|
+
rescue StandardError
|
|
577
|
+
total&.free
|
|
578
|
+
raise
|
|
579
|
+
end
|
|
580
|
+
|
|
581
|
+
# Reduction over (value, index) pairs. Ties keep the smaller index (strict
|
|
582
|
+
# comparison, left-biased). Later passes keep indices ascending, so the same
|
|
583
|
+
# strict comparison picks the same winners — device: true and host folding
|
|
584
|
+
# agree. See #reduce about the early return on :cpu.
|
|
585
|
+
def arg_reduce(kind, buffer, device: false)
|
|
586
|
+
require_computable!(buffer)
|
|
587
|
+
if Tilework.host_fallback?
|
|
588
|
+
index = host_arg_reduce(kind, buffer.to_a)
|
|
589
|
+
return device ? scalar_buffer(index, :int32) : index
|
|
590
|
+
end
|
|
591
|
+
|
|
592
|
+
values, indices = arg_reduce_pass(kind, buffer, nil)
|
|
593
|
+
unless device
|
|
594
|
+
index = host_arg_reduce(kind, values.to_a, indices.to_a)
|
|
595
|
+
values.free
|
|
596
|
+
indices.free
|
|
597
|
+
return index
|
|
598
|
+
end
|
|
599
|
+
|
|
600
|
+
until values.length == 1
|
|
601
|
+
next_values, next_indices = arg_reduce_pass(kind, values, indices)
|
|
602
|
+
values.free
|
|
603
|
+
indices.free
|
|
604
|
+
values, indices = next_values, next_indices
|
|
605
|
+
end
|
|
606
|
+
values.free
|
|
607
|
+
indices # only the indices are the answer
|
|
608
|
+
end
|
|
609
|
+
|
|
610
|
+
# One pass. With indices nil the indices come from thread ids (the first
|
|
611
|
+
# pass); otherwise the indices chosen by the previous pass are carried along.
|
|
612
|
+
def arg_reduce_pass(kind, source, indices)
|
|
613
|
+
groups = (source.length / BLOCK.to_f).ceil
|
|
614
|
+
# Borrow and return symmetrically (the same reason as reduce_tree). pval
|
|
615
|
+
# is always a temporary freed within this operation; only the final pidx
|
|
616
|
+
# goes to the caller and becomes the Plan's.
|
|
617
|
+
pval = allocate_output(source.dtype, groups, [groups])
|
|
618
|
+
pidx = allocate_output(:int32, groups, [groups])
|
|
619
|
+
args = indices ? [source, indices, pval, pidx] : [source, pval, pidx]
|
|
620
|
+
launch_kernel(arg_reduce_kernel(kind, source.dtype, carry: !indices.nil?),
|
|
621
|
+
grid: groups, block: BLOCK, args: [*args, source.length])
|
|
622
|
+
[pval, pidx]
|
|
623
|
+
rescue StandardError
|
|
624
|
+
pval&.free
|
|
625
|
+
pidx&.free
|
|
626
|
+
raise
|
|
627
|
+
end
|
|
628
|
+
|
|
629
|
+
def arg_reduce_kernel(kind, dtype, carry:)
|
|
630
|
+
cmp = kind == :max ? ">" : "<"
|
|
631
|
+
ctype = CTYPE.fetch(dtype)
|
|
632
|
+
identity = REDUCTIONS.fetch(kind)[:identity].fetch(dtype)
|
|
633
|
+
name = carry ? "arg_#{kind}_pairs" : "arg_#{kind}"
|
|
634
|
+
params = ["device const #{ctype}* a"]
|
|
635
|
+
params << "device const int* idx" if carry
|
|
636
|
+
params += ["device #{ctype}* pval", "device int* pidx", "constant int& n"]
|
|
637
|
+
src = <<~KERNEL
|
|
638
|
+
GPU_KERNEL void #{name}(GPU_ARGS, #{params.join(', ')}) {
|
|
639
|
+
GPU_SHARED #{ctype} sv[#{BLOCK}];
|
|
640
|
+
GPU_SHARED int si[#{BLOCK}];
|
|
641
|
+
int gid = GLOBAL_ID_X;
|
|
642
|
+
int lid = LOCAL_ID_X;
|
|
643
|
+
sv[lid] = gid < n ? a[gid] : #{identity};
|
|
644
|
+
si[lid] = #{carry ? 'gid < n ? idx[gid] : 0' : 'gid'};
|
|
645
|
+
GPU_BARRIER;
|
|
646
|
+
for (int s = #{BLOCK / 2}; s > 0; s >>= 1) {
|
|
647
|
+
if (lid < s && sv[lid + s] #{cmp} sv[lid]) {
|
|
648
|
+
sv[lid] = sv[lid + s];
|
|
649
|
+
si[lid] = si[lid + s];
|
|
650
|
+
}
|
|
651
|
+
GPU_BARRIER;
|
|
652
|
+
}
|
|
653
|
+
if (lid == 0) {
|
|
654
|
+
pval[GROUP_ID_X] = sv[0];
|
|
655
|
+
pidx[GROUP_ID_X] = si[0];
|
|
656
|
+
}
|
|
657
|
+
}
|
|
658
|
+
KERNEL
|
|
659
|
+
|
|
660
|
+
compiled(
|
|
661
|
+
dialect_name: name, dialect_src: src,
|
|
662
|
+
cpu_name: name, cpu_src: -> { raise NotImplementedError },
|
|
663
|
+
)
|
|
664
|
+
end
|
|
665
|
+
|
|
666
|
+
# Host side of index reductions. Without indices, indices count from 0.
|
|
667
|
+
def host_arg_reduce(kind, values, indices = nil)
|
|
668
|
+
best_value = best_index = nil
|
|
669
|
+
values.each_with_index do |value, i|
|
|
670
|
+
next unless best_value.nil? || (kind == :max ? value > best_value : value < best_value)
|
|
671
|
+
|
|
672
|
+
best_value = value
|
|
673
|
+
best_index = indices ? indices[i] : i
|
|
674
|
+
end
|
|
675
|
+
best_index
|
|
676
|
+
end
|
|
677
|
+
end
|
|
678
|
+
|
|
679
|
+
class Buffer
|
|
680
|
+
include Ops
|
|
681
|
+
end
|
|
682
|
+
end
|