tilework 0.18.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE +21 -0
- data/lib/tilework/backend/cpu/dsl_codegen.rb +509 -0
- data/lib/tilework/backend/cpu/ops_codegen.rb +451 -0
- data/lib/tilework/backend/cpu.rb +156 -0
- data/lib/tilework/buffer.rb +753 -0
- data/lib/tilework/capabilities.rb +183 -0
- data/lib/tilework/compat/cuda.h +66 -0
- data/lib/tilework/compat/hip.h +35 -0
- data/lib/tilework/compat/metal.h +45 -0
- data/lib/tilework/conformance.rb +2296 -0
- data/lib/tilework/core_ext.rb +21 -0
- data/lib/tilework/dialect.rb +67 -0
- data/lib/tilework/dsl.rb +722 -0
- data/lib/tilework/dtype/bfloat.rb +31 -0
- data/lib/tilework/dtype/cast.rb +88 -0
- data/lib/tilework/dtype/half.rb +98 -0
- data/lib/tilework/graph/builder.rb +1073 -0
- data/lib/tilework/graph/fusion.rb +520 -0
- data/lib/tilework/graph/grad.rb +443 -0
- data/lib/tilework/graph/high_level.rb +266 -0
- data/lib/tilework/graph/kernels.rb +403 -0
- data/lib/tilework/graph/node.rb +27 -0
- data/lib/tilework/graph/op.rb +290 -0
- data/lib/tilework/graph/optim.rb +183 -0
- data/lib/tilework/graph/plan.rb +1086 -0
- data/lib/tilework/graph/pool.rb +178 -0
- data/lib/tilework/graph/region.rb +210 -0
- data/lib/tilework/graph/residency.rb +449 -0
- data/lib/tilework/graph/state.rb +64 -0
- data/lib/tilework/graph/tensor.rb +124 -0
- data/lib/tilework/graph.rb +213 -0
- data/lib/tilework/kernel.rb +163 -0
- data/lib/tilework/math/conv.rb +270 -0
- data/lib/tilework/math/fft.rb +172 -0
- data/lib/tilework/math/nn.rb +1103 -0
- data/lib/tilework/math/philox.rb +194 -0
- data/lib/tilework/math/random.rb +232 -0
- data/lib/tilework/math/tensor_core.rb +269 -0
- data/lib/tilework/ops.rb +682 -0
- data/lib/tilework/runtime/offload.rb +498 -0
- data/lib/tilework/runtime/submission.rb +163 -0
- data/lib/tilework/runtime/telemetry.rb +112 -0
- data/lib/tilework/scope.rb +33 -0
- data/lib/tilework/version.rb +5 -0
- data/lib/tilework.rb +345 -0
- metadata +92 -0
|
@@ -0,0 +1,451 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Tilework
|
|
4
|
+
module CPU
|
|
5
|
+
# The :cpu backend for the array-op layer. Generates Ruby source for the
|
|
6
|
+
# same op set as lib/tilework/ops.rb (arithmetic, comparisons, elementwise
|
|
7
|
+
# composition, math functions, clamp, matmul). ops.rb is left untouched.
|
|
8
|
+
#
|
|
9
|
+
# Whole-buffer reduce/arg_reduce are not here: threadgroup reductions with
|
|
10
|
+
# GPU_SHARED/GPU_BARRIER would need an interpreter that emulates
|
|
11
|
+
# barrier-separated phases to reproduce correctly on the CPU, which is not
|
|
12
|
+
# worth it for two uses internal to Ops (not exposed to user
|
|
13
|
+
# Tilework.kernel {}). Instead ops.rb computes them directly in Ruby when
|
|
14
|
+
# Tilework.host_fallback? is true, bypassing kernel generation.
|
|
15
|
+
#
|
|
16
|
+
# The generated source follows the same convention as
|
|
17
|
+
# lib/tilework/backend/cpu/dsl_codegen.rb:
|
|
18
|
+
# `{ arg_kinds: [...], proc: lambda { |bufs, scalars, gx, gy, gz, lx, ly,
|
|
19
|
+
# lz, bx, by, bz| ... } }`. The argument order must match what
|
|
20
|
+
# lib/tilework/ops.rb passes to `kernel.launch(args: [...])`.
|
|
21
|
+
module OpsCodegen
|
|
22
|
+
# A descriptor for one argument. role: :buffer | :scalar, dtype: :float32 | :int32.
|
|
23
|
+
Arg = Struct.new(:name, :role, :dtype)
|
|
24
|
+
|
|
25
|
+
module_function
|
|
26
|
+
|
|
27
|
+
def arithmetic_expr(op)
|
|
28
|
+
->(lhs, rhs) { "(#{lhs} #{op} #{rhs})" }
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
def compare_expr(op)
|
|
32
|
+
->(lhs, rhs) { "((#{lhs} #{op} #{rhs}) ? 1 : 0)" }
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def binary_fn_expr(fn)
|
|
36
|
+
case fn
|
|
37
|
+
when "max" then ->(lhs, rhs) { "[#{lhs}, #{rhs}].max" }
|
|
38
|
+
when "min" then ->(lhs, rhs) { "[#{lhs}, #{rhs}].min" }
|
|
39
|
+
else ->(lhs, rhs) { "#{fn}(#{lhs}, #{rhs})" }
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
# --- Elementwise binary (Buffer op Buffer) --- args: [a, b, out, n]
|
|
44
|
+
def buffer_binary(_name, a_dtype:, b_dtype:, out_dtype:, compute_dtype:) # rubocop:disable Lint/UnusedMethodArgument
|
|
45
|
+
cast = out_dtype == :int32 ? "to_i" : "to_f"
|
|
46
|
+
body = <<~RUBY
|
|
47
|
+
i = gx
|
|
48
|
+
if (i < n)
|
|
49
|
+
c[i] = (#{yield("a[i]", "b[i]")}).#{cast}
|
|
50
|
+
end
|
|
51
|
+
RUBY
|
|
52
|
+
wrap([Arg.new("a", :buffer, a_dtype), Arg.new("b", :buffer, b_dtype),
|
|
53
|
+
Arg.new("c", :buffer, out_dtype), Arg.new("n", :scalar, :int32)], body)
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
# --- Elementwise binary (Buffer op scalar) --- args: [a, s, out, n]
|
|
57
|
+
def scalar_binary(_name, a_dtype:, out_dtype:, compute_dtype:, reversed: false) # rubocop:disable Lint/UnusedMethodArgument
|
|
58
|
+
lhs, rhs = "a[i]", "s"
|
|
59
|
+
lhs, rhs = rhs, lhs if reversed
|
|
60
|
+
cast = out_dtype == :int32 ? "to_i" : "to_f"
|
|
61
|
+
body = <<~RUBY
|
|
62
|
+
i = gx
|
|
63
|
+
if (i < n)
|
|
64
|
+
c[i] = (#{yield(lhs, rhs)}).#{cast}
|
|
65
|
+
end
|
|
66
|
+
RUBY
|
|
67
|
+
wrap([Arg.new("a", :buffer, a_dtype), Arg.new("s", :scalar, compute_dtype),
|
|
68
|
+
Arg.new("c", :buffer, out_dtype), Arg.new("n", :scalar, :int32)], body)
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
# --- Unary (math functions, abs) --- args: [a, out, n]
|
|
72
|
+
def unary(_name, a_dtype:, out_dtype:)
|
|
73
|
+
cast = out_dtype == :int32 ? "to_i" : "to_f"
|
|
74
|
+
body = <<~RUBY
|
|
75
|
+
i = gx
|
|
76
|
+
if (i < n)
|
|
77
|
+
c[i] = (#{yield('a[i]')}).#{cast}
|
|
78
|
+
end
|
|
79
|
+
RUBY
|
|
80
|
+
wrap([Arg.new("a", :buffer, a_dtype), Arg.new("c", :buffer, out_dtype),
|
|
81
|
+
Arg.new("n", :scalar, :int32)], body)
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
def unary_math(fn, a_dtype:)
|
|
85
|
+
unary("ew_#{fn}", a_dtype: a_dtype, out_dtype: :float32) { |x| "Math.#{fn}((#{x}).to_f)" }
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
def unary_abs(a_dtype:)
|
|
89
|
+
unary("abs_#{a_dtype}", a_dtype: a_dtype, out_dtype: a_dtype) { |x| "(#{x}).abs" }
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
# --- clamp --- args: [a, lo, hi, out, n]
|
|
93
|
+
def clamp(a_dtype:, dtype:)
|
|
94
|
+
cast = dtype == :int32 ? "to_i" : "to_f"
|
|
95
|
+
body = <<~RUBY
|
|
96
|
+
i = gx
|
|
97
|
+
if (i < n)
|
|
98
|
+
c[i] = (a[i] < lo ? lo : (a[i] > hi ? hi : a[i])).#{cast}
|
|
99
|
+
end
|
|
100
|
+
RUBY
|
|
101
|
+
wrap([Arg.new("a", :buffer, a_dtype), Arg.new("lo", :scalar, dtype), Arg.new("hi", :scalar, dtype),
|
|
102
|
+
Arg.new("c", :buffer, dtype), Arg.new("n", :scalar, :int32)], body)
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
# --- Matrix multiply --- args: [a, b, out, m, k, n]
|
|
106
|
+
def matmul(a_dtype:, b_dtype:, dtype:)
|
|
107
|
+
cast = dtype == :int32 ? "to_i" : "to_f"
|
|
108
|
+
body = <<~RUBY
|
|
109
|
+
i = gx
|
|
110
|
+
if (i < m * n)
|
|
111
|
+
row = i / n
|
|
112
|
+
col = i % n
|
|
113
|
+
acc = 0
|
|
114
|
+
(0...k).each do |t|
|
|
115
|
+
acc = acc + a[row * k + t] * b[t * n + col]
|
|
116
|
+
end
|
|
117
|
+
c[i] = acc.#{cast}
|
|
118
|
+
end
|
|
119
|
+
RUBY
|
|
120
|
+
wrap([Arg.new("a", :buffer, a_dtype), Arg.new("b", :buffer, b_dtype), Arg.new("c", :buffer, dtype),
|
|
121
|
+
Arg.new("m", :scalar, :int32), Arg.new("k", :scalar, :int32), Arg.new("n", :scalar, :int32)], body)
|
|
122
|
+
end
|
|
123
|
+
|
|
124
|
+
# --- Cast --- args: [a, out, n]
|
|
125
|
+
#
|
|
126
|
+
# to_i truncates toward zero, matching the dialect's C cast (Float#round
|
|
127
|
+
# or floor would not).
|
|
128
|
+
def cast(from:, to:)
|
|
129
|
+
body = <<~RUBY
|
|
130
|
+
i = gx
|
|
131
|
+
if (i < n)
|
|
132
|
+
c[i] = (a[i]).#{to == :int32 ? 'to_i' : 'to_f'}
|
|
133
|
+
end
|
|
134
|
+
RUBY
|
|
135
|
+
wrap([Arg.new("a", :buffer, from), Arg.new("c", :buffer, to),
|
|
136
|
+
Arg.new("n", :scalar, :int32)], body)
|
|
137
|
+
end
|
|
138
|
+
|
|
139
|
+
# --- Select by condition --- args: [cond, a, b, out, n]
|
|
140
|
+
#
|
|
141
|
+
# The positions are always [cond, a, b, out, n]; only the side that became
|
|
142
|
+
# a scalar changes role (the same order as the dialect side).
|
|
143
|
+
def select(dtype:, a_scalar:, b_scalar:)
|
|
144
|
+
cast = dtype == :int32 ? "to_i" : "to_f"
|
|
145
|
+
left = a_scalar ? "a" : "a[i]"
|
|
146
|
+
right = b_scalar ? "b" : "b[i]"
|
|
147
|
+
body = <<~RUBY
|
|
148
|
+
i = gx
|
|
149
|
+
if (i < n)
|
|
150
|
+
c[i] = ((cond[i] != 0) ? #{left} : #{right}).#{cast}
|
|
151
|
+
end
|
|
152
|
+
RUBY
|
|
153
|
+
wrap([Arg.new("cond", :buffer, :int32),
|
|
154
|
+
Arg.new("a", a_scalar ? :scalar : :buffer, dtype),
|
|
155
|
+
Arg.new("b", b_scalar ? :scalar : :buffer, dtype),
|
|
156
|
+
Arg.new("c", :buffer, dtype), Arg.new("n", :scalar, :int32)], body)
|
|
157
|
+
end
|
|
158
|
+
|
|
159
|
+
# --- Index along an axis --- args: [out, n]
|
|
160
|
+
def iota(dtype:, size:, inner:)
|
|
161
|
+
body = <<~RUBY
|
|
162
|
+
i = gx
|
|
163
|
+
if (i < n)
|
|
164
|
+
c[i] = ((i / #{inner}) % #{size}).#{dtype == :int32 ? 'to_i' : 'to_f'}
|
|
165
|
+
end
|
|
166
|
+
RUBY
|
|
167
|
+
wrap([Arg.new("c", :buffer, dtype), Arg.new("n", :scalar, :int32)], body)
|
|
168
|
+
end
|
|
169
|
+
|
|
170
|
+
# --- Axis reduction (Tilework::Graph::Kernels) --- args: [a, out, axis_len, inner, n]
|
|
171
|
+
#
|
|
172
|
+
# As noted at the top, **whole-buffer** reduce/arg_reduce use
|
|
173
|
+
# GPU_SHARED/GPU_BARRIER and never come here. An axis reduction is one
|
|
174
|
+
# thread per output element folding in order, with no shared memory or
|
|
175
|
+
# barrier, so it is generated in the same shape as the dialect kernel.
|
|
176
|
+
def axis_reduce(kind:, in_dtype:, out_dtype:)
|
|
177
|
+
cast = out_dtype == :int32 ? "to_i" : "to_f"
|
|
178
|
+
body = <<~RUBY
|
|
179
|
+
i = gx
|
|
180
|
+
if (i < n)
|
|
181
|
+
base = (i / inner) * axis_len * inner + (i % inner)
|
|
182
|
+
acc = a[base]
|
|
183
|
+
(1...axis_len).each do |t|
|
|
184
|
+
v = a[base + t * inner]
|
|
185
|
+
#{axis_combine(kind)}
|
|
186
|
+
end
|
|
187
|
+
c[i] = (#{kind == :mean ? 'acc / axis_len.to_f' : 'acc'}).#{cast}
|
|
188
|
+
end
|
|
189
|
+
RUBY
|
|
190
|
+
wrap([Arg.new("a", :buffer, in_dtype), Arg.new("c", :buffer, out_dtype),
|
|
191
|
+
Arg.new("axis_len", :scalar, :int32), Arg.new("inner", :scalar, :int32),
|
|
192
|
+
Arg.new("n", :scalar, :int32)], body)
|
|
193
|
+
end
|
|
194
|
+
|
|
195
|
+
# --- Index reduction along an axis --- args: [a, out, axis_len, inner, n]
|
|
196
|
+
def axis_arg_reduce(kind:, in_dtype:)
|
|
197
|
+
cmp = kind == :max ? ">" : "<"
|
|
198
|
+
body = <<~RUBY
|
|
199
|
+
i = gx
|
|
200
|
+
if (i < n)
|
|
201
|
+
base = (i / inner) * axis_len * inner + (i % inner)
|
|
202
|
+
best = a[base]
|
|
203
|
+
at = 0
|
|
204
|
+
(1...axis_len).each do |t|
|
|
205
|
+
v = a[base + t * inner]
|
|
206
|
+
if (v #{cmp} best)
|
|
207
|
+
best = v
|
|
208
|
+
at = t
|
|
209
|
+
end
|
|
210
|
+
end
|
|
211
|
+
c[i] = at
|
|
212
|
+
end
|
|
213
|
+
RUBY
|
|
214
|
+
wrap([Arg.new("a", :buffer, in_dtype), Arg.new("c", :buffer, :int32),
|
|
215
|
+
Arg.new("axis_len", :scalar, :int32), Arg.new("inner", :scalar, :int32),
|
|
216
|
+
Arg.new("n", :scalar, :int32)], body)
|
|
217
|
+
end
|
|
218
|
+
|
|
219
|
+
# --- Table lookup --- args: [a, idx, out, n]
|
|
220
|
+
def gather(dtype:, axis_len:, idx_len:, inner:)
|
|
221
|
+
body = <<~RUBY
|
|
222
|
+
i = gx
|
|
223
|
+
if (i < n)
|
|
224
|
+
block = #{idx_len * inner}
|
|
225
|
+
at = idx[(i % block) / #{inner}].to_i
|
|
226
|
+
at = at < 0 ? 0 : (at > #{axis_len - 1} ? #{axis_len - 1} : at)
|
|
227
|
+
c[i] = a[(i / block) * #{axis_len * inner} + at * #{inner} + (i % #{inner})]
|
|
228
|
+
end
|
|
229
|
+
RUBY
|
|
230
|
+
wrap([Arg.new("a", :buffer, dtype), Arg.new("idx", :buffer, :int32),
|
|
231
|
+
Arg.new("c", :buffer, dtype), Arg.new("n", :scalar, :int32)], body)
|
|
232
|
+
end
|
|
233
|
+
|
|
234
|
+
# --- Inverse of table lookup (scatter-add) --- args: [idx, dy, out, n]
|
|
235
|
+
def scatter_add(dtype:, axis_len:, idx_len:, inner:)
|
|
236
|
+
body = <<~RUBY
|
|
237
|
+
i = gx
|
|
238
|
+
if (i < n)
|
|
239
|
+
block = #{axis_len * inner}
|
|
240
|
+
outer = i / block
|
|
241
|
+
j = (i % block) / #{inner}
|
|
242
|
+
lane = i % #{inner}
|
|
243
|
+
acc = 0.0
|
|
244
|
+
t = 0
|
|
245
|
+
while t < #{idx_len}
|
|
246
|
+
at = idx[t].to_i
|
|
247
|
+
at = at < 0 ? 0 : (at > #{axis_len - 1} ? #{axis_len - 1} : at)
|
|
248
|
+
acc = acc + dy[outer * #{idx_len * inner} + t * #{inner} + lane] if at == j
|
|
249
|
+
t = t + 1
|
|
250
|
+
end
|
|
251
|
+
c[i] = acc
|
|
252
|
+
end
|
|
253
|
+
RUBY
|
|
254
|
+
wrap([Arg.new("idx", :buffer, :int32), Arg.new("dy", :buffer, dtype),
|
|
255
|
+
Arg.new("c", :buffer, dtype), Arg.new("n", :scalar, :int32)], body)
|
|
256
|
+
end
|
|
257
|
+
|
|
258
|
+
# --- Border fill --- args: [a, out, fill, n]
|
|
259
|
+
def pad(dtype:, dims:)
|
|
260
|
+
lines = dims.reverse.map do |out_dim, in_dim, before, stride|
|
|
261
|
+
coord = "p = rem % #{out_dim}; rem /= #{out_dim}"
|
|
262
|
+
if before.zero? && in_dim == out_dim
|
|
263
|
+
"#{coord}; src += p * #{stride}"
|
|
264
|
+
else
|
|
265
|
+
"#{coord}; p -= #{before}; inside = inside && p >= 0 && p < #{in_dim}; " \
|
|
266
|
+
"src += (p < 0 || p >= #{in_dim} ? 0 : p) * #{stride}"
|
|
267
|
+
end
|
|
268
|
+
end
|
|
269
|
+
body = <<~RUBY
|
|
270
|
+
i = gx
|
|
271
|
+
if (i < n)
|
|
272
|
+
rem = i
|
|
273
|
+
src = 0
|
|
274
|
+
inside = true
|
|
275
|
+
p = 0
|
|
276
|
+
#{lines.join("\n ")}
|
|
277
|
+
c[i] = inside ? a[src] : fill
|
|
278
|
+
end
|
|
279
|
+
RUBY
|
|
280
|
+
wrap([Arg.new("a", :buffer, dtype), Arg.new("c", :buffer, dtype),
|
|
281
|
+
Arg.new("fill", :scalar, dtype), Arg.new("n", :scalar, :int32)], body)
|
|
282
|
+
end
|
|
283
|
+
|
|
284
|
+
# --- One concat piece --- args: [piece, out, n]
|
|
285
|
+
def concat_piece(dtype:, piece_axis:, total_axis:, inner:, offset:)
|
|
286
|
+
body = <<~RUBY
|
|
287
|
+
i = gx
|
|
288
|
+
if (i < n)
|
|
289
|
+
block = #{piece_axis * inner}
|
|
290
|
+
c[(i / block) * #{total_axis * inner} + (#{offset} + (i % block) / #{inner}) * #{inner} +
|
|
291
|
+
(i % #{inner})] = a[i]
|
|
292
|
+
end
|
|
293
|
+
RUBY
|
|
294
|
+
wrap([Arg.new("a", :buffer, dtype), Arg.new("c", :buffer, dtype),
|
|
295
|
+
Arg.new("n", :scalar, :int32)], body)
|
|
296
|
+
end
|
|
297
|
+
|
|
298
|
+
# --- Index remap (broadcast / axis permutation) --- args: [a, out, n]
|
|
299
|
+
#
|
|
300
|
+
# dims is a list of [output size, input-side stride] from dim 0. A
|
|
301
|
+
# stretched dim has stride 0, so its term drops out of the index.
|
|
302
|
+
def remap(dtype:, dims:)
|
|
303
|
+
lines = dims.reverse.map do |size, stride|
|
|
304
|
+
stride.zero? ? "rem /= #{size}" : "src += (rem % #{size}) * #{stride}; rem /= #{size}"
|
|
305
|
+
end
|
|
306
|
+
body = <<~RUBY
|
|
307
|
+
i = gx
|
|
308
|
+
if (i < n)
|
|
309
|
+
rem = i
|
|
310
|
+
src = 0
|
|
311
|
+
#{lines.join("\n ")}
|
|
312
|
+
c[i] = a[src]
|
|
313
|
+
end
|
|
314
|
+
RUBY
|
|
315
|
+
wrap([Arg.new("a", :buffer, dtype), Arg.new("c", :buffer, dtype),
|
|
316
|
+
Arg.new("n", :scalar, :int32)], body)
|
|
317
|
+
end
|
|
318
|
+
|
|
319
|
+
def axis_combine(kind)
|
|
320
|
+
case kind
|
|
321
|
+
when :sum, :mean then "acc = acc + v"
|
|
322
|
+
when :min then "acc = v < acc ? v : acc"
|
|
323
|
+
when :max then "acc = v > acc ? v : acc"
|
|
324
|
+
else raise Error, "unknown reduction #{kind.inspect}"
|
|
325
|
+
end
|
|
326
|
+
end
|
|
327
|
+
|
|
328
|
+
# --- Fused elementwise (Tilework::Graph::Fusion) --- args: [a0.., s0.., out, n]
|
|
329
|
+
#
|
|
330
|
+
# core builds the expression tree (graph/fusion.rb); this only renders it
|
|
331
|
+
# as Ruby while **writing it the way the unfused path does**: no casts on
|
|
332
|
+
# reads, and each node's result rounded to its output type. That makes
|
|
333
|
+
# "fused == unfused" hold bit for bit.
|
|
334
|
+
def fused(expr, input_dtypes:, scalar_dtypes:, out_dtype:)
|
|
335
|
+
args = input_dtypes.each_with_index.map { |dtype, i| Arg.new("a#{i}", :buffer, dtype) }
|
|
336
|
+
args += scalar_dtypes.each_with_index.map { |dtype, i| Arg.new("s#{i}", :scalar, dtype) }
|
|
337
|
+
args << Arg.new("c", :buffer, out_dtype)
|
|
338
|
+
args << Arg.new("n", :scalar, :int32)
|
|
339
|
+
|
|
340
|
+
body = <<~RUBY
|
|
341
|
+
i = gx
|
|
342
|
+
if (i < n)
|
|
343
|
+
c[i] = #{fused_expr(expr)}
|
|
344
|
+
end
|
|
345
|
+
RUBY
|
|
346
|
+
wrap(args, body)
|
|
347
|
+
end
|
|
348
|
+
|
|
349
|
+
# --- Row-wise fusion (Tilework::Graph::Fusion) --- args: [a0.., s0.., out, rows, cols]
|
|
350
|
+
#
|
|
351
|
+
# The key is rounding the reduction's result through store. The unfused
|
|
352
|
+
# path writes the reduction to a buffer, rounding it to single precision;
|
|
353
|
+
# without reproducing that here the low bits would differ between fused
|
|
354
|
+
# and unfused (the same reason each node is rounded in elementwise fusion).
|
|
355
|
+
def row_fused(expr, reductions:, input_dtypes:, scalar_dtypes:, out_dtype:)
|
|
356
|
+
args = input_dtypes.each_with_index.map { |dtype, i| Arg.new("a#{i}", :buffer, dtype) }
|
|
357
|
+
args += scalar_dtypes.each_with_index.map { |dtype, i| Arg.new("s#{i}", :scalar, dtype) }
|
|
358
|
+
args << Arg.new("c", :buffer, out_dtype)
|
|
359
|
+
args << Arg.new("rows", :scalar, :int32)
|
|
360
|
+
args << Arg.new("cols", :scalar, :int32)
|
|
361
|
+
|
|
362
|
+
passes = reductions.each_with_index.map do |reduction, k|
|
|
363
|
+
<<~PASS.chomp
|
|
364
|
+
acc#{k} = nil
|
|
365
|
+
(0...cols).each do |ci|
|
|
366
|
+
i = base + ci
|
|
367
|
+
v = #{fused_expr(reduction[:expr])}
|
|
368
|
+
acc#{k} = (ci == 0) ? v : (#{row_combine(reduction[:kind], "acc#{k}", 'v')})
|
|
369
|
+
end
|
|
370
|
+
red#{k} = #{store(row_finish(reduction[:kind], "acc#{k}"), reduction[:dtype])}
|
|
371
|
+
PASS
|
|
372
|
+
end
|
|
373
|
+
|
|
374
|
+
body = <<~RUBY
|
|
375
|
+
r = gx
|
|
376
|
+
if (r < rows)
|
|
377
|
+
base = r * cols
|
|
378
|
+
#{passes.join("\n ")}
|
|
379
|
+
(0...cols).each do |ci|
|
|
380
|
+
i = base + ci
|
|
381
|
+
c[i] = #{fused_expr(expr)}
|
|
382
|
+
end
|
|
383
|
+
end
|
|
384
|
+
RUBY
|
|
385
|
+
wrap(args, body)
|
|
386
|
+
end
|
|
387
|
+
|
|
388
|
+
def row_combine(kind, acc, value)
|
|
389
|
+
case kind
|
|
390
|
+
when :sum, :mean then "#{acc} + #{value}"
|
|
391
|
+
when :min then "(#{value} < #{acc} ? #{value} : #{acc})"
|
|
392
|
+
when :max then "(#{value} > #{acc} ? #{value} : #{acc})"
|
|
393
|
+
else raise Error, "cannot fuse the reduction #{kind.inspect}"
|
|
394
|
+
end
|
|
395
|
+
end
|
|
396
|
+
|
|
397
|
+
def row_finish(kind, acc) = kind == :mean ? "(#{acc} / cols.to_f)" : acc
|
|
398
|
+
|
|
399
|
+
def fused_expr(expr)
|
|
400
|
+
case expr.kind
|
|
401
|
+
# Index remaps (folded broadcasts) use the same expression as the
|
|
402
|
+
# dialect side. Integer division truncates the same in Ruby and C.
|
|
403
|
+
when :buffer then "a#{expr.index}[#{Graph::Fusion.index_expr(expr.map)}]"
|
|
404
|
+
when :reduced then "red#{expr.index}"
|
|
405
|
+
when :scalar then "s#{expr.index}"
|
|
406
|
+
when :binary then store("(#{fused_expr(expr.children[0])} #{expr.op} #{fused_expr(expr.children[1])})",
|
|
407
|
+
expr.dtype)
|
|
408
|
+
when :compare then store("((#{fused_expr(expr.children[0])} #{expr.op} " \
|
|
409
|
+
"#{fused_expr(expr.children[1])}) ? 1 : 0)", expr.dtype)
|
|
410
|
+
when :math then store("Math.#{expr.op}((#{fused_expr(expr.children.first)}).to_f)", expr.dtype)
|
|
411
|
+
when :abs then store("(#{fused_expr(expr.children.first)}).abs", expr.dtype)
|
|
412
|
+
when :cast then store(fused_expr(expr.children.first), expr.dtype)
|
|
413
|
+
when :select then store("((#{fused_expr(expr.children[0])}) != 0 ? " \
|
|
414
|
+
"#{fused_expr(expr.children[1])} : #{fused_expr(expr.children[2])})",
|
|
415
|
+
expr.dtype)
|
|
416
|
+
else raise Error, "unknown fused expression #{expr.kind.inspect}"
|
|
417
|
+
end
|
|
418
|
+
end
|
|
419
|
+
|
|
420
|
+
# The rounding that corresponds to the unfused path's write to a buffer.
|
|
421
|
+
#
|
|
422
|
+
# The pack/unpack at float32 is not decoration: Ruby computes in double,
|
|
423
|
+
# so fusion would **drop the per-node rounding to single precision from
|
|
424
|
+
# the middle of the expression**. Without reproducing it here, the low
|
|
425
|
+
# bits would differ between fused and unfused (dialect backends keep
|
|
426
|
+
# intermediates in float, so they never have this problem).
|
|
427
|
+
def store(rendered, dtype)
|
|
428
|
+
return "(#{rendered}).to_i" if dtype == :int32
|
|
429
|
+
|
|
430
|
+
"[#{rendered}].pack(\"f\").unpack1(\"f\")"
|
|
431
|
+
end
|
|
432
|
+
|
|
433
|
+
def wrap(args, body)
|
|
434
|
+
arg_kinds = args.map { |a| :"#{a.dtype}_#{a.role}" }
|
|
435
|
+
preamble = args.each_with_index.map do |a, idx|
|
|
436
|
+
a.role == :buffer ? "#{a.name} = bufs[#{idx}]" : "#{a.name} = scalars[#{idx}]"
|
|
437
|
+
end
|
|
438
|
+
<<~RUBY
|
|
439
|
+
{
|
|
440
|
+
arg_kinds: #{arg_kinds.inspect},
|
|
441
|
+
proc: lambda do |bufs, scalars, gx, gy, gz, lx, ly, lz, bx, by, bz|
|
|
442
|
+
#{preamble.join("\n ")}
|
|
443
|
+
#{body}
|
|
444
|
+
end,
|
|
445
|
+
}
|
|
446
|
+
RUBY
|
|
447
|
+
end
|
|
448
|
+
private_class_method :wrap
|
|
449
|
+
end
|
|
450
|
+
end
|
|
451
|
+
end
|
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# The bridge for Tilework.backend = :cpu. Runs Tilework.kernel {} and Buffer
|
|
4
|
+
# operators without a GPU and serves as the conformance reference. It cannot
|
|
5
|
+
# interpret dialect strings, which it declares as capabilities[:dialect] = false.
|
|
6
|
+
#
|
|
7
|
+
# A pipeline's "source" is Ruby source generated by backend/cpu/dsl_codegen.rb
|
|
8
|
+
# and ops_codegen.rb; eval'ing it yields the Hash
|
|
9
|
+
# `{ arg_kinds: [...], proc: lambda { |bufs, scalars, gx,gy,gz, lx,ly,lz,
|
|
10
|
+
# bx,by,bz| ... } }`. dispatch uses it to:
|
|
11
|
+
# 1. unpack native_args per arg_kinds once for the whole dispatch
|
|
12
|
+
# (unpacking/packing per thread would be needlessly slow).
|
|
13
|
+
# 2. loop over grid[i]*block[i] threads in three nested loops, computing the
|
|
14
|
+
# GLOBAL_ID_X / LOCAL_ID_X / GROUP_ID_X equivalents and calling proc.
|
|
15
|
+
# 3. after the loop, write buffer arguments back to their bytes once.
|
|
16
|
+
require "tilework/backend/cpu/dsl_codegen"
|
|
17
|
+
require "tilework/backend/cpu/ops_codegen"
|
|
18
|
+
|
|
19
|
+
module Tilework
|
|
20
|
+
module CPUBridge
|
|
21
|
+
module_function
|
|
22
|
+
|
|
23
|
+
# The CPU has no workgroup/threadgroup size limit, so return a value large
|
|
24
|
+
# enough that Kernel#validate_block! never trips.
|
|
25
|
+
UNLIMITED_THREADS = 2**20
|
|
26
|
+
|
|
27
|
+
# offset is for views; always 0 on a real Handle.
|
|
28
|
+
Handle = Struct.new(:data, :freed, :offset)
|
|
29
|
+
|
|
30
|
+
def device_name
|
|
31
|
+
"CPU (reference)"
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def device_available?
|
|
35
|
+
true
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
# Some nn.rb / ops.rb operations are faster computed directly in Ruby than
|
|
39
|
+
# through a kernel. That is an optimization, not something the capabilities
|
|
40
|
+
# vocabulary (what a backend can do) can express, so the bridge declares it
|
|
41
|
+
# itself instead of core naming the backend (== :cpu).
|
|
42
|
+
def host_fallback?
|
|
43
|
+
true
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
# A backend that does not accept dialect kernel strings supplies its own
|
|
47
|
+
# source generators here. ops.rb / dsl.rb call them through Tilework.bridge
|
|
48
|
+
# and never see the name :cpu.
|
|
49
|
+
def ops_codegen
|
|
50
|
+
Tilework::CPU::OpsCodegen
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
def dsl_codegen
|
|
54
|
+
Tilework::CPU
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
def buffer_alloc(bytesize)
|
|
58
|
+
Handle.new(("\x00" * bytesize).b, false, 0)
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
# A view shares the same data with a shifted offset and frees nothing.
|
|
62
|
+
def buffer_view(handle, byte_offset)
|
|
63
|
+
Handle.new(handle.data, false, handle.offset + byte_offset)
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
def buffer_write(handle, bytes, offset)
|
|
67
|
+
handle.data[handle.offset + offset, bytes.bytesize] = bytes
|
|
68
|
+
nil
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
def buffer_read(handle, bytesize, offset)
|
|
72
|
+
handle.data[handle.offset + offset, bytesize]
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
def buffer_free(handle)
|
|
76
|
+
handle.freed = true
|
|
77
|
+
nil
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
def pipeline_create(src, _name)
|
|
81
|
+
eval(src) # rubocop:disable Security/Eval -- the source comes from cpu/dsl_codegen.rb / cpu/ops_codegen.rb, not external input
|
|
82
|
+
rescue StandardError, SyntaxError => e
|
|
83
|
+
# Bridge contract: raise compile errors as RuntimeError
|
|
84
|
+
# (Kernel.create_pipeline wraps it in Tilework::CompileError).
|
|
85
|
+
raise RuntimeError, e.message
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
def pipeline_max_threads(_pipeline)
|
|
89
|
+
UNLIMITED_THREADS
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
def pipeline_free(_pipeline)
|
|
93
|
+
nil
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
def dispatch(pipeline, native_args, grid, block, _wait)
|
|
97
|
+
arg_kinds = pipeline.fetch(:arg_kinds)
|
|
98
|
+
bufs = Array.new(arg_kinds.length)
|
|
99
|
+
scalars = Array.new(arg_kinds.length)
|
|
100
|
+
writeback = [] # [[idx, handle, encoder], ...] written back together after the loop
|
|
101
|
+
|
|
102
|
+
native_args.each_with_index do |arg, idx|
|
|
103
|
+
kind = arg_kinds[idx]
|
|
104
|
+
# Silently unpacking an unknown dtype as int32 would corrupt values
|
|
105
|
+
# without an error, so only known dtypes pass.
|
|
106
|
+
codec = case kind.to_s
|
|
107
|
+
when /\Afloat32_/ then [->(s) { s.unpack("f*") }, ->(a) { a.pack("f*") }, "f"]
|
|
108
|
+
when /\Aint32_/ then [->(s) { s.unpack("l*") }, ->(a) { a.pack("l*") }, "l"]
|
|
109
|
+
when /\Abfloat16_/ then [BFloat.method(:decode), BFloat.method(:encode), nil]
|
|
110
|
+
else raise Error, "the :cpu backend cannot handle argument kind #{kind.inspect}"
|
|
111
|
+
end
|
|
112
|
+
if kind.end_with?("_buffer")
|
|
113
|
+
# A view sees only what follows its offset. The tail runs past the
|
|
114
|
+
# logical length, but kernel guards never write out of range.
|
|
115
|
+
bufs[idx] = codec[0].call(arg.data.byteslice(arg.offset, arg.data.bytesize - arg.offset))
|
|
116
|
+
writeback << [idx, arg, codec[1]]
|
|
117
|
+
else
|
|
118
|
+
scalars[idx] = arg.unpack1(codec[2])
|
|
119
|
+
end
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
total = [grid[0] * block[0], grid[1] * block[1], grid[2] * block[2]]
|
|
123
|
+
proc = pipeline.fetch(:proc)
|
|
124
|
+
(0...total[2]).each do |gz|
|
|
125
|
+
bz = gz / block[2]
|
|
126
|
+
lz = gz % block[2]
|
|
127
|
+
(0...total[1]).each do |gy|
|
|
128
|
+
by = gy / block[1]
|
|
129
|
+
ly = gy % block[1]
|
|
130
|
+
(0...total[0]).each do |gx|
|
|
131
|
+
bx = gx / block[0]
|
|
132
|
+
lx = gx % block[0]
|
|
133
|
+
proc.call(bufs, scalars, gx, gy, gz, lx, ly, lz, bx, by, bz)
|
|
134
|
+
end
|
|
135
|
+
end
|
|
136
|
+
end
|
|
137
|
+
|
|
138
|
+
# A view writes back into the middle of the parent's data (replacing the
|
|
139
|
+
# data would detach the String shared with the parent).
|
|
140
|
+
writeback.each do |idx, handle, encoder|
|
|
141
|
+
handle.data[handle.offset, handle.data.bytesize - handle.offset] = encoder.call(bufs[idx])
|
|
142
|
+
end
|
|
143
|
+
nil
|
|
144
|
+
rescue StandardError => e
|
|
145
|
+
# Bridge contract: raise run-time errors as RuntimeError
|
|
146
|
+
# (Kernel#dispatch wraps it in Tilework::LaunchError).
|
|
147
|
+
raise RuntimeError, e.message
|
|
148
|
+
end
|
|
149
|
+
|
|
150
|
+
def synchronize
|
|
151
|
+
nil # :cpu dispatch is always synchronous, so there is nothing to do
|
|
152
|
+
end
|
|
153
|
+
end
|
|
154
|
+
|
|
155
|
+
register_backend(:cpu, CPUBridge)
|
|
156
|
+
end
|