tilework 0.18.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE +21 -0
- data/lib/tilework/backend/cpu/dsl_codegen.rb +509 -0
- data/lib/tilework/backend/cpu/ops_codegen.rb +451 -0
- data/lib/tilework/backend/cpu.rb +156 -0
- data/lib/tilework/buffer.rb +753 -0
- data/lib/tilework/capabilities.rb +183 -0
- data/lib/tilework/compat/cuda.h +66 -0
- data/lib/tilework/compat/hip.h +35 -0
- data/lib/tilework/compat/metal.h +45 -0
- data/lib/tilework/conformance.rb +2296 -0
- data/lib/tilework/core_ext.rb +21 -0
- data/lib/tilework/dialect.rb +67 -0
- data/lib/tilework/dsl.rb +722 -0
- data/lib/tilework/dtype/bfloat.rb +31 -0
- data/lib/tilework/dtype/cast.rb +88 -0
- data/lib/tilework/dtype/half.rb +98 -0
- data/lib/tilework/graph/builder.rb +1073 -0
- data/lib/tilework/graph/fusion.rb +520 -0
- data/lib/tilework/graph/grad.rb +443 -0
- data/lib/tilework/graph/high_level.rb +266 -0
- data/lib/tilework/graph/kernels.rb +403 -0
- data/lib/tilework/graph/node.rb +27 -0
- data/lib/tilework/graph/op.rb +290 -0
- data/lib/tilework/graph/optim.rb +183 -0
- data/lib/tilework/graph/plan.rb +1086 -0
- data/lib/tilework/graph/pool.rb +178 -0
- data/lib/tilework/graph/region.rb +210 -0
- data/lib/tilework/graph/residency.rb +449 -0
- data/lib/tilework/graph/state.rb +64 -0
- data/lib/tilework/graph/tensor.rb +124 -0
- data/lib/tilework/graph.rb +213 -0
- data/lib/tilework/kernel.rb +163 -0
- data/lib/tilework/math/conv.rb +270 -0
- data/lib/tilework/math/fft.rb +172 -0
- data/lib/tilework/math/nn.rb +1103 -0
- data/lib/tilework/math/philox.rb +194 -0
- data/lib/tilework/math/random.rb +232 -0
- data/lib/tilework/math/tensor_core.rb +269 -0
- data/lib/tilework/ops.rb +682 -0
- data/lib/tilework/runtime/offload.rb +498 -0
- data/lib/tilework/runtime/submission.rb +163 -0
- data/lib/tilework/runtime/telemetry.rb +112 -0
- data/lib/tilework/scope.rb +33 -0
- data/lib/tilework/version.rb +5 -0
- data/lib/tilework.rb +345 -0
- metadata +92 -0
|
@@ -0,0 +1,266 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Tilework
|
|
4
|
+
module Graph
|
|
5
|
+
# The contract for high-level ops (Tier 1 in docs/graph-ir.md).
|
|
6
|
+
#
|
|
7
|
+
# A function like `NN.softmax` is high-level **because it is fast**, not
|
|
8
|
+
# because it cannot be written any other way — that is this layer's
|
|
9
|
+
# premise, and this table makes it checkable. Each row says how to write
|
|
10
|
+
# the function in primitives alone.
|
|
11
|
+
#
|
|
12
|
+
# HighLevel.define(:softmax) do |_g, inputs, _tensors, _args, _opts|
|
|
13
|
+
# e = (inputs[0] - inputs[0].max(axis: -1, keepdims: true)).exp
|
|
14
|
+
# e / e.sum(axis: -1, keepdims: true)
|
|
15
|
+
# end
|
|
16
|
+
#
|
|
17
|
+
# The default execution path is still the hand-written fused kernel (a
|
|
18
|
+
# `:nn` node); the decomposition does not run. Writing it down still pays
|
|
19
|
+
# three ways:
|
|
20
|
+
#
|
|
21
|
+
# 1. **Portability** — there is a form that runs on a backend without
|
|
22
|
+
# that fused kernel (every backend has them today, so it is unused)
|
|
23
|
+
# 2. **Backward (#22)** — gradients follow from primitive vjps alone, so a
|
|
24
|
+
# high-level op can be "decomposed, then differentiated"; hand-written
|
|
25
|
+
# vjps are added only where speed demands
|
|
26
|
+
# 3. **Oracle** — `Graph.lower_high_level = true` switches to the
|
|
27
|
+
# decomposition to compare against the fused kernel, the same trick as
|
|
28
|
+
# verifying Fusion on :cpu, so both paths are checked without a GPU in CI
|
|
29
|
+
#
|
|
30
|
+
# Agreement is to a tolerance, not bits: the decomposition reduces in a
|
|
31
|
+
# different order and accumulates in a different type than the hand-written
|
|
32
|
+
# kernel, which cannot be matched.
|
|
33
|
+
#
|
|
34
|
+
# **Some functions have no decomposition.** `gelu_exact` needs erf, which
|
|
35
|
+
# is not a primitive, and `groupnorm` folds across channels, which is not a
|
|
36
|
+
# row-wise reduction. Functions not in the table stay `:nn` nodes — the
|
|
37
|
+
# table never pretends to express what it cannot.
|
|
38
|
+
module HighLevel
|
|
39
|
+
REGISTRY = {}
|
|
40
|
+
# The capability the fast path needs. None means every backend has it
|
|
41
|
+
# (NN's fused kernels can be generated anywhere).
|
|
42
|
+
REQUIREMENTS = {}
|
|
43
|
+
|
|
44
|
+
module_function
|
|
45
|
+
|
|
46
|
+
def define(name, requires: nil, &lower)
|
|
47
|
+
raise Error, "high-level op #{name.inspect} is already defined" if REGISTRY.key?(name)
|
|
48
|
+
raise ArgumentError, "define needs a block" if lower.nil?
|
|
49
|
+
|
|
50
|
+
REGISTRY[name] = lower
|
|
51
|
+
REQUIREMENTS[name] = requires if requires
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
def defined?(name) = REGISTRY.key?(name)
|
|
55
|
+
def names = REGISTRY.keys
|
|
56
|
+
|
|
57
|
+
# Decompose in two cases: **explicitly switched on as the oracle**, and
|
|
58
|
+
# **this backend lacks the fast path**. The latter is portability itself;
|
|
59
|
+
# conv2d is the first instance (:cpu has no convolution).
|
|
60
|
+
def lower?(name)
|
|
61
|
+
return false unless REGISTRY.key?(name)
|
|
62
|
+
|
|
63
|
+
Graph.lower_high_level || !available?(name)
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
# Whether the fast path is available. A function with no requirement always is.
|
|
67
|
+
def available?(name)
|
|
68
|
+
capability = REQUIREMENTS[name]
|
|
69
|
+
capability.nil? || Op.satisfied?(capability)
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
def lower(builder, name, inputs, tensors:, args:, opts:)
|
|
73
|
+
REGISTRY.fetch(name).call(builder, inputs, tensors, args, opts)
|
|
74
|
+
end
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
# Every activation is a single expression, so they are written together
|
|
78
|
+
# here, in the same form as nn.rb's ACTIVATIONS. sigmoid / silu split on
|
|
79
|
+
# the sign because `exp(-v)` overflows for very negative v (that side uses
|
|
80
|
+
# the rearranged form).
|
|
81
|
+
ACTIVATION_EXPRS = {
|
|
82
|
+
relu: ->(g, x) { g.where(x > 0.0, x, 0.0) },
|
|
83
|
+
sigmoid: ->(g, x) { g.where(x >= 0.0, 1.0 / (1.0 + (-1.0 * x).exp), x.exp / (1.0 + x.exp)) },
|
|
84
|
+
silu: ->(g, x) { x * ACTIVATION_EXPRS[:sigmoid].call(g, x) },
|
|
85
|
+
gelu_tanh: lambda { |_g, x|
|
|
86
|
+
inner = 0.7978845608028654 * (x + (0.044715 * x * x * x))
|
|
87
|
+
0.5 * x * (1.0 + inner.tanh)
|
|
88
|
+
},
|
|
89
|
+
}.freeze
|
|
90
|
+
|
|
91
|
+
HighLevel.define(:softmax) do |_g, inputs, _tensors, _args, _opts|
|
|
92
|
+
x = inputs[0]
|
|
93
|
+
# Subtract the max before exp, like the hand-written kernel (prevents overflow).
|
|
94
|
+
e = (x - x.max(axis: -1, keepdims: true)).exp
|
|
95
|
+
e / e.sum(axis: -1, keepdims: true)
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
HighLevel.define(:layernorm) do |_g, inputs, tensors, _args, opts|
|
|
99
|
+
x = inputs[0]
|
|
100
|
+
centered = x - x.mean(axis: -1, keepdims: true)
|
|
101
|
+
variance = (centered * centered).mean(axis: -1, keepdims: true)
|
|
102
|
+
y = centered / (variance + opts.fetch(:eps)).sqrt
|
|
103
|
+
y *= tensors[:gamma] if tensors[:gamma]
|
|
104
|
+
y += tensors[:beta] if tensors[:beta]
|
|
105
|
+
y
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
HighLevel.define(:rmsnorm) do |_g, inputs, tensors, _args, opts|
|
|
109
|
+
x = inputs[0]
|
|
110
|
+
# No mean subtraction (that is the definition of rmsnorm).
|
|
111
|
+
y = x / ((x * x).mean(axis: -1, keepdims: true) + opts.fetch(:eps)).sqrt
|
|
112
|
+
y *= tensors[:gamma] if tensors[:gamma]
|
|
113
|
+
y
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
# Loss reduction. Anything but :none folds everything into shape [1] at the end.
|
|
117
|
+
LOSS_REDUCE = lambda { |value, reduction|
|
|
118
|
+
case reduction
|
|
119
|
+
when :none then value
|
|
120
|
+
when :sum then value.sum
|
|
121
|
+
else value.mean
|
|
122
|
+
end
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
HighLevel.define(:mse) do |_g, inputs, _tensors, _args, opts|
|
|
126
|
+
# Losses are kept in float (the same rule as nn.rb's loss_dtype).
|
|
127
|
+
pred, target = inputs.map { |t| Ops::LOW_PRECISION_DTYPES.include?(t.dtype) ? t.cast(:float32) : t }
|
|
128
|
+
d = pred - target
|
|
129
|
+
LOSS_REDUCE.call(d * d, opts.fetch(:reduction))
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
# log-sum-exp in plain ops. Like the hand-written kernel it subtracts the
|
|
133
|
+
# max before exp, so large logits do not overflow.
|
|
134
|
+
#
|
|
135
|
+
# The label's logit is picked with an iota equality mask rather than
|
|
136
|
+
# gather. That needs only elementwise ops and a row reduction, so it
|
|
137
|
+
# differentiates without going through gather's vjp (scatter-add).
|
|
138
|
+
# Clamping out-of-range labels to the ends also matches the hand-written kernel.
|
|
139
|
+
HighLevel.define(:cross_entropy) do |g, inputs, _tensors, _args, opts|
|
|
140
|
+
logits, labels = inputs
|
|
141
|
+
x = Ops::LOW_PRECISION_DTYPES.include?(logits.dtype) ? logits.cast(:float32) : logits
|
|
142
|
+
cols = x.shape[-1]
|
|
143
|
+
rows_shape = x.shape[0..-2]
|
|
144
|
+
|
|
145
|
+
m = x.max(axis: -1, keepdims: true)
|
|
146
|
+
lse = m + (x - m).exp.sum(axis: -1, keepdims: true).log
|
|
147
|
+
|
|
148
|
+
clamped = g.where(labels < 0, 0, labels)
|
|
149
|
+
clamped = g.where(clamped > (cols - 1), cols - 1, clamped)
|
|
150
|
+
picked = g.where(g.iota(x.shape, axis: -1).eq(clamped.reshape(*rows_shape, 1)), x, 0.0)
|
|
151
|
+
.sum(axis: -1, keepdims: true)
|
|
152
|
+
|
|
153
|
+
LOSS_REDUCE.call((lse - picked).reshape(*rows_shape), opts.fetch(:reduction))
|
|
154
|
+
end
|
|
155
|
+
|
|
156
|
+
HighLevel.define(:transpose) do |g, inputs, _tensors, _args, _opts|
|
|
157
|
+
g.permute(inputs[0], 1, 0)
|
|
158
|
+
end
|
|
159
|
+
|
|
160
|
+
# Heads split the last dim evenly ([n, d] with d = heads x dh). Split
|
|
161
|
+
# (reshape), bring heads to the front (permute), compute all heads with a
|
|
162
|
+
# batched matmul, then restore. bias arrives as [heads * n, m], which is
|
|
163
|
+
# heads stacked vertically, so reading it as [heads, n, m] is enough (same
|
|
164
|
+
# element order).
|
|
165
|
+
HighLevel.define(:multi_head_attention) do |_g, inputs, tensors, _args, opts|
|
|
166
|
+
q, k, v = inputs
|
|
167
|
+
heads = opts.fetch(:heads)
|
|
168
|
+
n, d = q.shape
|
|
169
|
+
m = k.shape[0]
|
|
170
|
+
dh = d / heads
|
|
171
|
+
|
|
172
|
+
scores = q.reshape(n, heads, dh).permute(1, 0, 2) # [h, n, dh]
|
|
173
|
+
.matmul(k.reshape(m, heads, dh).permute(1, 2, 0)) * opts.fetch(:scale)
|
|
174
|
+
scores += tensors[:bias].reshape(heads, n, m) if tensors[:bias]
|
|
175
|
+
NN.softmax(scores) # rows = last dim
|
|
176
|
+
.matmul(v.reshape(m, heads, dh).permute(1, 0, 2)) # [h, n, dh]
|
|
177
|
+
.permute(1, 0, 2).reshape(n, d)
|
|
178
|
+
end
|
|
179
|
+
|
|
180
|
+
# Opens a 2D convolution into im2col + matmul.
|
|
181
|
+
#
|
|
182
|
+
# It never runs while a fast path (cuDNN / MPSGraph) exists. It is here so
|
|
183
|
+
# graphs run on a backend without convolution (:cpu) and so backward
|
|
184
|
+
# follows from primitive vjps alone.
|
|
185
|
+
#
|
|
186
|
+
# The index table is built inside the graph from iota and integer
|
|
187
|
+
# arithmetic (recording touches no device).
|
|
188
|
+
HighLevel.define(:conv2d, requires: :conv) do |g, inputs, _tensors, _args, opts|
|
|
189
|
+
x, weight = inputs
|
|
190
|
+
n, cin, h, w = x.shape
|
|
191
|
+
cout, _, kh, kw = weight.shape
|
|
192
|
+
ph, pw = opts.fetch(:pad)
|
|
193
|
+
sh, sw = opts.fetch(:stride)
|
|
194
|
+
dh, dw = opts.fetch(:dilation, [1, 1])
|
|
195
|
+
groups = opts.fetch(:groups, 1)
|
|
196
|
+
# dilation acts through the effective kernel width, leaving (d - 1) gaps between taps.
|
|
197
|
+
effective_kh = ((kh - 1) * dh) + 1
|
|
198
|
+
effective_kw = ((kw - 1) * dw) + 1
|
|
199
|
+
hp = h + (2 * ph)
|
|
200
|
+
wp = w + (2 * pw)
|
|
201
|
+
oh = ((hp - effective_kh) / sh) + 1
|
|
202
|
+
ow = ((wp - effective_kw) / sw) + 1
|
|
203
|
+
rows = n * oh * ow
|
|
204
|
+
|
|
205
|
+
padded = g.pad(x, before: [0, 0, ph, pw], after: [0, 0, ph, pw], value: 0.0)
|
|
206
|
+
.reshape(n * cin * hp * wp)
|
|
207
|
+
|
|
208
|
+
modulo = ->(value, by) { value - ((value / by) * by) }
|
|
209
|
+
|
|
210
|
+
# im2col + matmul for one group. channel_offset is the channel of the
|
|
211
|
+
# padded input where that group starts (always 0 when groups == 1).
|
|
212
|
+
im2col_matmul = lambda do |weight_slice, cin_group, cout_group, channel_offset|
|
|
213
|
+
k = cin_group * kh * kw
|
|
214
|
+
r = g.iota([rows, k], axis: 0)
|
|
215
|
+
c = g.iota([rows, k], axis: 1)
|
|
216
|
+
batch = r / (oh * ow)
|
|
217
|
+
out_y = modulo.call(r / ow, oh)
|
|
218
|
+
out_x = modulo.call(r, ow)
|
|
219
|
+
channel = (c / (kh * kw)) + channel_offset
|
|
220
|
+
tap_y = modulo.call(c / kw, kh)
|
|
221
|
+
tap_x = modulo.call(c, kw)
|
|
222
|
+
|
|
223
|
+
index = (batch * (cin * hp * wp)) + (channel * (hp * wp)) +
|
|
224
|
+
(((out_y * sh) + (tap_y * dh)) * wp) + ((out_x * sw) + (tap_x * dw))
|
|
225
|
+
|
|
226
|
+
g.gather(padded, index) # [rows, k]
|
|
227
|
+
.matmul(weight_slice.reshape(cout_group, k).permute(1, 0)) # [rows, cout_group]
|
|
228
|
+
end
|
|
229
|
+
|
|
230
|
+
# Open output row r = (n, oh, ow) and column c = (cin_per_group, kh, kw)
|
|
231
|
+
# into indices. % is not a primitive, so it is written a - (a / b) * b
|
|
232
|
+
# (integer division truncates the same on the dialect side and on :cpu).
|
|
233
|
+
cols = if groups == 1
|
|
234
|
+
im2col_matmul.call(weight, cin, cout, 0)
|
|
235
|
+
else
|
|
236
|
+
# groups > 1: build an independent im2col + matmul per group and
|
|
237
|
+
# concatenate along cout (axis 1, not the outermost). Groups'
|
|
238
|
+
# input channel ranges do not overlap, so this is correct.
|
|
239
|
+
cin_per_group = cin / groups
|
|
240
|
+
cout_per_group = cout / groups
|
|
241
|
+
group_outputs = (0...groups).map do |gi|
|
|
242
|
+
first = gi * cout_per_group
|
|
243
|
+
weight_slice = weight.slice(first..(first + cout_per_group - 1))
|
|
244
|
+
im2col_matmul.call(weight_slice, cin_per_group, cout_per_group, gi * cin_per_group)
|
|
245
|
+
end
|
|
246
|
+
g.concat(group_outputs, axis: 1)
|
|
247
|
+
end
|
|
248
|
+
|
|
249
|
+
cols.permute(1, 0).reshape(cout, n, oh, ow)
|
|
250
|
+
.permute(1, 0, 2, 3)
|
|
251
|
+
end
|
|
252
|
+
|
|
253
|
+
HighLevel.define(:activation) do |g, inputs, _tensors, args, _opts|
|
|
254
|
+
kind = args.first
|
|
255
|
+
expr = ACTIVATION_EXPRS[kind]
|
|
256
|
+
# gelu_exact cannot be decomposed (erf is not a primitive). Failing beats
|
|
257
|
+
# silently substituting another approximation.
|
|
258
|
+
if expr.nil?
|
|
259
|
+
raise Error, "#{kind} has no decomposition (erf is not a primitive), so it cannot run " \
|
|
260
|
+
"with Graph.lower_high_level = true"
|
|
261
|
+
end
|
|
262
|
+
|
|
263
|
+
expr.call(g, inputs[0])
|
|
264
|
+
end
|
|
265
|
+
end
|
|
266
|
+
end
|
|
@@ -0,0 +1,403 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Tilework
|
|
4
|
+
module Graph
|
|
5
|
+
# Kernels the graph layer owns: ops with no eager (Ops) counterpart, such
|
|
6
|
+
# as axis reductions and index remaps.
|
|
7
|
+
#
|
|
8
|
+
# Their output takes the same two forms as everything else (dialect source /
|
|
9
|
+
# :cpu Ruby source), so it rides Kernel's SHA cache as is. No branching on
|
|
10
|
+
# backend names; Ops.compiled routes by capabilities[:dialect].
|
|
11
|
+
#
|
|
12
|
+
# ## Numerics
|
|
13
|
+
#
|
|
14
|
+
# Axis reductions **fold in order from the first element in one thread**.
|
|
15
|
+
# Unlike Ops.reduce's tree, the order is fully fixed, so the same input gives
|
|
16
|
+
# the same value every run. The accumulator is float (ACC) on the dialect
|
|
17
|
+
# side and Ruby double on :cpu, so the two differ by what accumulates before
|
|
18
|
+
# the final rounding — which is why conformance uses a tolerance for axis
|
|
19
|
+
# reductions (bit identity as with fusion is not available).
|
|
20
|
+
module Kernels
|
|
21
|
+
module_function
|
|
22
|
+
|
|
23
|
+
# Axis reduction. Viewed as [outer, axis_len, inner], one thread per
|
|
24
|
+
# output element folds axis_len values. outer / inner / axis_len are
|
|
25
|
+
# launch-time scalars, so the source does not change with the shape (no
|
|
26
|
+
# recompilation).
|
|
27
|
+
#
|
|
28
|
+
# args: [a, out, axis_len, inner, n]
|
|
29
|
+
def axis_reduce(kind:, in_dtype:, out_dtype:)
|
|
30
|
+
acc = Ops::CTYPE.fetch(out_dtype)
|
|
31
|
+
name = "axis_#{kind}_#{in_dtype}_#{out_dtype}"
|
|
32
|
+
src = <<~KERNEL
|
|
33
|
+
GPU_KERNEL void #{name}(GPU_ARGS,
|
|
34
|
+
device const #{Ops::CTYPE.fetch(in_dtype)}* a,
|
|
35
|
+
device #{acc}* c,
|
|
36
|
+
constant int& axis_len,
|
|
37
|
+
constant int& inner,
|
|
38
|
+
constant int& n) {
|
|
39
|
+
int i = GLOBAL_ID_X;
|
|
40
|
+
if (i < n) {
|
|
41
|
+
int base = (i / inner) * axis_len * inner + (i % inner);
|
|
42
|
+
#{acc} acc = (#{acc})a[base];
|
|
43
|
+
for (int t = 1; t < axis_len; ++t) {
|
|
44
|
+
#{acc} v = (#{acc})a[base + t * inner];
|
|
45
|
+
#{combine_expr(kind)}
|
|
46
|
+
}
|
|
47
|
+
c[i] = #{finish_expr(kind, acc)};
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
KERNEL
|
|
51
|
+
|
|
52
|
+
Ops.compiled(dialect_name: name, dialect_src: src, cpu_name: name,
|
|
53
|
+
cpu_src: -> { Ops.codegen.axis_reduce(kind: kind, in_dtype: in_dtype, out_dtype: out_dtype) })
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
# Index reduction along an axis: returns positions, not values. Ties keep
|
|
57
|
+
# the smaller index (left-biased like Ops.arg_reduce).
|
|
58
|
+
#
|
|
59
|
+
# args: [a, out, axis_len, inner, n]
|
|
60
|
+
def axis_arg_reduce(kind:, in_dtype:)
|
|
61
|
+
acc = Ops::CTYPE.fetch(Ops::LOW_PRECISION_DTYPES.include?(in_dtype) ? :float32 : in_dtype)
|
|
62
|
+
cmp = kind == :max ? ">" : "<"
|
|
63
|
+
name = "axis_arg#{kind}_#{in_dtype}"
|
|
64
|
+
src = <<~KERNEL
|
|
65
|
+
GPU_KERNEL void #{name}(GPU_ARGS,
|
|
66
|
+
device const #{Ops::CTYPE.fetch(in_dtype)}* a,
|
|
67
|
+
device int* c,
|
|
68
|
+
constant int& axis_len,
|
|
69
|
+
constant int& inner,
|
|
70
|
+
constant int& n) {
|
|
71
|
+
int i = GLOBAL_ID_X;
|
|
72
|
+
if (i < n) {
|
|
73
|
+
int base = (i / inner) * axis_len * inner + (i % inner);
|
|
74
|
+
#{acc} best = (#{acc})a[base];
|
|
75
|
+
int at = 0;
|
|
76
|
+
for (int t = 1; t < axis_len; ++t) {
|
|
77
|
+
#{acc} v = (#{acc})a[base + t * inner];
|
|
78
|
+
if (v #{cmp} best) { best = v; at = t; }
|
|
79
|
+
}
|
|
80
|
+
c[i] = at;
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
KERNEL
|
|
84
|
+
|
|
85
|
+
Ops.compiled(dialect_name: name, dialect_src: src, cpu_name: name,
|
|
86
|
+
cpu_src: -> { Ops.codegen.axis_arg_reduce(kind: kind, in_dtype: in_dtype) })
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
# Cast. It **only changes the storage type**, with C cast semantics (float
|
|
91
|
+
# -> int truncates toward zero). :cpu's to_i rounds the same way, so both
|
|
92
|
+
# paths give the same values.
|
|
93
|
+
#
|
|
94
|
+
# args: [a, out, n]
|
|
95
|
+
def cast(from:, to:)
|
|
96
|
+
name = "cast_#{from}_#{to}"
|
|
97
|
+
read = Ops.read_dtype(from, "a[i]")
|
|
98
|
+
src = <<~KERNEL
|
|
99
|
+
GPU_KERNEL void #{name}(GPU_ARGS,
|
|
100
|
+
device const #{Ops::CTYPE.fetch(from)}* a,
|
|
101
|
+
device #{Ops::CTYPE.fetch(to)}* c,
|
|
102
|
+
constant int& n) {
|
|
103
|
+
int i = GLOBAL_ID_X;
|
|
104
|
+
if (i < n) c[i] = (#{Ops::CTYPE.fetch(to)})#{read};
|
|
105
|
+
}
|
|
106
|
+
KERNEL
|
|
107
|
+
|
|
108
|
+
Ops.compiled(dialect_name: name, dialect_src: src, cpu_name: name,
|
|
109
|
+
cpu_src: -> { Ops.codegen.cast(from: from, to: to) })
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
# Select by condition. a / b may each be a buffer or a scalar, which the
|
|
113
|
+
# host already knows (graph shapes are static), so the argument layout is
|
|
114
|
+
# baked into the source. The positions are always [cond, a, b, out, n];
|
|
115
|
+
# only the side that became a scalar turns into a constant reference.
|
|
116
|
+
def select(dtype:, a_scalar:, b_scalar:)
|
|
117
|
+
ct = Ops::CTYPE.fetch(dtype)
|
|
118
|
+
name = "select_#{dtype}#{a_scalar ? '_sa' : ''}#{b_scalar ? '_sb' : ''}"
|
|
119
|
+
operand = ->(sym, scalar) { scalar ? sym.to_s : "#{sym}[i]" }
|
|
120
|
+
param = ->(sym, scalar) { scalar ? "constant #{ct}& #{sym}" : "device const #{ct}* #{sym}" }
|
|
121
|
+
src = <<~KERNEL
|
|
122
|
+
GPU_KERNEL void #{name}(GPU_ARGS,
|
|
123
|
+
device const int* cond,
|
|
124
|
+
#{param.call(:a, a_scalar)},
|
|
125
|
+
#{param.call(:b, b_scalar)},
|
|
126
|
+
device #{ct}* c,
|
|
127
|
+
constant int& n) {
|
|
128
|
+
int i = GLOBAL_ID_X;
|
|
129
|
+
if (i < n) c[i] = cond[i] ? (#{ct})#{operand.call(:a, a_scalar)} : (#{ct})#{operand.call(:b, b_scalar)};
|
|
130
|
+
}
|
|
131
|
+
KERNEL
|
|
132
|
+
|
|
133
|
+
Ops.compiled(dialect_name: name, dialect_src: src, cpu_name: name,
|
|
134
|
+
cpu_src: -> { Ops.codegen.select(dtype: dtype, a_scalar: a_scalar, b_scalar: b_scalar) })
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
# Index along an axis: builds masks and positions without bringing
|
|
138
|
+
# constants in from the host.
|
|
139
|
+
#
|
|
140
|
+
# args: [out, n]
|
|
141
|
+
def iota(shape:, axis:, dtype:)
|
|
142
|
+
size = shape[axis]
|
|
143
|
+
inner = shape.drop(axis + 1).inject(1, :*)
|
|
144
|
+
ct = Ops::CTYPE.fetch(dtype)
|
|
145
|
+
name = "iota_#{dtype}"
|
|
146
|
+
src = <<~KERNEL
|
|
147
|
+
GPU_KERNEL void #{name}(GPU_ARGS,
|
|
148
|
+
device #{ct}* c,
|
|
149
|
+
constant int& n) {
|
|
150
|
+
int i = GLOBAL_ID_X;
|
|
151
|
+
if (i < n) c[i] = (#{ct})((i / #{inner}) % #{size});
|
|
152
|
+
}
|
|
153
|
+
KERNEL
|
|
154
|
+
|
|
155
|
+
Ops.compiled(dialect_name: name, dialect_src: src, cpu_name: name,
|
|
156
|
+
cpu_src: -> { Ops.codegen.iota(dtype: dtype, size: size, inner: inner) })
|
|
157
|
+
end
|
|
158
|
+
|
|
159
|
+
# Table lookup. Each element of indices points at a position along
|
|
160
|
+
# source's axis — exactly what an embedding does (pulling rows from a
|
|
161
|
+
# vocabulary table).
|
|
162
|
+
#
|
|
163
|
+
# **Indices are clamped into range.** Reading out of range on a GPU is
|
|
164
|
+
# undefined, so clamping to the ends beats breaking silently (:cpu clamps
|
|
165
|
+
# the same way, so the answer does not depend on the path).
|
|
166
|
+
#
|
|
167
|
+
# args: [a, idx, out, n]
|
|
168
|
+
def gather(dtype:, axis_len:, idx_len:, inner:)
|
|
169
|
+
ct = Ops::CTYPE.fetch(dtype)
|
|
170
|
+
name = "gather_#{dtype}"
|
|
171
|
+
src = <<~KERNEL
|
|
172
|
+
GPU_KERNEL void #{name}(GPU_ARGS,
|
|
173
|
+
device const #{ct}* a,
|
|
174
|
+
device const int* idx,
|
|
175
|
+
device #{ct}* c,
|
|
176
|
+
constant int& n) {
|
|
177
|
+
int i = GLOBAL_ID_X;
|
|
178
|
+
if (i < n) {
|
|
179
|
+
int block = #{idx_len * inner};
|
|
180
|
+
int j = (i % block) / #{inner};
|
|
181
|
+
int at = idx[j];
|
|
182
|
+
at = at < 0 ? 0 : (at > #{axis_len - 1} ? #{axis_len - 1} : at);
|
|
183
|
+
c[i] = a[(i / block) * #{axis_len * inner} + at * #{inner} + (i % #{inner})];
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
KERNEL
|
|
187
|
+
|
|
188
|
+
Ops.compiled(dialect_name: name, dialect_src: src, cpu_name: name,
|
|
189
|
+
cpu_src: lambda {
|
|
190
|
+
Ops.codegen.gather(dtype: dtype, axis_len: axis_len, idx_len: idx_len, inner: inner)
|
|
191
|
+
})
|
|
192
|
+
end
|
|
193
|
+
|
|
194
|
+
# The inverse of table lookup (scatter-add): adds cotangents back to where
|
|
195
|
+
# gather read.
|
|
196
|
+
#
|
|
197
|
+
# dsource[o, j, k] = Σ_i [ids[i] == j] · dy[o, i, k]
|
|
198
|
+
#
|
|
199
|
+
# **No atomics.** One thread per output element scans the index list and
|
|
200
|
+
# adds only the entries addressed to it. Each thread owns its destination,
|
|
201
|
+
# so nothing races, and the fold order is fixed, so **every run gives the
|
|
202
|
+
# same value** (an atomic scatter-add moves the last bits as its order
|
|
203
|
+
# varies). The same reason keeps GPU_ATOMIC_ADD out of the dialect — a
|
|
204
|
+
# macro whose meaning differs across backends violates the compat header
|
|
205
|
+
# policy.
|
|
206
|
+
#
|
|
207
|
+
# The work is index count × output elements, the same as a matmul with a
|
|
208
|
+
# match matrix. The difference is not materializing the [table rows,
|
|
209
|
+
# index count] mask, which matters when the table is large, as with
|
|
210
|
+
# embeddings.
|
|
211
|
+
#
|
|
212
|
+
# args: [idx, dy, out, n]
|
|
213
|
+
def scatter_add(dtype:, axis_len:, idx_len:, inner:)
|
|
214
|
+
ct = Ops::CTYPE.fetch(dtype)
|
|
215
|
+
acc = Ops::CTYPE.fetch(Ops::LOW_PRECISION_DTYPES.include?(dtype) ? :float32 : dtype)
|
|
216
|
+
name = "scatter_add_#{dtype}"
|
|
217
|
+
src = <<~KERNEL
|
|
218
|
+
GPU_KERNEL void #{name}(GPU_ARGS,
|
|
219
|
+
device const int* idx,
|
|
220
|
+
device const #{ct}* dy,
|
|
221
|
+
device #{ct}* c,
|
|
222
|
+
constant int& n) {
|
|
223
|
+
int i = GLOBAL_ID_X;
|
|
224
|
+
if (i < n) {
|
|
225
|
+
int block = #{axis_len * inner};
|
|
226
|
+
int outer = i / block;
|
|
227
|
+
int j = (i % block) / #{inner};
|
|
228
|
+
int lane = i % #{inner};
|
|
229
|
+
#{acc} acc = (#{acc})0;
|
|
230
|
+
for (int t = 0; t < #{idx_len}; ++t) {
|
|
231
|
+
int at = idx[t];
|
|
232
|
+
at = at < 0 ? 0 : (at > #{axis_len - 1} ? #{axis_len - 1} : at);
|
|
233
|
+
if (at == j) acc += (#{acc})dy[outer * #{idx_len * inner} + t * #{inner} + lane];
|
|
234
|
+
}
|
|
235
|
+
c[i] = (#{ct})acc;
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
KERNEL
|
|
239
|
+
|
|
240
|
+
Ops.compiled(dialect_name: name, dialect_src: src, cpu_name: name,
|
|
241
|
+
cpu_src: lambda {
|
|
242
|
+
Ops.codegen.scatter_add(dtype: dtype, axis_len: axis_len,
|
|
243
|
+
idx_len: idx_len, inner: inner)
|
|
244
|
+
})
|
|
245
|
+
end
|
|
246
|
+
|
|
247
|
+
# Fills a border. dims holds, per output dim,
|
|
248
|
+
# [output size, input size, amount added in front, input-side stride].
|
|
249
|
+
#
|
|
250
|
+
# args: [a, out, n]
|
|
251
|
+
def pad(dims:, dtype:, value:)
|
|
252
|
+
ct = Ops::CTYPE.fetch(dtype)
|
|
253
|
+
name = "pad_#{dtype}"
|
|
254
|
+
lines = pad_lines(dims)
|
|
255
|
+
src = <<~KERNEL
|
|
256
|
+
GPU_KERNEL void #{name}(GPU_ARGS,
|
|
257
|
+
device const #{ct}* a,
|
|
258
|
+
device #{ct}* c,
|
|
259
|
+
constant #{ct}& fill,
|
|
260
|
+
constant int& n) {
|
|
261
|
+
int i = GLOBAL_ID_X;
|
|
262
|
+
if (i < n) {
|
|
263
|
+
int rem = i;
|
|
264
|
+
int src = 0;
|
|
265
|
+
int inside = 1;
|
|
266
|
+
int p = 0;
|
|
267
|
+
#{lines.map { |line| " #{line}" }.join("\n")}
|
|
268
|
+
c[i] = inside ? a[src] : fill;
|
|
269
|
+
}
|
|
270
|
+
}
|
|
271
|
+
KERNEL
|
|
272
|
+
|
|
273
|
+
Ops.compiled(dialect_name: name, dialect_src: src, cpu_name: name,
|
|
274
|
+
cpu_src: -> { Ops.codegen.pad(dtype: dtype, dims: dims) })
|
|
275
|
+
end
|
|
276
|
+
|
|
277
|
+
# Builds the index per dim, checking whether the coordinate falls inside the input.
|
|
278
|
+
def pad_lines(dims)
|
|
279
|
+
dims.reverse.map do |out_dim, in_dim, before, stride|
|
|
280
|
+
coord = "p = rem % #{out_dim}; rem /= #{out_dim};"
|
|
281
|
+
next "#{coord} src += p * #{stride};" if before.zero? && in_dim == out_dim
|
|
282
|
+
|
|
283
|
+
"#{coord} p -= #{before}; " \
|
|
284
|
+
"inside = inside && p >= 0 && p < #{in_dim}; " \
|
|
285
|
+
"src += (p < 0 ? 0 : (p >= #{in_dim} ? 0 : p)) * #{stride};"
|
|
286
|
+
end
|
|
287
|
+
end
|
|
288
|
+
|
|
289
|
+
# Writes one concat piece into its place in the output. Along the
|
|
290
|
+
# outermost dim pieces are contiguous and one copy suffices (Plan picks
|
|
291
|
+
# that); along an inner dim they are scattered, so each element's
|
|
292
|
+
# destination is computed.
|
|
293
|
+
#
|
|
294
|
+
# args: [piece, out, n]
|
|
295
|
+
def concat_piece(dtype:, piece_axis:, total_axis:, inner:, offset:)
|
|
296
|
+
ct = Ops::CTYPE.fetch(dtype)
|
|
297
|
+
name = "concat_piece_#{dtype}"
|
|
298
|
+
src = <<~KERNEL
|
|
299
|
+
GPU_KERNEL void #{name}(GPU_ARGS,
|
|
300
|
+
device const #{ct}* a,
|
|
301
|
+
device #{ct}* c,
|
|
302
|
+
constant int& n) {
|
|
303
|
+
int i = GLOBAL_ID_X;
|
|
304
|
+
if (i < n) {
|
|
305
|
+
int block = #{piece_axis * inner};
|
|
306
|
+
c[(i / block) * #{total_axis * inner} +
|
|
307
|
+
(#{offset} + (i % block) / #{inner}) * #{inner} + (i % #{inner})] = a[i];
|
|
308
|
+
}
|
|
309
|
+
}
|
|
310
|
+
KERNEL
|
|
311
|
+
|
|
312
|
+
Ops.compiled(dialect_name: name, dialect_src: src, cpu_name: name,
|
|
313
|
+
cpu_src: lambda {
|
|
314
|
+
Ops.codegen.concat_piece(dtype: dtype, piece_axis: piece_axis,
|
|
315
|
+
total_axis: total_axis, inner: inner, offset: offset)
|
|
316
|
+
})
|
|
317
|
+
end
|
|
318
|
+
|
|
319
|
+
# Index remap: decompose the output index per dim and rebuild the input
|
|
320
|
+
# index. **Broadcast and axis permutation are the same kernel**; only the
|
|
321
|
+
# table of "how far the input advances per output dim (stride)" differs:
|
|
322
|
+
#
|
|
323
|
+
# broadcast a stretched dim has stride 0 (always the same element)
|
|
324
|
+
# permute output dim d takes the stride of input dim axes[d]
|
|
325
|
+
#
|
|
326
|
+
# The table is known on the host (graph shapes are static), so it is
|
|
327
|
+
# **baked into the kernel as constants**; each shape combination compiles once.
|
|
328
|
+
#
|
|
329
|
+
# args: [a, out, n]
|
|
330
|
+
def remap(dims:, dtype:, name: "remap")
|
|
331
|
+
ctype = Ops::CTYPE.fetch(dtype)
|
|
332
|
+
name = "#{name}_#{dtype}"
|
|
333
|
+
src = <<~KERNEL
|
|
334
|
+
GPU_KERNEL void #{name}(GPU_ARGS,
|
|
335
|
+
device const #{ctype}* a,
|
|
336
|
+
device #{ctype}* c,
|
|
337
|
+
constant int& n) {
|
|
338
|
+
int i = GLOBAL_ID_X;
|
|
339
|
+
if (i < n) {
|
|
340
|
+
int rem = i;
|
|
341
|
+
int src = 0;
|
|
342
|
+
#{dims.reverse.map { |dim| " #{index_line(dim)}" }.join("\n")}
|
|
343
|
+
c[i] = a[src];
|
|
344
|
+
}
|
|
345
|
+
}
|
|
346
|
+
KERNEL
|
|
347
|
+
|
|
348
|
+
Ops.compiled(dialect_name: name, dialect_src: src, cpu_name: name,
|
|
349
|
+
cpu_src: -> { Ops.codegen.remap(dtype: dtype, dims: dims) })
|
|
350
|
+
end
|
|
351
|
+
|
|
352
|
+
# The stride table for an index-remapping node (:broadcast / :permute).
|
|
353
|
+
def remap_dims(node)
|
|
354
|
+
source = node.inputs.first
|
|
355
|
+
case node.op
|
|
356
|
+
when :broadcast then broadcast_dims(source.shape, node.shape)
|
|
357
|
+
when :permute then permute_dims(source.shape, node.attrs[:axes])
|
|
358
|
+
else raise Error, "#{node.op.inspect} does not remap indices"
|
|
359
|
+
end
|
|
360
|
+
end
|
|
361
|
+
|
|
362
|
+
# [output size, input-side stride] per output dim. A stretched dim has
|
|
363
|
+
# stride 0 — advancing along it always points at the same input element.
|
|
364
|
+
def broadcast_dims(in_shape, out_shape)
|
|
365
|
+
padded = Array.new(out_shape.length - in_shape.length, 1) + in_shape
|
|
366
|
+
strides = row_major_strides(padded)
|
|
367
|
+
out_shape.each_with_index.map do |dim, d|
|
|
368
|
+
[dim, padded[d] == 1 && dim != 1 ? 0 : strides[d]]
|
|
369
|
+
end
|
|
370
|
+
end
|
|
371
|
+
|
|
372
|
+
# Output dim d corresponds to input dim axes[d], so it takes that stride as is.
|
|
373
|
+
def permute_dims(in_shape, axes)
|
|
374
|
+
strides = row_major_strides(in_shape)
|
|
375
|
+
axes.map { |axis| [in_shape[axis], strides[axis]] }
|
|
376
|
+
end
|
|
377
|
+
|
|
378
|
+
def row_major_strides(shape)
|
|
379
|
+
stride = 1
|
|
380
|
+
shape.reverse.map { |dim| stride.tap { stride *= dim } }.reverse
|
|
381
|
+
end
|
|
382
|
+
|
|
383
|
+
# Index arithmetic for one dim, peeling dims off from the last one.
|
|
384
|
+
def index_line(dim)
|
|
385
|
+
size, stride = dim
|
|
386
|
+
return "rem /= #{size};" if stride.zero?
|
|
387
|
+
|
|
388
|
+
"src += (rem % #{size}) * #{stride}; rem /= #{size};"
|
|
389
|
+
end
|
|
390
|
+
|
|
391
|
+
def combine_expr(kind)
|
|
392
|
+
case kind
|
|
393
|
+
when :sum, :mean then "acc = acc + v;"
|
|
394
|
+
when :min then "acc = v < acc ? v : acc;"
|
|
395
|
+
when :max then "acc = v > acc ? v : acc;"
|
|
396
|
+
else raise Error, "unknown reduction #{kind.inspect}"
|
|
397
|
+
end
|
|
398
|
+
end
|
|
399
|
+
|
|
400
|
+
def finish_expr(kind, acc) = kind == :mean ? "acc / (#{acc})axis_len" : "acc"
|
|
401
|
+
end
|
|
402
|
+
end
|
|
403
|
+
end
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Tilework
|
|
4
|
+
module Graph
|
|
5
|
+
# One graph node. It holds **no value** — only dtype and shape (the
|
|
6
|
+
# metadata part of a Buffer) and how it is produced. Plan#run assigns the
|
|
7
|
+
# actual buffer.
|
|
8
|
+
#
|
|
9
|
+
# A node only knows which op it is. How that op behaves (may it be freed,
|
|
10
|
+
# does it run synchronously, can it fuse, how it prints) lives in the
|
|
11
|
+
# Graph::Op table.
|
|
12
|
+
#
|
|
13
|
+
# order_deps are ordering edges that carry no value, added by
|
|
14
|
+
# Plan#compile. They express the WAR rule for mutable state ("a write runs
|
|
15
|
+
# after the readers").
|
|
16
|
+
Node = Struct.new(:id, :op, :inputs, :attrs, :dtype, :shape) do
|
|
17
|
+
def length = shape.inject(1, :*)
|
|
18
|
+
def state = attrs[:state]
|
|
19
|
+
def region = attrs[:region]
|
|
20
|
+
def spec = Op.fetch(op)
|
|
21
|
+
def writes_state? = spec.writes_state?
|
|
22
|
+
def label = spec.label_for(self)
|
|
23
|
+
|
|
24
|
+
def to_s = "##{id} #{label} #{shape.inspect} #{dtype}"
|
|
25
|
+
end
|
|
26
|
+
end
|
|
27
|
+
end
|