tilework 0.18.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE +21 -0
- data/lib/tilework/backend/cpu/dsl_codegen.rb +509 -0
- data/lib/tilework/backend/cpu/ops_codegen.rb +451 -0
- data/lib/tilework/backend/cpu.rb +156 -0
- data/lib/tilework/buffer.rb +753 -0
- data/lib/tilework/capabilities.rb +183 -0
- data/lib/tilework/compat/cuda.h +66 -0
- data/lib/tilework/compat/hip.h +35 -0
- data/lib/tilework/compat/metal.h +45 -0
- data/lib/tilework/conformance.rb +2296 -0
- data/lib/tilework/core_ext.rb +21 -0
- data/lib/tilework/dialect.rb +67 -0
- data/lib/tilework/dsl.rb +722 -0
- data/lib/tilework/dtype/bfloat.rb +31 -0
- data/lib/tilework/dtype/cast.rb +88 -0
- data/lib/tilework/dtype/half.rb +98 -0
- data/lib/tilework/graph/builder.rb +1073 -0
- data/lib/tilework/graph/fusion.rb +520 -0
- data/lib/tilework/graph/grad.rb +443 -0
- data/lib/tilework/graph/high_level.rb +266 -0
- data/lib/tilework/graph/kernels.rb +403 -0
- data/lib/tilework/graph/node.rb +27 -0
- data/lib/tilework/graph/op.rb +290 -0
- data/lib/tilework/graph/optim.rb +183 -0
- data/lib/tilework/graph/plan.rb +1086 -0
- data/lib/tilework/graph/pool.rb +178 -0
- data/lib/tilework/graph/region.rb +210 -0
- data/lib/tilework/graph/residency.rb +449 -0
- data/lib/tilework/graph/state.rb +64 -0
- data/lib/tilework/graph/tensor.rb +124 -0
- data/lib/tilework/graph.rb +213 -0
- data/lib/tilework/kernel.rb +163 -0
- data/lib/tilework/math/conv.rb +270 -0
- data/lib/tilework/math/fft.rb +172 -0
- data/lib/tilework/math/nn.rb +1103 -0
- data/lib/tilework/math/philox.rb +194 -0
- data/lib/tilework/math/random.rb +232 -0
- data/lib/tilework/math/tensor_core.rb +269 -0
- data/lib/tilework/ops.rb +682 -0
- data/lib/tilework/runtime/offload.rb +498 -0
- data/lib/tilework/runtime/submission.rb +163 -0
- data/lib/tilework/runtime/telemetry.rb +112 -0
- data/lib/tilework/scope.rb +33 -0
- data/lib/tilework/version.rb +5 -0
- data/lib/tilework.rb +345 -0
- metadata +92 -0
|
@@ -0,0 +1,520 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Tilework
|
|
4
|
+
module Graph
|
|
5
|
+
# Elementwise fusion (#17 Phase 2).
|
|
6
|
+
#
|
|
7
|
+
# Folds chained elementwise/unary/scalar nodes into one kernel. A five-step
|
|
8
|
+
# chain's four intermediates and ten memory round trips become one read and
|
|
9
|
+
# one write.
|
|
10
|
+
#
|
|
11
|
+
# ## How far it fuses
|
|
12
|
+
#
|
|
13
|
+
# A node is absorbed into its parent **only if it has exactly one
|
|
14
|
+
# consumer**. A value read from two places needs materializing somewhere
|
|
15
|
+
# anyway, so absorbing it saves no round trip and duplicates the
|
|
16
|
+
# expression, computing it twice. This one rule folds every "a into b, b
|
|
17
|
+
# into c" chain into one and safely declines diamonds (reconvergence).
|
|
18
|
+
#
|
|
19
|
+
# ## Values do not change
|
|
20
|
+
#
|
|
21
|
+
# The unfused path repeats "widen to the compute type → operate → write back
|
|
22
|
+
# in the output type" per node. Wrapping each expression-tree node **in a
|
|
23
|
+
# cast to its output type** reproduces that write-back exactly, so results
|
|
24
|
+
# agree bit for bit (conformance's graph_fused_elementwise_matches_the_unfused_path
|
|
25
|
+
# pins this on every backend).
|
|
26
|
+
#
|
|
27
|
+
# The one exception is float16, where intermediates would need re-rounding
|
|
28
|
+
# to half: skipping it changes values between fused and unfused, and doing
|
|
29
|
+
# it undoes the point of fusing. **Nodes with float16 intermediates are not
|
|
30
|
+
# fused** (leaves and roots may be float16 — they are real buffers and
|
|
31
|
+
# round anyway).
|
|
32
|
+
#
|
|
33
|
+
# ## :cpu fuses too
|
|
34
|
+
#
|
|
35
|
+
# The issue's original plan excluded :cpu (no dialect), which would have
|
|
36
|
+
# made "fused == unfused" checkable only on hardware (CI has no GPU). core
|
|
37
|
+
# builds the expression tree, this file renders dialect source and
|
|
38
|
+
# bridge.ops_codegen.fused renders Ruby source, making **:cpu the oracle
|
|
39
|
+
# for fusion**.
|
|
40
|
+
module Fusion
|
|
41
|
+
# One expression-tree node.
|
|
42
|
+
# kind :buffer (input buffer) / :scalar / :binary / :compare / :math /
|
|
43
|
+
# :abs / :cast / :select
|
|
44
|
+
# dtype this node's output type (= the type it would be written back
|
|
45
|
+
# in without fusion)
|
|
46
|
+
# compute the type used for computation
|
|
47
|
+
# op the operator or function name
|
|
48
|
+
# index the argument position for :buffer / :scalar
|
|
49
|
+
# map the index remap for :buffer (what a folded broadcast leaves
|
|
50
|
+
# behind); nil means plain a[i]
|
|
51
|
+
Expr = Struct.new(:kind, :dtype, :compute, :op, :children, :index, :map, keyword_init: true)
|
|
52
|
+
|
|
53
|
+
# A cluster that becomes one kernel.
|
|
54
|
+
# root the node owning the output (materialized)
|
|
55
|
+
# members node ids absorbed into root (not materialized)
|
|
56
|
+
# inputs nodes of buffers coming from outside (in kernel-argument order)
|
|
57
|
+
# maps index remaps in the same order as inputs (folded broadcasts);
|
|
58
|
+
# nil for inputs read plainly
|
|
59
|
+
# scalars scalars passed as kernel arguments
|
|
60
|
+
# reductions row-wise reductions (in dependency order); empty for an
|
|
61
|
+
# elementwise cluster
|
|
62
|
+
Cluster = Struct.new(:root, :members, :inputs, :maps, :scalars, :expr, :reductions,
|
|
63
|
+
keyword_init: true) do
|
|
64
|
+
def length = root.length
|
|
65
|
+
def shape = root.shape
|
|
66
|
+
def dtype = root.dtype
|
|
67
|
+
|
|
68
|
+
# A cluster containing row-wise reductions launches over **rows**, not elements.
|
|
69
|
+
def row_wise? = !reductions.empty?
|
|
70
|
+
def cols = shape.last
|
|
71
|
+
def rows = length / cols
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
# The lower bound for trying row-wise fusion. One thread owns one row, so
|
|
75
|
+
# the row count is the parallelism — with few rows, per-element
|
|
76
|
+
# parallelism wins even at the cost of more kernels. **This threshold is
|
|
77
|
+
# not measured on hardware**: it is provisionally one block's worth of
|
|
78
|
+
# threads, so fix it once measured (Graph.row_fusion = false turns the
|
|
79
|
+
# path off).
|
|
80
|
+
MIN_ROWS = Ops::BLOCK
|
|
81
|
+
|
|
82
|
+
module_function
|
|
83
|
+
|
|
84
|
+
# Plans fusion. Returns [root.id => Cluster, the set of absorbed node ids].
|
|
85
|
+
#
|
|
86
|
+
# pinned receives nodes named by ordering (WAR) edges. Absorbing one moves
|
|
87
|
+
# "the moment it finished" to root, leaving room to overtake a mutable
|
|
88
|
+
# state write, so they are excluded from fusion.
|
|
89
|
+
def plan(order:, output_ids:, consumers:, pinned:)
|
|
90
|
+
absorbed = {}
|
|
91
|
+
members = Hash.new { |hash, key| hash[key] = [] }
|
|
92
|
+
|
|
93
|
+
order.each do |node|
|
|
94
|
+
next unless fusable?(node)
|
|
95
|
+
|
|
96
|
+
node.inputs.each do |input|
|
|
97
|
+
# Pull a row-wise reduction into the cluster along with its subtree;
|
|
98
|
+
# if it fits, "reduce → stretch → use" becomes one kernel.
|
|
99
|
+
if (found = row_reduce_behind(input, node, output_ids, consumers, pinned))
|
|
100
|
+
absorb_row_reduce(found, input, node, absorbed, members, consumers, output_ids, pinned)
|
|
101
|
+
next
|
|
102
|
+
end
|
|
103
|
+
# A broadcast **computes nothing**, so it can fold however many
|
|
104
|
+
# readers it has (each just reads the unstretched buffer remapped).
|
|
105
|
+
# The single-consumer rule, which exists to avoid duplicated
|
|
106
|
+
# expressions, does not apply.
|
|
107
|
+
if foldable_broadcast?(input, output_ids, consumers, pinned)
|
|
108
|
+
absorbed[input.id] = true
|
|
109
|
+
members[node.id] |= [input.id]
|
|
110
|
+
next
|
|
111
|
+
end
|
|
112
|
+
next unless absorbable?(input, node, output_ids, consumers, pinned)
|
|
113
|
+
next if absorbed[input.id]
|
|
114
|
+
|
|
115
|
+
absorbed[input.id] = true
|
|
116
|
+
members[node.id] = members[node.id] + (members.delete(input.id) || []) + [input.id]
|
|
117
|
+
end
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
clusters = {}
|
|
121
|
+
order.each do |node|
|
|
122
|
+
next if absorbed[node.id] || members[node.id].empty?
|
|
123
|
+
|
|
124
|
+
clusters[node.id] = build_cluster(node, members[node.id], absorbed)
|
|
125
|
+
end
|
|
126
|
+
[clusters, absorbed]
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
# Whether an op can fuse is in the op table (only ops whose launch domain
|
|
130
|
+
# is the element count alone).
|
|
131
|
+
def fusable?(node) = Op.fetch(node.op).fusable?
|
|
132
|
+
|
|
133
|
+
def absorbable?(input, node, output_ids, consumers, pinned)
|
|
134
|
+
return false unless fusable?(input)
|
|
135
|
+
# With half-precision intermediates, values change between fused and unfused unless rounding is reproduced.
|
|
136
|
+
return false if Ops::LOW_PRECISION_DTYPES.include?(input.dtype)
|
|
137
|
+
# A different shape is a different launch domain (a different kernel even with the same element count).
|
|
138
|
+
return false unless input.shape == node.shape
|
|
139
|
+
return false if output_ids.include?(input.id)
|
|
140
|
+
return false if pinned.include?(input.id)
|
|
141
|
+
|
|
142
|
+
# A value read from two or more places stays materialized even if
|
|
143
|
+
# absorbed, and its expression would be duplicated. Counting edges also
|
|
144
|
+
# rules out self-references like a * a.
|
|
145
|
+
consumers[input.id].length == 1
|
|
146
|
+
end
|
|
147
|
+
|
|
148
|
+
# --- Row-wise fusion ---------------------------------------------------
|
|
149
|
+
#
|
|
150
|
+
# Folds the "reduce a row, spread it back over that row" shape (the
|
|
151
|
+
# skeleton of softmax / layernorm / rmsnorm) into one kernel. A reduction
|
|
152
|
+
# always comes back through a broadcast, so that broadcast is the entry point.
|
|
153
|
+
#
|
|
154
|
+
# One thread owns one row and scans it once per reduction. **No shared
|
|
155
|
+
# memory or barriers**, because :cpu's codegen cannot express them — a
|
|
156
|
+
# block-cooperative fold would create a path CI cannot verify (see the note
|
|
157
|
+
# at the top of backend/cpu/ops_codegen.rb). The hand-written NN kernels
|
|
158
|
+
# cooperate within a block because it is faster; this takes the side that
|
|
159
|
+
# works for general expressions.
|
|
160
|
+
def row_reduce_behind(input, consumer, output_ids, consumers, pinned)
|
|
161
|
+
return nil unless Graph.row_fusion
|
|
162
|
+
return nil unless input.op == :broadcast
|
|
163
|
+
return nil if output_ids.include?(input.id) || pinned.include?(input.id)
|
|
164
|
+
# Exactly one reader. This pulls the reduction and its subtree into the
|
|
165
|
+
# cluster, so if two clusters pulled in the same broadcast, the later
|
|
166
|
+
# one would reference an absorbed node as input and pass a nil buffer at
|
|
167
|
+
# run time. With several readers the broadcast folds as an index and the
|
|
168
|
+
# reduction is materialized (cheaper than redoing it per reader).
|
|
169
|
+
return nil unless consumers[input.id].length == 1
|
|
170
|
+
return nil unless consumers[input.id].all? { |reader| fusable?(reader) && reader.shape == input.shape }
|
|
171
|
+
|
|
172
|
+
# An expression touching only the folded value may sit between the
|
|
173
|
+
# reduction and the broadcast (rmsnorm's `sqrt(mean + eps)`). It is
|
|
174
|
+
# constant within a row, so it joins the cluster too. Only expressions
|
|
175
|
+
# combining **with scalars only** qualify — one reading a buffer would
|
|
176
|
+
# need the row index.
|
|
177
|
+
chain = []
|
|
178
|
+
node = input.inputs.first
|
|
179
|
+
while node.op != :reduce
|
|
180
|
+
return nil unless post_reduce_link?(node, input.inputs.first.shape, consumers, output_ids, pinned)
|
|
181
|
+
|
|
182
|
+
chain << node
|
|
183
|
+
node = node.inputs.first
|
|
184
|
+
end
|
|
185
|
+
return nil unless row_reduce?(node, consumer.shape)
|
|
186
|
+
return nil if output_ids.include?(node.id) || pinned.include?(node.id)
|
|
187
|
+
return nil unless consumers[node.id].length == 1
|
|
188
|
+
|
|
189
|
+
[node, chain]
|
|
190
|
+
end
|
|
191
|
+
|
|
192
|
+
# Nodes allowed in the cluster as expressions over the folded value.
|
|
193
|
+
# Having exactly one input (unary, cast, binary with a scalar) is what
|
|
194
|
+
# guarantees they read no buffer.
|
|
195
|
+
def post_reduce_link?(node, shape, consumers, output_ids, pinned)
|
|
196
|
+
return false unless %i[unary cast binary].include?(node.op)
|
|
197
|
+
return false if node.op == :binary && !node.attrs.key?(:scalar)
|
|
198
|
+
return false unless node.inputs.length == 1 && node.shape == shape
|
|
199
|
+
return false if Ops::LOW_PRECISION_DTYPES.include?(node.dtype)
|
|
200
|
+
return false if output_ids.include?(node.id) || pinned.include?(node.id)
|
|
201
|
+
|
|
202
|
+
consumers[node.id].length == 1
|
|
203
|
+
end
|
|
204
|
+
|
|
205
|
+
# A keepdims reduction over the last dim whose pre-reduction shape matches
|
|
206
|
+
# the cluster's. Not folded while rows are few (see MIN_ROWS above).
|
|
207
|
+
def row_reduce?(node, cluster_shape)
|
|
208
|
+
return false unless node.op == :reduce
|
|
209
|
+
return false if Ops::LOW_PRECISION_DTYPES.include?(node.dtype)
|
|
210
|
+
return false unless node.attrs[:keepdims]
|
|
211
|
+
|
|
212
|
+
source = node.inputs.first
|
|
213
|
+
axis = node.attrs[:axis]
|
|
214
|
+
return false if axis.nil? || axis != source.shape.length - 1
|
|
215
|
+
return false unless source.shape == cluster_shape
|
|
216
|
+
|
|
217
|
+
cluster_shape[0..-2].inject(1, :*) >= MIN_ROWS
|
|
218
|
+
end
|
|
219
|
+
|
|
220
|
+
# Puts the reduction and the elementwise ops before it into the cluster.
|
|
221
|
+
# When what comes before cannot be absorbed (read from two places, etc.)
|
|
222
|
+
# it stops there and the reduction reads that materialized value.
|
|
223
|
+
def absorb_row_reduce(found, broadcast, root, absorbed, members, consumers, output_ids, pinned)
|
|
224
|
+
reduce, chain = found
|
|
225
|
+
[broadcast, reduce, *chain].each do |node|
|
|
226
|
+
absorbed[node.id] = true
|
|
227
|
+
members[root.id] |= [node.id]
|
|
228
|
+
end
|
|
229
|
+
|
|
230
|
+
stack = reduce.inputs.dup
|
|
231
|
+
while (node = stack.pop)
|
|
232
|
+
next if absorbed[node.id]
|
|
233
|
+
next unless fusable?(node) && node.shape == reduce.inputs.first.shape
|
|
234
|
+
next if Ops::LOW_PRECISION_DTYPES.include?(node.dtype)
|
|
235
|
+
next if output_ids.include?(node.id) || pinned.include?(node.id)
|
|
236
|
+
next unless consumers[node.id].length == 1
|
|
237
|
+
|
|
238
|
+
absorbed[node.id] = true
|
|
239
|
+
members[root.id] |= (members.delete(node.id) || []) + [node.id]
|
|
240
|
+
stack.concat(node.inputs)
|
|
241
|
+
end
|
|
242
|
+
end
|
|
243
|
+
|
|
244
|
+
# If every reader folds, the broadcast need not be materialized. If even
|
|
245
|
+
# one needs the real buffer (a reduction, matmul, :nn, or the output
|
|
246
|
+
# itself), it must be materialized for that reader, so it is not folded
|
|
247
|
+
# — folding and materializing both would hold the same value two ways,
|
|
248
|
+
# adding lifetime bookkeeping for no gain.
|
|
249
|
+
def foldable_broadcast?(input, output_ids, consumers, pinned)
|
|
250
|
+
return false unless Op.fetch(input.op).index_map?
|
|
251
|
+
return false if Ops::LOW_PRECISION_DTYPES.include?(input.dtype)
|
|
252
|
+
return false if output_ids.include?(input.id) || pinned.include?(input.id)
|
|
253
|
+
|
|
254
|
+
readers = consumers[input.id]
|
|
255
|
+
readers.length > 1 && readers.all? { |reader| fusable?(reader) && reader.shape == input.shape }
|
|
256
|
+
end
|
|
257
|
+
|
|
258
|
+
def build_cluster(root, member_ids, absorbed)
|
|
259
|
+
inside = (member_ids + [root.id]).to_h { |id| [id, true] }
|
|
260
|
+
inputs = []
|
|
261
|
+
maps = []
|
|
262
|
+
scalars = []
|
|
263
|
+
reductions = []
|
|
264
|
+
expr = build_expr(root, inside, [inputs, maps, reductions], scalars, absorbed)
|
|
265
|
+
Cluster.new(root: root, members: member_ids, inputs: inputs, maps: maps,
|
|
266
|
+
scalars: scalars, expr: expr, reductions: reductions)
|
|
267
|
+
end
|
|
268
|
+
|
|
269
|
+
# Walks the tree from the root into an expression; values from outside
|
|
270
|
+
# the cluster become buffer arguments.
|
|
271
|
+
def build_expr(node, inside, args, scalars, absorbed)
|
|
272
|
+
return leaf(node, args) unless inside[node.id]
|
|
273
|
+
# A broadcast becomes **an index, not an expression**: the unstretched
|
|
274
|
+
# buffer is the argument and the way it is read (which dims repeat how
|
|
275
|
+
# often) folds into the index expression. That removes the kernel
|
|
276
|
+
# writing out the stretched value and the round trip reading it back.
|
|
277
|
+
return remap_leaf(node, inside, args, scalars, absorbed) if Op.fetch(node.op).index_map?
|
|
278
|
+
return reduce_expr(node, inside, args, scalars, absorbed) if node.op == :reduce
|
|
279
|
+
|
|
280
|
+
children = node.inputs.map { |input| build_expr(input, inside, args, scalars, absorbed) }
|
|
281
|
+
case node.op
|
|
282
|
+
when :unary then unary_expr(node, children.first)
|
|
283
|
+
when :binary then binary_expr(node, children, scalars)
|
|
284
|
+
when :cast then cast_expr(node, children.first)
|
|
285
|
+
when :where then select_expr(node, children, scalars)
|
|
286
|
+
else raise Error, "cannot fuse #{node.op.inspect}"
|
|
287
|
+
end
|
|
288
|
+
end
|
|
289
|
+
|
|
290
|
+
# The same node read with a different map is a separate argument. Plain
|
|
291
|
+
# and stretched reads can share one expression, so identity is [node, map].
|
|
292
|
+
def leaf(node, args, map = nil)
|
|
293
|
+
inputs, maps, = args
|
|
294
|
+
index = inputs.each_index.find { |i| inputs[i].equal?(node) && maps[i] == map }
|
|
295
|
+
unless index
|
|
296
|
+
index = inputs.length
|
|
297
|
+
inputs << node
|
|
298
|
+
maps << map
|
|
299
|
+
end
|
|
300
|
+
Expr.new(kind: :buffer, dtype: node.dtype, compute: node.dtype, index: index, map: map, children: [])
|
|
301
|
+
end
|
|
302
|
+
|
|
303
|
+
# When what is behind the broadcast is inside the cluster (= a row-wise
|
|
304
|
+
# reduction was pulled in), it becomes **that row's reduced value**, not an
|
|
305
|
+
# index remap. It is constant within the row, so no index is needed.
|
|
306
|
+
def remap_leaf(node, inside, args, scalars, absorbed)
|
|
307
|
+
source = node.inputs.first
|
|
308
|
+
return build_expr(source, inside, args, scalars, absorbed) if inside[source.id]
|
|
309
|
+
|
|
310
|
+
leaf(source, args, Kernels.remap_dims(node))
|
|
311
|
+
end
|
|
312
|
+
|
|
313
|
+
# A row-wise reduction. The expression is the pre-reduction value, and the
|
|
314
|
+
# result is computed once per row under the name red<k>.
|
|
315
|
+
def reduce_expr(node, inside, args, scalars, absorbed)
|
|
316
|
+
reductions = args[2]
|
|
317
|
+
expr = build_expr(node.inputs.first, inside, args, scalars, absorbed)
|
|
318
|
+
index = reductions.length
|
|
319
|
+
reductions << { kind: node.attrs[:kind], dtype: node.dtype, expr: expr }
|
|
320
|
+
Expr.new(kind: :reduced, dtype: node.dtype, compute: node.dtype, index: index, children: [])
|
|
321
|
+
end
|
|
322
|
+
|
|
323
|
+
def unary_expr(node, child)
|
|
324
|
+
fn = node.attrs[:fn]
|
|
325
|
+
kind = fn == :abs ? :abs : :math
|
|
326
|
+
Expr.new(kind: kind, dtype: node.dtype, compute: node.dtype == :int32 ? :int32 : :float32,
|
|
327
|
+
op: fn, children: [child])
|
|
328
|
+
end
|
|
329
|
+
|
|
330
|
+
def binary_expr(node, children, scalars)
|
|
331
|
+
kind = node.attrs[:kind] == :compare ? :compare : :binary
|
|
332
|
+
compute = compute_dtype(node)
|
|
333
|
+
operands = children
|
|
334
|
+
if node.attrs.key?(:scalar)
|
|
335
|
+
index = scalars.length
|
|
336
|
+
value = compute == :float32 ? node.attrs[:scalar].to_f : node.attrs[:scalar]
|
|
337
|
+
scalars << { value: value, dtype: compute }
|
|
338
|
+
scalar = Expr.new(kind: :scalar, dtype: compute, compute: compute, index: index, children: [])
|
|
339
|
+
operands = node.attrs[:reversed] ? [scalar, children.first] : [children.first, scalar]
|
|
340
|
+
end
|
|
341
|
+
Expr.new(kind: kind, dtype: node.dtype, compute: compute, op: node.attrs[:op], children: operands)
|
|
342
|
+
end
|
|
343
|
+
|
|
344
|
+
# A cast is exactly "a cast to the output type", the same as the unfused
|
|
345
|
+
# path's write-back, so in the tree it wraps like any other node.
|
|
346
|
+
def cast_expr(node, child)
|
|
347
|
+
Expr.new(kind: :cast, dtype: node.dtype, compute: node.dtype, children: [child])
|
|
348
|
+
end
|
|
349
|
+
|
|
350
|
+
# Select by condition. Children are [cond, a, b]; a side that is a scalar
|
|
351
|
+
# goes in as a kernel scalar argument (like a binary op's scalar).
|
|
352
|
+
def select_expr(node, children, scalars)
|
|
353
|
+
rest = children.drop(1)
|
|
354
|
+
operands = %i[a b].map do |key|
|
|
355
|
+
value = node.attrs[:"#{key}_scalar"]
|
|
356
|
+
next rest.shift if value.nil?
|
|
357
|
+
|
|
358
|
+
index = scalars.length
|
|
359
|
+
scalars << { value: value, dtype: node.dtype }
|
|
360
|
+
Expr.new(kind: :scalar, dtype: node.dtype, compute: node.dtype, index: index, children: [])
|
|
361
|
+
end
|
|
362
|
+
Expr.new(kind: :select, dtype: node.dtype, compute: node.dtype,
|
|
363
|
+
children: [children.first, *operands])
|
|
364
|
+
end
|
|
365
|
+
|
|
366
|
+
# Ops "widens to the compute type, then operates". A comparison outputs
|
|
367
|
+
# int32 but computes in its operands' type, so the output type cannot
|
|
368
|
+
# decide it.
|
|
369
|
+
def compute_dtype(node)
|
|
370
|
+
return node.dtype unless node.attrs[:kind] == :compare
|
|
371
|
+
|
|
372
|
+
operands = node.inputs.map(&:dtype)
|
|
373
|
+
operands << (node.attrs[:scalar].is_a?(Integer) ? :int32 : :float32) if node.attrs.key?(:scalar)
|
|
374
|
+
operands.all?(:int32) ? :int32 : :float32
|
|
375
|
+
end
|
|
376
|
+
|
|
377
|
+
# --- Dialect source ----------------------------------------------------
|
|
378
|
+
|
|
379
|
+
# A cluster with row-wise reductions. One thread owns one row, scanning it
|
|
380
|
+
# once per reduction and then once more to write. **Intermediates are
|
|
381
|
+
# recomputed rather than kept** — as in the hand-written NN kernels
|
|
382
|
+
# (softmax computes exp twice); not holding a row's worth of storage is
|
|
383
|
+
# this form's advantage.
|
|
384
|
+
def row_dialect_source(cluster)
|
|
385
|
+
params = kernel_params(cluster)
|
|
386
|
+
params << "constant int& rows"
|
|
387
|
+
params << "constant int& cols"
|
|
388
|
+
|
|
389
|
+
passes = cluster.reductions.each_with_index.map do |reduction, k|
|
|
390
|
+
acc = Ops::CTYPE.fetch(reduction[:dtype])
|
|
391
|
+
<<~PASS.chomp
|
|
392
|
+
#{acc} acc#{k} = (#{acc})0;
|
|
393
|
+
for (int c = 0; c < cols; ++c) {
|
|
394
|
+
int i = base + c;
|
|
395
|
+
#{acc} v = (#{acc})#{dialect_expr(reduction[:expr])};
|
|
396
|
+
acc#{k} = (c == 0) ? v : (#{combine(reduction[:kind], "acc#{k}", 'v')});
|
|
397
|
+
}
|
|
398
|
+
#{acc} red#{k} = #{finish(reduction[:kind], "acc#{k}", acc)};
|
|
399
|
+
PASS
|
|
400
|
+
end
|
|
401
|
+
|
|
402
|
+
<<~KERNEL
|
|
403
|
+
GPU_KERNEL void row_fused(GPU_ARGS,
|
|
404
|
+
#{params.join(",\n ")}) {
|
|
405
|
+
int r = GLOBAL_ID_X;
|
|
406
|
+
if (r < rows) {
|
|
407
|
+
int base = r * cols;
|
|
408
|
+
#{passes.join("\n")}
|
|
409
|
+
for (int c = 0; c < cols; ++c) {
|
|
410
|
+
int i = base + c;
|
|
411
|
+
out[i] = #{dialect_expr(cluster.expr)};
|
|
412
|
+
}
|
|
413
|
+
}
|
|
414
|
+
}
|
|
415
|
+
KERNEL
|
|
416
|
+
end
|
|
417
|
+
|
|
418
|
+
# How a reduction folds. The order is "from the first element", the same
|
|
419
|
+
# as the axis-reduction kernel (Kernels.axis_reduce), so fused and unfused
|
|
420
|
+
# give the same value.
|
|
421
|
+
def combine(kind, acc, value)
|
|
422
|
+
case kind
|
|
423
|
+
when :sum, :mean then "#{acc} + #{value}"
|
|
424
|
+
when :min then "#{value} < #{acc} ? #{value} : #{acc}"
|
|
425
|
+
when :max then "#{value} > #{acc} ? #{value} : #{acc}"
|
|
426
|
+
else raise Error, "cannot fuse the reduction #{kind.inspect}"
|
|
427
|
+
end
|
|
428
|
+
end
|
|
429
|
+
|
|
430
|
+
def finish(kind, acc, ctype) = kind == :mean ? "#{acc} / (#{ctype})cols" : acc
|
|
431
|
+
|
|
432
|
+
def kernel_params(cluster)
|
|
433
|
+
params = cluster.inputs.each_with_index.map do |node, i|
|
|
434
|
+
"device const #{Ops::CTYPE.fetch(node.dtype)}* a#{i}"
|
|
435
|
+
end
|
|
436
|
+
params += cluster.scalars.each_with_index.map do |scalar, i|
|
|
437
|
+
"constant #{Ops::CTYPE.fetch(scalar[:dtype])}& s#{i}"
|
|
438
|
+
end
|
|
439
|
+
params << "device #{Ops::CTYPE.fetch(cluster.dtype)}* out"
|
|
440
|
+
params
|
|
441
|
+
end
|
|
442
|
+
|
|
443
|
+
# Dialect source for a fused kernel. The name follows from the expression's
|
|
444
|
+
# shape, so clusters of the same shape collapse to one kernel in Kernel's
|
|
445
|
+
# SHA cache (32 layers with the same expression compile once).
|
|
446
|
+
def dialect_source(cluster)
|
|
447
|
+
return row_dialect_source(cluster) if cluster.row_wise?
|
|
448
|
+
|
|
449
|
+
params = kernel_params(cluster)
|
|
450
|
+
params << "constant int& n"
|
|
451
|
+
|
|
452
|
+
<<~KERNEL
|
|
453
|
+
GPU_KERNEL void fused(GPU_ARGS,
|
|
454
|
+
#{params.join(",\n ")}) {
|
|
455
|
+
int i = GLOBAL_ID_X;
|
|
456
|
+
if (i < n) out[i] = #{dialect_expr(cluster.expr)};
|
|
457
|
+
}
|
|
458
|
+
KERNEL
|
|
459
|
+
end
|
|
460
|
+
|
|
461
|
+
# Wraps each node **in a cast to its output type**, reproducing inside the
|
|
462
|
+
# expression the write-back the unfused path performed.
|
|
463
|
+
def dialect_expr(expr)
|
|
464
|
+
ct = Ops::CTYPE.fetch(expr.dtype)
|
|
465
|
+
case expr.kind
|
|
466
|
+
when :buffer then "(#{ct})a#{expr.index}[#{index_expr(expr.map)}]"
|
|
467
|
+
when :reduced then "red#{expr.index}"
|
|
468
|
+
when :scalar then "s#{expr.index}"
|
|
469
|
+
when :binary then "(#{ct})(#{operand(expr, 0)} #{expr.op} #{operand(expr, 1)})"
|
|
470
|
+
when :compare then "(#{ct})((#{operand(expr, 0)} #{expr.op} #{operand(expr, 1)}) ? 1 : 0)"
|
|
471
|
+
when :math then "(#{ct})#{expr.op}((float)#{dialect_expr(expr.children.first)})"
|
|
472
|
+
when :cast then "(#{ct})#{dialect_expr(expr.children.first)}"
|
|
473
|
+
when :select
|
|
474
|
+
"(#{ct})(#{dialect_expr(expr.children[0])} ? #{operand(expr, 1)} : #{operand(expr, 2)})"
|
|
475
|
+
when :abs
|
|
476
|
+
fn = expr.dtype == :int32 ? "abs" : "fabs"
|
|
477
|
+
"(#{ct})#{fn}(#{dialect_expr(expr.children.first)})"
|
|
478
|
+
else raise Error, "unknown fused expression #{expr.kind.inspect}"
|
|
479
|
+
end
|
|
480
|
+
end
|
|
481
|
+
|
|
482
|
+
# Index remap. Splits the output index i per dim and rebuilds the input
|
|
483
|
+
# index, dropping stretched dims (stride 0). **It is a single
|
|
484
|
+
# expression**, so it fits inside a kernel body that cannot hold statements.
|
|
485
|
+
#
|
|
486
|
+
# [d] -> [n, d] -> i % d
|
|
487
|
+
# [n, 1] -> [n, d] -> (i / d) % n
|
|
488
|
+
def index_expr(map)
|
|
489
|
+
return "i" if map.nil?
|
|
490
|
+
|
|
491
|
+
total = map.inject(1) { |acc, (size, _)| acc * size }
|
|
492
|
+
suffix = 1
|
|
493
|
+
terms = map.reverse.filter_map do |size, stride|
|
|
494
|
+
span = suffix
|
|
495
|
+
suffix *= size
|
|
496
|
+
# A stretched dim does not affect the input index, so its term drops.
|
|
497
|
+
next if stride.zero?
|
|
498
|
+
|
|
499
|
+
base = span == 1 ? "i" : "(i / #{span})"
|
|
500
|
+
# The outermost dim is already in range after the division, so no remainder is needed.
|
|
501
|
+
term = suffix == total ? base : "(#{base} % #{size})"
|
|
502
|
+
stride == 1 ? term : "#{term} * #{stride}"
|
|
503
|
+
end
|
|
504
|
+
terms.empty? ? "0" : terms.join(" + ")
|
|
505
|
+
end
|
|
506
|
+
|
|
507
|
+
# Operands are widened to the parent's compute type first (as in Ops.binary).
|
|
508
|
+
def operand(expr, index)
|
|
509
|
+
child = expr.children[index]
|
|
510
|
+
rendered = dialect_expr(child)
|
|
511
|
+
return rendered if child.dtype == expr.compute
|
|
512
|
+
|
|
513
|
+
"(#{Ops::CTYPE.fetch(expr.compute)})#{rendered}"
|
|
514
|
+
end
|
|
515
|
+
|
|
516
|
+
# No need for a different name per cluster (the cache key is the source's SHA).
|
|
517
|
+
def kernel_name(_cluster) = "fused"
|
|
518
|
+
end
|
|
519
|
+
end
|
|
520
|
+
end
|