tilework 0.18.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE +21 -0
  3. data/lib/tilework/backend/cpu/dsl_codegen.rb +509 -0
  4. data/lib/tilework/backend/cpu/ops_codegen.rb +451 -0
  5. data/lib/tilework/backend/cpu.rb +156 -0
  6. data/lib/tilework/buffer.rb +753 -0
  7. data/lib/tilework/capabilities.rb +183 -0
  8. data/lib/tilework/compat/cuda.h +66 -0
  9. data/lib/tilework/compat/hip.h +35 -0
  10. data/lib/tilework/compat/metal.h +45 -0
  11. data/lib/tilework/conformance.rb +2296 -0
  12. data/lib/tilework/core_ext.rb +21 -0
  13. data/lib/tilework/dialect.rb +67 -0
  14. data/lib/tilework/dsl.rb +722 -0
  15. data/lib/tilework/dtype/bfloat.rb +31 -0
  16. data/lib/tilework/dtype/cast.rb +88 -0
  17. data/lib/tilework/dtype/half.rb +98 -0
  18. data/lib/tilework/graph/builder.rb +1073 -0
  19. data/lib/tilework/graph/fusion.rb +520 -0
  20. data/lib/tilework/graph/grad.rb +443 -0
  21. data/lib/tilework/graph/high_level.rb +266 -0
  22. data/lib/tilework/graph/kernels.rb +403 -0
  23. data/lib/tilework/graph/node.rb +27 -0
  24. data/lib/tilework/graph/op.rb +290 -0
  25. data/lib/tilework/graph/optim.rb +183 -0
  26. data/lib/tilework/graph/plan.rb +1086 -0
  27. data/lib/tilework/graph/pool.rb +178 -0
  28. data/lib/tilework/graph/region.rb +210 -0
  29. data/lib/tilework/graph/residency.rb +449 -0
  30. data/lib/tilework/graph/state.rb +64 -0
  31. data/lib/tilework/graph/tensor.rb +124 -0
  32. data/lib/tilework/graph.rb +213 -0
  33. data/lib/tilework/kernel.rb +163 -0
  34. data/lib/tilework/math/conv.rb +270 -0
  35. data/lib/tilework/math/fft.rb +172 -0
  36. data/lib/tilework/math/nn.rb +1103 -0
  37. data/lib/tilework/math/philox.rb +194 -0
  38. data/lib/tilework/math/random.rb +232 -0
  39. data/lib/tilework/math/tensor_core.rb +269 -0
  40. data/lib/tilework/ops.rb +682 -0
  41. data/lib/tilework/runtime/offload.rb +498 -0
  42. data/lib/tilework/runtime/submission.rb +163 -0
  43. data/lib/tilework/runtime/telemetry.rb +112 -0
  44. data/lib/tilework/scope.rb +33 -0
  45. data/lib/tilework/version.rb +5 -0
  46. data/lib/tilework.rb +345 -0
  47. metadata +92 -0
@@ -0,0 +1,520 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Tilework
4
+ module Graph
5
+ # Elementwise fusion (#17 Phase 2).
6
+ #
7
+ # Folds chained elementwise/unary/scalar nodes into one kernel. A five-step
8
+ # chain's four intermediates and ten memory round trips become one read and
9
+ # one write.
10
+ #
11
+ # ## How far it fuses
12
+ #
13
+ # A node is absorbed into its parent **only if it has exactly one
14
+ # consumer**. A value read from two places needs materializing somewhere
15
+ # anyway, so absorbing it saves no round trip and duplicates the
16
+ # expression, computing it twice. This one rule folds every "a into b, b
17
+ # into c" chain into one and safely declines diamonds (reconvergence).
18
+ #
19
+ # ## Values do not change
20
+ #
21
+ # The unfused path repeats "widen to the compute type → operate → write back
22
+ # in the output type" per node. Wrapping each expression-tree node **in a
23
+ # cast to its output type** reproduces that write-back exactly, so results
24
+ # agree bit for bit (conformance's graph_fused_elementwise_matches_the_unfused_path
25
+ # pins this on every backend).
26
+ #
27
+ # The one exception is float16, where intermediates would need re-rounding
28
+ # to half: skipping it changes values between fused and unfused, and doing
29
+ # it undoes the point of fusing. **Nodes with float16 intermediates are not
30
+ # fused** (leaves and roots may be float16 — they are real buffers and
31
+ # round anyway).
32
+ #
33
+ # ## :cpu fuses too
34
+ #
35
+ # The issue's original plan excluded :cpu (no dialect), which would have
36
+ # made "fused == unfused" checkable only on hardware (CI has no GPU). core
37
+ # builds the expression tree, this file renders dialect source and
38
+ # bridge.ops_codegen.fused renders Ruby source, making **:cpu the oracle
39
+ # for fusion**.
40
+ module Fusion
41
+ # One expression-tree node.
42
+ # kind :buffer (input buffer) / :scalar / :binary / :compare / :math /
43
+ # :abs / :cast / :select
44
+ # dtype this node's output type (= the type it would be written back
45
+ # in without fusion)
46
+ # compute the type used for computation
47
+ # op the operator or function name
48
+ # index the argument position for :buffer / :scalar
49
+ # map the index remap for :buffer (what a folded broadcast leaves
50
+ # behind); nil means plain a[i]
51
+ Expr = Struct.new(:kind, :dtype, :compute, :op, :children, :index, :map, keyword_init: true)
52
+
53
+ # A cluster that becomes one kernel.
54
+ # root the node owning the output (materialized)
55
+ # members node ids absorbed into root (not materialized)
56
+ # inputs nodes of buffers coming from outside (in kernel-argument order)
57
+ # maps index remaps in the same order as inputs (folded broadcasts);
58
+ # nil for inputs read plainly
59
+ # scalars scalars passed as kernel arguments
60
+ # reductions row-wise reductions (in dependency order); empty for an
61
+ # elementwise cluster
62
+ Cluster = Struct.new(:root, :members, :inputs, :maps, :scalars, :expr, :reductions,
63
+ keyword_init: true) do
64
+ def length = root.length
65
+ def shape = root.shape
66
+ def dtype = root.dtype
67
+
68
+ # A cluster containing row-wise reductions launches over **rows**, not elements.
69
+ def row_wise? = !reductions.empty?
70
+ def cols = shape.last
71
+ def rows = length / cols
72
+ end
73
+
74
+ # The lower bound for trying row-wise fusion. One thread owns one row, so
75
+ # the row count is the parallelism — with few rows, per-element
76
+ # parallelism wins even at the cost of more kernels. **This threshold is
77
+ # not measured on hardware**: it is provisionally one block's worth of
78
+ # threads, so fix it once measured (Graph.row_fusion = false turns the
79
+ # path off).
80
+ MIN_ROWS = Ops::BLOCK
81
+
82
+ module_function
83
+
84
+ # Plans fusion. Returns [root.id => Cluster, the set of absorbed node ids].
85
+ #
86
+ # pinned receives nodes named by ordering (WAR) edges. Absorbing one moves
87
+ # "the moment it finished" to root, leaving room to overtake a mutable
88
+ # state write, so they are excluded from fusion.
89
+ def plan(order:, output_ids:, consumers:, pinned:)
90
+ absorbed = {}
91
+ members = Hash.new { |hash, key| hash[key] = [] }
92
+
93
+ order.each do |node|
94
+ next unless fusable?(node)
95
+
96
+ node.inputs.each do |input|
97
+ # Pull a row-wise reduction into the cluster along with its subtree;
98
+ # if it fits, "reduce → stretch → use" becomes one kernel.
99
+ if (found = row_reduce_behind(input, node, output_ids, consumers, pinned))
100
+ absorb_row_reduce(found, input, node, absorbed, members, consumers, output_ids, pinned)
101
+ next
102
+ end
103
+ # A broadcast **computes nothing**, so it can fold however many
104
+ # readers it has (each just reads the unstretched buffer remapped).
105
+ # The single-consumer rule, which exists to avoid duplicated
106
+ # expressions, does not apply.
107
+ if foldable_broadcast?(input, output_ids, consumers, pinned)
108
+ absorbed[input.id] = true
109
+ members[node.id] |= [input.id]
110
+ next
111
+ end
112
+ next unless absorbable?(input, node, output_ids, consumers, pinned)
113
+ next if absorbed[input.id]
114
+
115
+ absorbed[input.id] = true
116
+ members[node.id] = members[node.id] + (members.delete(input.id) || []) + [input.id]
117
+ end
118
+ end
119
+
120
+ clusters = {}
121
+ order.each do |node|
122
+ next if absorbed[node.id] || members[node.id].empty?
123
+
124
+ clusters[node.id] = build_cluster(node, members[node.id], absorbed)
125
+ end
126
+ [clusters, absorbed]
127
+ end
128
+
129
+ # Whether an op can fuse is in the op table (only ops whose launch domain
130
+ # is the element count alone).
131
+ def fusable?(node) = Op.fetch(node.op).fusable?
132
+
133
+ def absorbable?(input, node, output_ids, consumers, pinned)
134
+ return false unless fusable?(input)
135
+ # With half-precision intermediates, values change between fused and unfused unless rounding is reproduced.
136
+ return false if Ops::LOW_PRECISION_DTYPES.include?(input.dtype)
137
+ # A different shape is a different launch domain (a different kernel even with the same element count).
138
+ return false unless input.shape == node.shape
139
+ return false if output_ids.include?(input.id)
140
+ return false if pinned.include?(input.id)
141
+
142
+ # A value read from two or more places stays materialized even if
143
+ # absorbed, and its expression would be duplicated. Counting edges also
144
+ # rules out self-references like a * a.
145
+ consumers[input.id].length == 1
146
+ end
147
+
148
+ # --- Row-wise fusion ---------------------------------------------------
149
+ #
150
+ # Folds the "reduce a row, spread it back over that row" shape (the
151
+ # skeleton of softmax / layernorm / rmsnorm) into one kernel. A reduction
152
+ # always comes back through a broadcast, so that broadcast is the entry point.
153
+ #
154
+ # One thread owns one row and scans it once per reduction. **No shared
155
+ # memory or barriers**, because :cpu's codegen cannot express them — a
156
+ # block-cooperative fold would create a path CI cannot verify (see the note
157
+ # at the top of backend/cpu/ops_codegen.rb). The hand-written NN kernels
158
+ # cooperate within a block because it is faster; this takes the side that
159
+ # works for general expressions.
160
+ def row_reduce_behind(input, consumer, output_ids, consumers, pinned)
161
+ return nil unless Graph.row_fusion
162
+ return nil unless input.op == :broadcast
163
+ return nil if output_ids.include?(input.id) || pinned.include?(input.id)
164
+ # Exactly one reader. This pulls the reduction and its subtree into the
165
+ # cluster, so if two clusters pulled in the same broadcast, the later
166
+ # one would reference an absorbed node as input and pass a nil buffer at
167
+ # run time. With several readers the broadcast folds as an index and the
168
+ # reduction is materialized (cheaper than redoing it per reader).
169
+ return nil unless consumers[input.id].length == 1
170
+ return nil unless consumers[input.id].all? { |reader| fusable?(reader) && reader.shape == input.shape }
171
+
172
+ # An expression touching only the folded value may sit between the
173
+ # reduction and the broadcast (rmsnorm's `sqrt(mean + eps)`). It is
174
+ # constant within a row, so it joins the cluster too. Only expressions
175
+ # combining **with scalars only** qualify — one reading a buffer would
176
+ # need the row index.
177
+ chain = []
178
+ node = input.inputs.first
179
+ while node.op != :reduce
180
+ return nil unless post_reduce_link?(node, input.inputs.first.shape, consumers, output_ids, pinned)
181
+
182
+ chain << node
183
+ node = node.inputs.first
184
+ end
185
+ return nil unless row_reduce?(node, consumer.shape)
186
+ return nil if output_ids.include?(node.id) || pinned.include?(node.id)
187
+ return nil unless consumers[node.id].length == 1
188
+
189
+ [node, chain]
190
+ end
191
+
192
+ # Nodes allowed in the cluster as expressions over the folded value.
193
+ # Having exactly one input (unary, cast, binary with a scalar) is what
194
+ # guarantees they read no buffer.
195
+ def post_reduce_link?(node, shape, consumers, output_ids, pinned)
196
+ return false unless %i[unary cast binary].include?(node.op)
197
+ return false if node.op == :binary && !node.attrs.key?(:scalar)
198
+ return false unless node.inputs.length == 1 && node.shape == shape
199
+ return false if Ops::LOW_PRECISION_DTYPES.include?(node.dtype)
200
+ return false if output_ids.include?(node.id) || pinned.include?(node.id)
201
+
202
+ consumers[node.id].length == 1
203
+ end
204
+
205
+ # A keepdims reduction over the last dim whose pre-reduction shape matches
206
+ # the cluster's. Not folded while rows are few (see MIN_ROWS above).
207
+ def row_reduce?(node, cluster_shape)
208
+ return false unless node.op == :reduce
209
+ return false if Ops::LOW_PRECISION_DTYPES.include?(node.dtype)
210
+ return false unless node.attrs[:keepdims]
211
+
212
+ source = node.inputs.first
213
+ axis = node.attrs[:axis]
214
+ return false if axis.nil? || axis != source.shape.length - 1
215
+ return false unless source.shape == cluster_shape
216
+
217
+ cluster_shape[0..-2].inject(1, :*) >= MIN_ROWS
218
+ end
219
+
220
+ # Puts the reduction and the elementwise ops before it into the cluster.
221
+ # When what comes before cannot be absorbed (read from two places, etc.)
222
+ # it stops there and the reduction reads that materialized value.
223
+ def absorb_row_reduce(found, broadcast, root, absorbed, members, consumers, output_ids, pinned)
224
+ reduce, chain = found
225
+ [broadcast, reduce, *chain].each do |node|
226
+ absorbed[node.id] = true
227
+ members[root.id] |= [node.id]
228
+ end
229
+
230
+ stack = reduce.inputs.dup
231
+ while (node = stack.pop)
232
+ next if absorbed[node.id]
233
+ next unless fusable?(node) && node.shape == reduce.inputs.first.shape
234
+ next if Ops::LOW_PRECISION_DTYPES.include?(node.dtype)
235
+ next if output_ids.include?(node.id) || pinned.include?(node.id)
236
+ next unless consumers[node.id].length == 1
237
+
238
+ absorbed[node.id] = true
239
+ members[root.id] |= (members.delete(node.id) || []) + [node.id]
240
+ stack.concat(node.inputs)
241
+ end
242
+ end
243
+
244
+ # If every reader folds, the broadcast need not be materialized. If even
245
+ # one needs the real buffer (a reduction, matmul, :nn, or the output
246
+ # itself), it must be materialized for that reader, so it is not folded
247
+ # — folding and materializing both would hold the same value two ways,
248
+ # adding lifetime bookkeeping for no gain.
249
+ def foldable_broadcast?(input, output_ids, consumers, pinned)
250
+ return false unless Op.fetch(input.op).index_map?
251
+ return false if Ops::LOW_PRECISION_DTYPES.include?(input.dtype)
252
+ return false if output_ids.include?(input.id) || pinned.include?(input.id)
253
+
254
+ readers = consumers[input.id]
255
+ readers.length > 1 && readers.all? { |reader| fusable?(reader) && reader.shape == input.shape }
256
+ end
257
+
258
+ def build_cluster(root, member_ids, absorbed)
259
+ inside = (member_ids + [root.id]).to_h { |id| [id, true] }
260
+ inputs = []
261
+ maps = []
262
+ scalars = []
263
+ reductions = []
264
+ expr = build_expr(root, inside, [inputs, maps, reductions], scalars, absorbed)
265
+ Cluster.new(root: root, members: member_ids, inputs: inputs, maps: maps,
266
+ scalars: scalars, expr: expr, reductions: reductions)
267
+ end
268
+
269
+ # Walks the tree from the root into an expression; values from outside
270
+ # the cluster become buffer arguments.
271
+ def build_expr(node, inside, args, scalars, absorbed)
272
+ return leaf(node, args) unless inside[node.id]
273
+ # A broadcast becomes **an index, not an expression**: the unstretched
274
+ # buffer is the argument and the way it is read (which dims repeat how
275
+ # often) folds into the index expression. That removes the kernel
276
+ # writing out the stretched value and the round trip reading it back.
277
+ return remap_leaf(node, inside, args, scalars, absorbed) if Op.fetch(node.op).index_map?
278
+ return reduce_expr(node, inside, args, scalars, absorbed) if node.op == :reduce
279
+
280
+ children = node.inputs.map { |input| build_expr(input, inside, args, scalars, absorbed) }
281
+ case node.op
282
+ when :unary then unary_expr(node, children.first)
283
+ when :binary then binary_expr(node, children, scalars)
284
+ when :cast then cast_expr(node, children.first)
285
+ when :where then select_expr(node, children, scalars)
286
+ else raise Error, "cannot fuse #{node.op.inspect}"
287
+ end
288
+ end
289
+
290
+ # The same node read with a different map is a separate argument. Plain
291
+ # and stretched reads can share one expression, so identity is [node, map].
292
+ def leaf(node, args, map = nil)
293
+ inputs, maps, = args
294
+ index = inputs.each_index.find { |i| inputs[i].equal?(node) && maps[i] == map }
295
+ unless index
296
+ index = inputs.length
297
+ inputs << node
298
+ maps << map
299
+ end
300
+ Expr.new(kind: :buffer, dtype: node.dtype, compute: node.dtype, index: index, map: map, children: [])
301
+ end
302
+
303
+ # When what is behind the broadcast is inside the cluster (= a row-wise
304
+ # reduction was pulled in), it becomes **that row's reduced value**, not an
305
+ # index remap. It is constant within the row, so no index is needed.
306
+ def remap_leaf(node, inside, args, scalars, absorbed)
307
+ source = node.inputs.first
308
+ return build_expr(source, inside, args, scalars, absorbed) if inside[source.id]
309
+
310
+ leaf(source, args, Kernels.remap_dims(node))
311
+ end
312
+
313
+ # A row-wise reduction. The expression is the pre-reduction value, and the
314
+ # result is computed once per row under the name red<k>.
315
+ def reduce_expr(node, inside, args, scalars, absorbed)
316
+ reductions = args[2]
317
+ expr = build_expr(node.inputs.first, inside, args, scalars, absorbed)
318
+ index = reductions.length
319
+ reductions << { kind: node.attrs[:kind], dtype: node.dtype, expr: expr }
320
+ Expr.new(kind: :reduced, dtype: node.dtype, compute: node.dtype, index: index, children: [])
321
+ end
322
+
323
+ def unary_expr(node, child)
324
+ fn = node.attrs[:fn]
325
+ kind = fn == :abs ? :abs : :math
326
+ Expr.new(kind: kind, dtype: node.dtype, compute: node.dtype == :int32 ? :int32 : :float32,
327
+ op: fn, children: [child])
328
+ end
329
+
330
+ def binary_expr(node, children, scalars)
331
+ kind = node.attrs[:kind] == :compare ? :compare : :binary
332
+ compute = compute_dtype(node)
333
+ operands = children
334
+ if node.attrs.key?(:scalar)
335
+ index = scalars.length
336
+ value = compute == :float32 ? node.attrs[:scalar].to_f : node.attrs[:scalar]
337
+ scalars << { value: value, dtype: compute }
338
+ scalar = Expr.new(kind: :scalar, dtype: compute, compute: compute, index: index, children: [])
339
+ operands = node.attrs[:reversed] ? [scalar, children.first] : [children.first, scalar]
340
+ end
341
+ Expr.new(kind: kind, dtype: node.dtype, compute: compute, op: node.attrs[:op], children: operands)
342
+ end
343
+
344
+ # A cast is exactly "a cast to the output type", the same as the unfused
345
+ # path's write-back, so in the tree it wraps like any other node.
346
+ def cast_expr(node, child)
347
+ Expr.new(kind: :cast, dtype: node.dtype, compute: node.dtype, children: [child])
348
+ end
349
+
350
+ # Select by condition. Children are [cond, a, b]; a side that is a scalar
351
+ # goes in as a kernel scalar argument (like a binary op's scalar).
352
+ def select_expr(node, children, scalars)
353
+ rest = children.drop(1)
354
+ operands = %i[a b].map do |key|
355
+ value = node.attrs[:"#{key}_scalar"]
356
+ next rest.shift if value.nil?
357
+
358
+ index = scalars.length
359
+ scalars << { value: value, dtype: node.dtype }
360
+ Expr.new(kind: :scalar, dtype: node.dtype, compute: node.dtype, index: index, children: [])
361
+ end
362
+ Expr.new(kind: :select, dtype: node.dtype, compute: node.dtype,
363
+ children: [children.first, *operands])
364
+ end
365
+
366
+ # Ops "widens to the compute type, then operates". A comparison outputs
367
+ # int32 but computes in its operands' type, so the output type cannot
368
+ # decide it.
369
+ def compute_dtype(node)
370
+ return node.dtype unless node.attrs[:kind] == :compare
371
+
372
+ operands = node.inputs.map(&:dtype)
373
+ operands << (node.attrs[:scalar].is_a?(Integer) ? :int32 : :float32) if node.attrs.key?(:scalar)
374
+ operands.all?(:int32) ? :int32 : :float32
375
+ end
376
+
377
+ # --- Dialect source ----------------------------------------------------
378
+
379
+ # A cluster with row-wise reductions. One thread owns one row, scanning it
380
+ # once per reduction and then once more to write. **Intermediates are
381
+ # recomputed rather than kept** — as in the hand-written NN kernels
382
+ # (softmax computes exp twice); not holding a row's worth of storage is
383
+ # this form's advantage.
384
+ def row_dialect_source(cluster)
385
+ params = kernel_params(cluster)
386
+ params << "constant int& rows"
387
+ params << "constant int& cols"
388
+
389
+ passes = cluster.reductions.each_with_index.map do |reduction, k|
390
+ acc = Ops::CTYPE.fetch(reduction[:dtype])
391
+ <<~PASS.chomp
392
+ #{acc} acc#{k} = (#{acc})0;
393
+ for (int c = 0; c < cols; ++c) {
394
+ int i = base + c;
395
+ #{acc} v = (#{acc})#{dialect_expr(reduction[:expr])};
396
+ acc#{k} = (c == 0) ? v : (#{combine(reduction[:kind], "acc#{k}", 'v')});
397
+ }
398
+ #{acc} red#{k} = #{finish(reduction[:kind], "acc#{k}", acc)};
399
+ PASS
400
+ end
401
+
402
+ <<~KERNEL
403
+ GPU_KERNEL void row_fused(GPU_ARGS,
404
+ #{params.join(",\n ")}) {
405
+ int r = GLOBAL_ID_X;
406
+ if (r < rows) {
407
+ int base = r * cols;
408
+ #{passes.join("\n")}
409
+ for (int c = 0; c < cols; ++c) {
410
+ int i = base + c;
411
+ out[i] = #{dialect_expr(cluster.expr)};
412
+ }
413
+ }
414
+ }
415
+ KERNEL
416
+ end
417
+
418
+ # How a reduction folds. The order is "from the first element", the same
419
+ # as the axis-reduction kernel (Kernels.axis_reduce), so fused and unfused
420
+ # give the same value.
421
+ def combine(kind, acc, value)
422
+ case kind
423
+ when :sum, :mean then "#{acc} + #{value}"
424
+ when :min then "#{value} < #{acc} ? #{value} : #{acc}"
425
+ when :max then "#{value} > #{acc} ? #{value} : #{acc}"
426
+ else raise Error, "cannot fuse the reduction #{kind.inspect}"
427
+ end
428
+ end
429
+
430
+ def finish(kind, acc, ctype) = kind == :mean ? "#{acc} / (#{ctype})cols" : acc
431
+
432
+ def kernel_params(cluster)
433
+ params = cluster.inputs.each_with_index.map do |node, i|
434
+ "device const #{Ops::CTYPE.fetch(node.dtype)}* a#{i}"
435
+ end
436
+ params += cluster.scalars.each_with_index.map do |scalar, i|
437
+ "constant #{Ops::CTYPE.fetch(scalar[:dtype])}& s#{i}"
438
+ end
439
+ params << "device #{Ops::CTYPE.fetch(cluster.dtype)}* out"
440
+ params
441
+ end
442
+
443
+ # Dialect source for a fused kernel. The name follows from the expression's
444
+ # shape, so clusters of the same shape collapse to one kernel in Kernel's
445
+ # SHA cache (32 layers with the same expression compile once).
446
+ def dialect_source(cluster)
447
+ return row_dialect_source(cluster) if cluster.row_wise?
448
+
449
+ params = kernel_params(cluster)
450
+ params << "constant int& n"
451
+
452
+ <<~KERNEL
453
+ GPU_KERNEL void fused(GPU_ARGS,
454
+ #{params.join(",\n ")}) {
455
+ int i = GLOBAL_ID_X;
456
+ if (i < n) out[i] = #{dialect_expr(cluster.expr)};
457
+ }
458
+ KERNEL
459
+ end
460
+
461
+ # Wraps each node **in a cast to its output type**, reproducing inside the
462
+ # expression the write-back the unfused path performed.
463
+ def dialect_expr(expr)
464
+ ct = Ops::CTYPE.fetch(expr.dtype)
465
+ case expr.kind
466
+ when :buffer then "(#{ct})a#{expr.index}[#{index_expr(expr.map)}]"
467
+ when :reduced then "red#{expr.index}"
468
+ when :scalar then "s#{expr.index}"
469
+ when :binary then "(#{ct})(#{operand(expr, 0)} #{expr.op} #{operand(expr, 1)})"
470
+ when :compare then "(#{ct})((#{operand(expr, 0)} #{expr.op} #{operand(expr, 1)}) ? 1 : 0)"
471
+ when :math then "(#{ct})#{expr.op}((float)#{dialect_expr(expr.children.first)})"
472
+ when :cast then "(#{ct})#{dialect_expr(expr.children.first)}"
473
+ when :select
474
+ "(#{ct})(#{dialect_expr(expr.children[0])} ? #{operand(expr, 1)} : #{operand(expr, 2)})"
475
+ when :abs
476
+ fn = expr.dtype == :int32 ? "abs" : "fabs"
477
+ "(#{ct})#{fn}(#{dialect_expr(expr.children.first)})"
478
+ else raise Error, "unknown fused expression #{expr.kind.inspect}"
479
+ end
480
+ end
481
+
482
+ # Index remap. Splits the output index i per dim and rebuilds the input
483
+ # index, dropping stretched dims (stride 0). **It is a single
484
+ # expression**, so it fits inside a kernel body that cannot hold statements.
485
+ #
486
+ # [d] -> [n, d] -> i % d
487
+ # [n, 1] -> [n, d] -> (i / d) % n
488
+ def index_expr(map)
489
+ return "i" if map.nil?
490
+
491
+ total = map.inject(1) { |acc, (size, _)| acc * size }
492
+ suffix = 1
493
+ terms = map.reverse.filter_map do |size, stride|
494
+ span = suffix
495
+ suffix *= size
496
+ # A stretched dim does not affect the input index, so its term drops.
497
+ next if stride.zero?
498
+
499
+ base = span == 1 ? "i" : "(i / #{span})"
500
+ # The outermost dim is already in range after the division, so no remainder is needed.
501
+ term = suffix == total ? base : "(#{base} % #{size})"
502
+ stride == 1 ? term : "#{term} * #{stride}"
503
+ end
504
+ terms.empty? ? "0" : terms.join(" + ")
505
+ end
506
+
507
+ # Operands are widened to the parent's compute type first (as in Ops.binary).
508
+ def operand(expr, index)
509
+ child = expr.children[index]
510
+ rendered = dialect_expr(child)
511
+ return rendered if child.dtype == expr.compute
512
+
513
+ "(#{Ops::CTYPE.fetch(expr.compute)})#{rendered}"
514
+ end
515
+
516
+ # No need for a different name per cluster (the cache key is the source's SHA).
517
+ def kernel_name(_cluster) = "fused"
518
+ end
519
+ end
520
+ end