tilework 0.18.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE +21 -0
  3. data/lib/tilework/backend/cpu/dsl_codegen.rb +509 -0
  4. data/lib/tilework/backend/cpu/ops_codegen.rb +451 -0
  5. data/lib/tilework/backend/cpu.rb +156 -0
  6. data/lib/tilework/buffer.rb +753 -0
  7. data/lib/tilework/capabilities.rb +183 -0
  8. data/lib/tilework/compat/cuda.h +66 -0
  9. data/lib/tilework/compat/hip.h +35 -0
  10. data/lib/tilework/compat/metal.h +45 -0
  11. data/lib/tilework/conformance.rb +2296 -0
  12. data/lib/tilework/core_ext.rb +21 -0
  13. data/lib/tilework/dialect.rb +67 -0
  14. data/lib/tilework/dsl.rb +722 -0
  15. data/lib/tilework/dtype/bfloat.rb +31 -0
  16. data/lib/tilework/dtype/cast.rb +88 -0
  17. data/lib/tilework/dtype/half.rb +98 -0
  18. data/lib/tilework/graph/builder.rb +1073 -0
  19. data/lib/tilework/graph/fusion.rb +520 -0
  20. data/lib/tilework/graph/grad.rb +443 -0
  21. data/lib/tilework/graph/high_level.rb +266 -0
  22. data/lib/tilework/graph/kernels.rb +403 -0
  23. data/lib/tilework/graph/node.rb +27 -0
  24. data/lib/tilework/graph/op.rb +290 -0
  25. data/lib/tilework/graph/optim.rb +183 -0
  26. data/lib/tilework/graph/plan.rb +1086 -0
  27. data/lib/tilework/graph/pool.rb +178 -0
  28. data/lib/tilework/graph/region.rb +210 -0
  29. data/lib/tilework/graph/residency.rb +449 -0
  30. data/lib/tilework/graph/state.rb +64 -0
  31. data/lib/tilework/graph/tensor.rb +124 -0
  32. data/lib/tilework/graph.rb +213 -0
  33. data/lib/tilework/kernel.rb +163 -0
  34. data/lib/tilework/math/conv.rb +270 -0
  35. data/lib/tilework/math/fft.rb +172 -0
  36. data/lib/tilework/math/nn.rb +1103 -0
  37. data/lib/tilework/math/philox.rb +194 -0
  38. data/lib/tilework/math/random.rb +232 -0
  39. data/lib/tilework/math/tensor_core.rb +269 -0
  40. data/lib/tilework/ops.rb +682 -0
  41. data/lib/tilework/runtime/offload.rb +498 -0
  42. data/lib/tilework/runtime/submission.rb +163 -0
  43. data/lib/tilework/runtime/telemetry.rb +112 -0
  44. data/lib/tilework/scope.rb +33 -0
  45. data/lib/tilework/version.rb +5 -0
  46. data/lib/tilework.rb +345 -0
  47. metadata +92 -0
@@ -0,0 +1,266 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Tilework
4
+ module Graph
5
+ # The contract for high-level ops (Tier 1 in docs/graph-ir.md).
6
+ #
7
+ # A function like `NN.softmax` is high-level **because it is fast**, not
8
+ # because it cannot be written any other way — that is this layer's
9
+ # premise, and this table makes it checkable. Each row says how to write
10
+ # the function in primitives alone.
11
+ #
12
+ # HighLevel.define(:softmax) do |_g, inputs, _tensors, _args, _opts|
13
+ # e = (inputs[0] - inputs[0].max(axis: -1, keepdims: true)).exp
14
+ # e / e.sum(axis: -1, keepdims: true)
15
+ # end
16
+ #
17
+ # The default execution path is still the hand-written fused kernel (a
18
+ # `:nn` node); the decomposition does not run. Writing it down still pays
19
+ # three ways:
20
+ #
21
+ # 1. **Portability** — there is a form that runs on a backend without
22
+ # that fused kernel (every backend has them today, so it is unused)
23
+ # 2. **Backward (#22)** — gradients follow from primitive vjps alone, so a
24
+ # high-level op can be "decomposed, then differentiated"; hand-written
25
+ # vjps are added only where speed demands
26
+ # 3. **Oracle** — `Graph.lower_high_level = true` switches to the
27
+ # decomposition to compare against the fused kernel, the same trick as
28
+ # verifying Fusion on :cpu, so both paths are checked without a GPU in CI
29
+ #
30
+ # Agreement is to a tolerance, not bits: the decomposition reduces in a
31
+ # different order and accumulates in a different type than the hand-written
32
+ # kernel, which cannot be matched.
33
+ #
34
+ # **Some functions have no decomposition.** `gelu_exact` needs erf, which
35
+ # is not a primitive, and `groupnorm` folds across channels, which is not a
36
+ # row-wise reduction. Functions not in the table stay `:nn` nodes — the
37
+ # table never pretends to express what it cannot.
38
+ module HighLevel
39
+ REGISTRY = {}
40
+ # The capability the fast path needs. None means every backend has it
41
+ # (NN's fused kernels can be generated anywhere).
42
+ REQUIREMENTS = {}
43
+
44
+ module_function
45
+
46
+ def define(name, requires: nil, &lower)
47
+ raise Error, "high-level op #{name.inspect} is already defined" if REGISTRY.key?(name)
48
+ raise ArgumentError, "define needs a block" if lower.nil?
49
+
50
+ REGISTRY[name] = lower
51
+ REQUIREMENTS[name] = requires if requires
52
+ end
53
+
54
+ def defined?(name) = REGISTRY.key?(name)
55
+ def names = REGISTRY.keys
56
+
57
+ # Decompose in two cases: **explicitly switched on as the oracle**, and
58
+ # **this backend lacks the fast path**. The latter is portability itself;
59
+ # conv2d is the first instance (:cpu has no convolution).
60
+ def lower?(name)
61
+ return false unless REGISTRY.key?(name)
62
+
63
+ Graph.lower_high_level || !available?(name)
64
+ end
65
+
66
+ # Whether the fast path is available. A function with no requirement always is.
67
+ def available?(name)
68
+ capability = REQUIREMENTS[name]
69
+ capability.nil? || Op.satisfied?(capability)
70
+ end
71
+
72
+ def lower(builder, name, inputs, tensors:, args:, opts:)
73
+ REGISTRY.fetch(name).call(builder, inputs, tensors, args, opts)
74
+ end
75
+ end
76
+
77
+ # Every activation is a single expression, so they are written together
78
+ # here, in the same form as nn.rb's ACTIVATIONS. sigmoid / silu split on
79
+ # the sign because `exp(-v)` overflows for very negative v (that side uses
80
+ # the rearranged form).
81
+ ACTIVATION_EXPRS = {
82
+ relu: ->(g, x) { g.where(x > 0.0, x, 0.0) },
83
+ sigmoid: ->(g, x) { g.where(x >= 0.0, 1.0 / (1.0 + (-1.0 * x).exp), x.exp / (1.0 + x.exp)) },
84
+ silu: ->(g, x) { x * ACTIVATION_EXPRS[:sigmoid].call(g, x) },
85
+ gelu_tanh: lambda { |_g, x|
86
+ inner = 0.7978845608028654 * (x + (0.044715 * x * x * x))
87
+ 0.5 * x * (1.0 + inner.tanh)
88
+ },
89
+ }.freeze
90
+
91
+ HighLevel.define(:softmax) do |_g, inputs, _tensors, _args, _opts|
92
+ x = inputs[0]
93
+ # Subtract the max before exp, like the hand-written kernel (prevents overflow).
94
+ e = (x - x.max(axis: -1, keepdims: true)).exp
95
+ e / e.sum(axis: -1, keepdims: true)
96
+ end
97
+
98
+ HighLevel.define(:layernorm) do |_g, inputs, tensors, _args, opts|
99
+ x = inputs[0]
100
+ centered = x - x.mean(axis: -1, keepdims: true)
101
+ variance = (centered * centered).mean(axis: -1, keepdims: true)
102
+ y = centered / (variance + opts.fetch(:eps)).sqrt
103
+ y *= tensors[:gamma] if tensors[:gamma]
104
+ y += tensors[:beta] if tensors[:beta]
105
+ y
106
+ end
107
+
108
+ HighLevel.define(:rmsnorm) do |_g, inputs, tensors, _args, opts|
109
+ x = inputs[0]
110
+ # No mean subtraction (that is the definition of rmsnorm).
111
+ y = x / ((x * x).mean(axis: -1, keepdims: true) + opts.fetch(:eps)).sqrt
112
+ y *= tensors[:gamma] if tensors[:gamma]
113
+ y
114
+ end
115
+
116
+ # Loss reduction. Anything but :none folds everything into shape [1] at the end.
117
+ LOSS_REDUCE = lambda { |value, reduction|
118
+ case reduction
119
+ when :none then value
120
+ when :sum then value.sum
121
+ else value.mean
122
+ end
123
+ }
124
+
125
+ HighLevel.define(:mse) do |_g, inputs, _tensors, _args, opts|
126
+ # Losses are kept in float (the same rule as nn.rb's loss_dtype).
127
+ pred, target = inputs.map { |t| Ops::LOW_PRECISION_DTYPES.include?(t.dtype) ? t.cast(:float32) : t }
128
+ d = pred - target
129
+ LOSS_REDUCE.call(d * d, opts.fetch(:reduction))
130
+ end
131
+
132
+ # log-sum-exp in plain ops. Like the hand-written kernel it subtracts the
133
+ # max before exp, so large logits do not overflow.
134
+ #
135
+ # The label's logit is picked with an iota equality mask rather than
136
+ # gather. That needs only elementwise ops and a row reduction, so it
137
+ # differentiates without going through gather's vjp (scatter-add).
138
+ # Clamping out-of-range labels to the ends also matches the hand-written kernel.
139
+ HighLevel.define(:cross_entropy) do |g, inputs, _tensors, _args, opts|
140
+ logits, labels = inputs
141
+ x = Ops::LOW_PRECISION_DTYPES.include?(logits.dtype) ? logits.cast(:float32) : logits
142
+ cols = x.shape[-1]
143
+ rows_shape = x.shape[0..-2]
144
+
145
+ m = x.max(axis: -1, keepdims: true)
146
+ lse = m + (x - m).exp.sum(axis: -1, keepdims: true).log
147
+
148
+ clamped = g.where(labels < 0, 0, labels)
149
+ clamped = g.where(clamped > (cols - 1), cols - 1, clamped)
150
+ picked = g.where(g.iota(x.shape, axis: -1).eq(clamped.reshape(*rows_shape, 1)), x, 0.0)
151
+ .sum(axis: -1, keepdims: true)
152
+
153
+ LOSS_REDUCE.call((lse - picked).reshape(*rows_shape), opts.fetch(:reduction))
154
+ end
155
+
156
+ HighLevel.define(:transpose) do |g, inputs, _tensors, _args, _opts|
157
+ g.permute(inputs[0], 1, 0)
158
+ end
159
+
160
+ # Heads split the last dim evenly ([n, d] with d = heads x dh). Split
161
+ # (reshape), bring heads to the front (permute), compute all heads with a
162
+ # batched matmul, then restore. bias arrives as [heads * n, m], which is
163
+ # heads stacked vertically, so reading it as [heads, n, m] is enough (same
164
+ # element order).
165
+ HighLevel.define(:multi_head_attention) do |_g, inputs, tensors, _args, opts|
166
+ q, k, v = inputs
167
+ heads = opts.fetch(:heads)
168
+ n, d = q.shape
169
+ m = k.shape[0]
170
+ dh = d / heads
171
+
172
+ scores = q.reshape(n, heads, dh).permute(1, 0, 2) # [h, n, dh]
173
+ .matmul(k.reshape(m, heads, dh).permute(1, 2, 0)) * opts.fetch(:scale)
174
+ scores += tensors[:bias].reshape(heads, n, m) if tensors[:bias]
175
+ NN.softmax(scores) # rows = last dim
176
+ .matmul(v.reshape(m, heads, dh).permute(1, 0, 2)) # [h, n, dh]
177
+ .permute(1, 0, 2).reshape(n, d)
178
+ end
179
+
180
+ # Opens a 2D convolution into im2col + matmul.
181
+ #
182
+ # It never runs while a fast path (cuDNN / MPSGraph) exists. It is here so
183
+ # graphs run on a backend without convolution (:cpu) and so backward
184
+ # follows from primitive vjps alone.
185
+ #
186
+ # The index table is built inside the graph from iota and integer
187
+ # arithmetic (recording touches no device).
188
+ HighLevel.define(:conv2d, requires: :conv) do |g, inputs, _tensors, _args, opts|
189
+ x, weight = inputs
190
+ n, cin, h, w = x.shape
191
+ cout, _, kh, kw = weight.shape
192
+ ph, pw = opts.fetch(:pad)
193
+ sh, sw = opts.fetch(:stride)
194
+ dh, dw = opts.fetch(:dilation, [1, 1])
195
+ groups = opts.fetch(:groups, 1)
196
+ # dilation acts through the effective kernel width, leaving (d - 1) gaps between taps.
197
+ effective_kh = ((kh - 1) * dh) + 1
198
+ effective_kw = ((kw - 1) * dw) + 1
199
+ hp = h + (2 * ph)
200
+ wp = w + (2 * pw)
201
+ oh = ((hp - effective_kh) / sh) + 1
202
+ ow = ((wp - effective_kw) / sw) + 1
203
+ rows = n * oh * ow
204
+
205
+ padded = g.pad(x, before: [0, 0, ph, pw], after: [0, 0, ph, pw], value: 0.0)
206
+ .reshape(n * cin * hp * wp)
207
+
208
+ modulo = ->(value, by) { value - ((value / by) * by) }
209
+
210
+ # im2col + matmul for one group. channel_offset is the channel of the
211
+ # padded input where that group starts (always 0 when groups == 1).
212
+ im2col_matmul = lambda do |weight_slice, cin_group, cout_group, channel_offset|
213
+ k = cin_group * kh * kw
214
+ r = g.iota([rows, k], axis: 0)
215
+ c = g.iota([rows, k], axis: 1)
216
+ batch = r / (oh * ow)
217
+ out_y = modulo.call(r / ow, oh)
218
+ out_x = modulo.call(r, ow)
219
+ channel = (c / (kh * kw)) + channel_offset
220
+ tap_y = modulo.call(c / kw, kh)
221
+ tap_x = modulo.call(c, kw)
222
+
223
+ index = (batch * (cin * hp * wp)) + (channel * (hp * wp)) +
224
+ (((out_y * sh) + (tap_y * dh)) * wp) + ((out_x * sw) + (tap_x * dw))
225
+
226
+ g.gather(padded, index) # [rows, k]
227
+ .matmul(weight_slice.reshape(cout_group, k).permute(1, 0)) # [rows, cout_group]
228
+ end
229
+
230
+ # Open output row r = (n, oh, ow) and column c = (cin_per_group, kh, kw)
231
+ # into indices. % is not a primitive, so it is written a - (a / b) * b
232
+ # (integer division truncates the same on the dialect side and on :cpu).
233
+ cols = if groups == 1
234
+ im2col_matmul.call(weight, cin, cout, 0)
235
+ else
236
+ # groups > 1: build an independent im2col + matmul per group and
237
+ # concatenate along cout (axis 1, not the outermost). Groups'
238
+ # input channel ranges do not overlap, so this is correct.
239
+ cin_per_group = cin / groups
240
+ cout_per_group = cout / groups
241
+ group_outputs = (0...groups).map do |gi|
242
+ first = gi * cout_per_group
243
+ weight_slice = weight.slice(first..(first + cout_per_group - 1))
244
+ im2col_matmul.call(weight_slice, cin_per_group, cout_per_group, gi * cin_per_group)
245
+ end
246
+ g.concat(group_outputs, axis: 1)
247
+ end
248
+
249
+ cols.permute(1, 0).reshape(cout, n, oh, ow)
250
+ .permute(1, 0, 2, 3)
251
+ end
252
+
253
+ HighLevel.define(:activation) do |g, inputs, _tensors, args, _opts|
254
+ kind = args.first
255
+ expr = ACTIVATION_EXPRS[kind]
256
+ # gelu_exact cannot be decomposed (erf is not a primitive). Failing beats
257
+ # silently substituting another approximation.
258
+ if expr.nil?
259
+ raise Error, "#{kind} has no decomposition (erf is not a primitive), so it cannot run " \
260
+ "with Graph.lower_high_level = true"
261
+ end
262
+
263
+ expr.call(g, inputs[0])
264
+ end
265
+ end
266
+ end
@@ -0,0 +1,403 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Tilework
4
+ module Graph
5
+ # Kernels the graph layer owns: ops with no eager (Ops) counterpart, such
6
+ # as axis reductions and index remaps.
7
+ #
8
+ # Their output takes the same two forms as everything else (dialect source /
9
+ # :cpu Ruby source), so it rides Kernel's SHA cache as is. No branching on
10
+ # backend names; Ops.compiled routes by capabilities[:dialect].
11
+ #
12
+ # ## Numerics
13
+ #
14
+ # Axis reductions **fold in order from the first element in one thread**.
15
+ # Unlike Ops.reduce's tree, the order is fully fixed, so the same input gives
16
+ # the same value every run. The accumulator is float (ACC) on the dialect
17
+ # side and Ruby double on :cpu, so the two differ by what accumulates before
18
+ # the final rounding — which is why conformance uses a tolerance for axis
19
+ # reductions (bit identity as with fusion is not available).
20
+ module Kernels
21
+ module_function
22
+
23
+ # Axis reduction. Viewed as [outer, axis_len, inner], one thread per
24
+ # output element folds axis_len values. outer / inner / axis_len are
25
+ # launch-time scalars, so the source does not change with the shape (no
26
+ # recompilation).
27
+ #
28
+ # args: [a, out, axis_len, inner, n]
29
+ def axis_reduce(kind:, in_dtype:, out_dtype:)
30
+ acc = Ops::CTYPE.fetch(out_dtype)
31
+ name = "axis_#{kind}_#{in_dtype}_#{out_dtype}"
32
+ src = <<~KERNEL
33
+ GPU_KERNEL void #{name}(GPU_ARGS,
34
+ device const #{Ops::CTYPE.fetch(in_dtype)}* a,
35
+ device #{acc}* c,
36
+ constant int& axis_len,
37
+ constant int& inner,
38
+ constant int& n) {
39
+ int i = GLOBAL_ID_X;
40
+ if (i < n) {
41
+ int base = (i / inner) * axis_len * inner + (i % inner);
42
+ #{acc} acc = (#{acc})a[base];
43
+ for (int t = 1; t < axis_len; ++t) {
44
+ #{acc} v = (#{acc})a[base + t * inner];
45
+ #{combine_expr(kind)}
46
+ }
47
+ c[i] = #{finish_expr(kind, acc)};
48
+ }
49
+ }
50
+ KERNEL
51
+
52
+ Ops.compiled(dialect_name: name, dialect_src: src, cpu_name: name,
53
+ cpu_src: -> { Ops.codegen.axis_reduce(kind: kind, in_dtype: in_dtype, out_dtype: out_dtype) })
54
+ end
55
+
56
+ # Index reduction along an axis: returns positions, not values. Ties keep
57
+ # the smaller index (left-biased like Ops.arg_reduce).
58
+ #
59
+ # args: [a, out, axis_len, inner, n]
60
+ def axis_arg_reduce(kind:, in_dtype:)
61
+ acc = Ops::CTYPE.fetch(Ops::LOW_PRECISION_DTYPES.include?(in_dtype) ? :float32 : in_dtype)
62
+ cmp = kind == :max ? ">" : "<"
63
+ name = "axis_arg#{kind}_#{in_dtype}"
64
+ src = <<~KERNEL
65
+ GPU_KERNEL void #{name}(GPU_ARGS,
66
+ device const #{Ops::CTYPE.fetch(in_dtype)}* a,
67
+ device int* c,
68
+ constant int& axis_len,
69
+ constant int& inner,
70
+ constant int& n) {
71
+ int i = GLOBAL_ID_X;
72
+ if (i < n) {
73
+ int base = (i / inner) * axis_len * inner + (i % inner);
74
+ #{acc} best = (#{acc})a[base];
75
+ int at = 0;
76
+ for (int t = 1; t < axis_len; ++t) {
77
+ #{acc} v = (#{acc})a[base + t * inner];
78
+ if (v #{cmp} best) { best = v; at = t; }
79
+ }
80
+ c[i] = at;
81
+ }
82
+ }
83
+ KERNEL
84
+
85
+ Ops.compiled(dialect_name: name, dialect_src: src, cpu_name: name,
86
+ cpu_src: -> { Ops.codegen.axis_arg_reduce(kind: kind, in_dtype: in_dtype) })
87
+ end
88
+
89
+
90
+ # Cast. It **only changes the storage type**, with C cast semantics (float
91
+ # -> int truncates toward zero). :cpu's to_i rounds the same way, so both
92
+ # paths give the same values.
93
+ #
94
+ # args: [a, out, n]
95
+ def cast(from:, to:)
96
+ name = "cast_#{from}_#{to}"
97
+ read = Ops.read_dtype(from, "a[i]")
98
+ src = <<~KERNEL
99
+ GPU_KERNEL void #{name}(GPU_ARGS,
100
+ device const #{Ops::CTYPE.fetch(from)}* a,
101
+ device #{Ops::CTYPE.fetch(to)}* c,
102
+ constant int& n) {
103
+ int i = GLOBAL_ID_X;
104
+ if (i < n) c[i] = (#{Ops::CTYPE.fetch(to)})#{read};
105
+ }
106
+ KERNEL
107
+
108
+ Ops.compiled(dialect_name: name, dialect_src: src, cpu_name: name,
109
+ cpu_src: -> { Ops.codegen.cast(from: from, to: to) })
110
+ end
111
+
112
+ # Select by condition. a / b may each be a buffer or a scalar, which the
113
+ # host already knows (graph shapes are static), so the argument layout is
114
+ # baked into the source. The positions are always [cond, a, b, out, n];
115
+ # only the side that became a scalar turns into a constant reference.
116
+ def select(dtype:, a_scalar:, b_scalar:)
117
+ ct = Ops::CTYPE.fetch(dtype)
118
+ name = "select_#{dtype}#{a_scalar ? '_sa' : ''}#{b_scalar ? '_sb' : ''}"
119
+ operand = ->(sym, scalar) { scalar ? sym.to_s : "#{sym}[i]" }
120
+ param = ->(sym, scalar) { scalar ? "constant #{ct}& #{sym}" : "device const #{ct}* #{sym}" }
121
+ src = <<~KERNEL
122
+ GPU_KERNEL void #{name}(GPU_ARGS,
123
+ device const int* cond,
124
+ #{param.call(:a, a_scalar)},
125
+ #{param.call(:b, b_scalar)},
126
+ device #{ct}* c,
127
+ constant int& n) {
128
+ int i = GLOBAL_ID_X;
129
+ if (i < n) c[i] = cond[i] ? (#{ct})#{operand.call(:a, a_scalar)} : (#{ct})#{operand.call(:b, b_scalar)};
130
+ }
131
+ KERNEL
132
+
133
+ Ops.compiled(dialect_name: name, dialect_src: src, cpu_name: name,
134
+ cpu_src: -> { Ops.codegen.select(dtype: dtype, a_scalar: a_scalar, b_scalar: b_scalar) })
135
+ end
136
+
137
+ # Index along an axis: builds masks and positions without bringing
138
+ # constants in from the host.
139
+ #
140
+ # args: [out, n]
141
+ def iota(shape:, axis:, dtype:)
142
+ size = shape[axis]
143
+ inner = shape.drop(axis + 1).inject(1, :*)
144
+ ct = Ops::CTYPE.fetch(dtype)
145
+ name = "iota_#{dtype}"
146
+ src = <<~KERNEL
147
+ GPU_KERNEL void #{name}(GPU_ARGS,
148
+ device #{ct}* c,
149
+ constant int& n) {
150
+ int i = GLOBAL_ID_X;
151
+ if (i < n) c[i] = (#{ct})((i / #{inner}) % #{size});
152
+ }
153
+ KERNEL
154
+
155
+ Ops.compiled(dialect_name: name, dialect_src: src, cpu_name: name,
156
+ cpu_src: -> { Ops.codegen.iota(dtype: dtype, size: size, inner: inner) })
157
+ end
158
+
159
+ # Table lookup. Each element of indices points at a position along
160
+ # source's axis — exactly what an embedding does (pulling rows from a
161
+ # vocabulary table).
162
+ #
163
+ # **Indices are clamped into range.** Reading out of range on a GPU is
164
+ # undefined, so clamping to the ends beats breaking silently (:cpu clamps
165
+ # the same way, so the answer does not depend on the path).
166
+ #
167
+ # args: [a, idx, out, n]
168
+ def gather(dtype:, axis_len:, idx_len:, inner:)
169
+ ct = Ops::CTYPE.fetch(dtype)
170
+ name = "gather_#{dtype}"
171
+ src = <<~KERNEL
172
+ GPU_KERNEL void #{name}(GPU_ARGS,
173
+ device const #{ct}* a,
174
+ device const int* idx,
175
+ device #{ct}* c,
176
+ constant int& n) {
177
+ int i = GLOBAL_ID_X;
178
+ if (i < n) {
179
+ int block = #{idx_len * inner};
180
+ int j = (i % block) / #{inner};
181
+ int at = idx[j];
182
+ at = at < 0 ? 0 : (at > #{axis_len - 1} ? #{axis_len - 1} : at);
183
+ c[i] = a[(i / block) * #{axis_len * inner} + at * #{inner} + (i % #{inner})];
184
+ }
185
+ }
186
+ KERNEL
187
+
188
+ Ops.compiled(dialect_name: name, dialect_src: src, cpu_name: name,
189
+ cpu_src: lambda {
190
+ Ops.codegen.gather(dtype: dtype, axis_len: axis_len, idx_len: idx_len, inner: inner)
191
+ })
192
+ end
193
+
194
+ # The inverse of table lookup (scatter-add): adds cotangents back to where
195
+ # gather read.
196
+ #
197
+ # dsource[o, j, k] = Σ_i [ids[i] == j] · dy[o, i, k]
198
+ #
199
+ # **No atomics.** One thread per output element scans the index list and
200
+ # adds only the entries addressed to it. Each thread owns its destination,
201
+ # so nothing races, and the fold order is fixed, so **every run gives the
202
+ # same value** (an atomic scatter-add moves the last bits as its order
203
+ # varies). The same reason keeps GPU_ATOMIC_ADD out of the dialect — a
204
+ # macro whose meaning differs across backends violates the compat header
205
+ # policy.
206
+ #
207
+ # The work is index count × output elements, the same as a matmul with a
208
+ # match matrix. The difference is not materializing the [table rows,
209
+ # index count] mask, which matters when the table is large, as with
210
+ # embeddings.
211
+ #
212
+ # args: [idx, dy, out, n]
213
+ def scatter_add(dtype:, axis_len:, idx_len:, inner:)
214
+ ct = Ops::CTYPE.fetch(dtype)
215
+ acc = Ops::CTYPE.fetch(Ops::LOW_PRECISION_DTYPES.include?(dtype) ? :float32 : dtype)
216
+ name = "scatter_add_#{dtype}"
217
+ src = <<~KERNEL
218
+ GPU_KERNEL void #{name}(GPU_ARGS,
219
+ device const int* idx,
220
+ device const #{ct}* dy,
221
+ device #{ct}* c,
222
+ constant int& n) {
223
+ int i = GLOBAL_ID_X;
224
+ if (i < n) {
225
+ int block = #{axis_len * inner};
226
+ int outer = i / block;
227
+ int j = (i % block) / #{inner};
228
+ int lane = i % #{inner};
229
+ #{acc} acc = (#{acc})0;
230
+ for (int t = 0; t < #{idx_len}; ++t) {
231
+ int at = idx[t];
232
+ at = at < 0 ? 0 : (at > #{axis_len - 1} ? #{axis_len - 1} : at);
233
+ if (at == j) acc += (#{acc})dy[outer * #{idx_len * inner} + t * #{inner} + lane];
234
+ }
235
+ c[i] = (#{ct})acc;
236
+ }
237
+ }
238
+ KERNEL
239
+
240
+ Ops.compiled(dialect_name: name, dialect_src: src, cpu_name: name,
241
+ cpu_src: lambda {
242
+ Ops.codegen.scatter_add(dtype: dtype, axis_len: axis_len,
243
+ idx_len: idx_len, inner: inner)
244
+ })
245
+ end
246
+
247
+ # Fills a border. dims holds, per output dim,
248
+ # [output size, input size, amount added in front, input-side stride].
249
+ #
250
+ # args: [a, out, n]
251
+ def pad(dims:, dtype:, value:)
252
+ ct = Ops::CTYPE.fetch(dtype)
253
+ name = "pad_#{dtype}"
254
+ lines = pad_lines(dims)
255
+ src = <<~KERNEL
256
+ GPU_KERNEL void #{name}(GPU_ARGS,
257
+ device const #{ct}* a,
258
+ device #{ct}* c,
259
+ constant #{ct}& fill,
260
+ constant int& n) {
261
+ int i = GLOBAL_ID_X;
262
+ if (i < n) {
263
+ int rem = i;
264
+ int src = 0;
265
+ int inside = 1;
266
+ int p = 0;
267
+ #{lines.map { |line| " #{line}" }.join("\n")}
268
+ c[i] = inside ? a[src] : fill;
269
+ }
270
+ }
271
+ KERNEL
272
+
273
+ Ops.compiled(dialect_name: name, dialect_src: src, cpu_name: name,
274
+ cpu_src: -> { Ops.codegen.pad(dtype: dtype, dims: dims) })
275
+ end
276
+
277
+ # Builds the index per dim, checking whether the coordinate falls inside the input.
278
+ def pad_lines(dims)
279
+ dims.reverse.map do |out_dim, in_dim, before, stride|
280
+ coord = "p = rem % #{out_dim}; rem /= #{out_dim};"
281
+ next "#{coord} src += p * #{stride};" if before.zero? && in_dim == out_dim
282
+
283
+ "#{coord} p -= #{before}; " \
284
+ "inside = inside && p >= 0 && p < #{in_dim}; " \
285
+ "src += (p < 0 ? 0 : (p >= #{in_dim} ? 0 : p)) * #{stride};"
286
+ end
287
+ end
288
+
289
+ # Writes one concat piece into its place in the output. Along the
290
+ # outermost dim pieces are contiguous and one copy suffices (Plan picks
291
+ # that); along an inner dim they are scattered, so each element's
292
+ # destination is computed.
293
+ #
294
+ # args: [piece, out, n]
295
+ def concat_piece(dtype:, piece_axis:, total_axis:, inner:, offset:)
296
+ ct = Ops::CTYPE.fetch(dtype)
297
+ name = "concat_piece_#{dtype}"
298
+ src = <<~KERNEL
299
+ GPU_KERNEL void #{name}(GPU_ARGS,
300
+ device const #{ct}* a,
301
+ device #{ct}* c,
302
+ constant int& n) {
303
+ int i = GLOBAL_ID_X;
304
+ if (i < n) {
305
+ int block = #{piece_axis * inner};
306
+ c[(i / block) * #{total_axis * inner} +
307
+ (#{offset} + (i % block) / #{inner}) * #{inner} + (i % #{inner})] = a[i];
308
+ }
309
+ }
310
+ KERNEL
311
+
312
+ Ops.compiled(dialect_name: name, dialect_src: src, cpu_name: name,
313
+ cpu_src: lambda {
314
+ Ops.codegen.concat_piece(dtype: dtype, piece_axis: piece_axis,
315
+ total_axis: total_axis, inner: inner, offset: offset)
316
+ })
317
+ end
318
+
319
+ # Index remap: decompose the output index per dim and rebuild the input
320
+ # index. **Broadcast and axis permutation are the same kernel**; only the
321
+ # table of "how far the input advances per output dim (stride)" differs:
322
+ #
323
+ # broadcast a stretched dim has stride 0 (always the same element)
324
+ # permute output dim d takes the stride of input dim axes[d]
325
+ #
326
+ # The table is known on the host (graph shapes are static), so it is
327
+ # **baked into the kernel as constants**; each shape combination compiles once.
328
+ #
329
+ # args: [a, out, n]
330
+ def remap(dims:, dtype:, name: "remap")
331
+ ctype = Ops::CTYPE.fetch(dtype)
332
+ name = "#{name}_#{dtype}"
333
+ src = <<~KERNEL
334
+ GPU_KERNEL void #{name}(GPU_ARGS,
335
+ device const #{ctype}* a,
336
+ device #{ctype}* c,
337
+ constant int& n) {
338
+ int i = GLOBAL_ID_X;
339
+ if (i < n) {
340
+ int rem = i;
341
+ int src = 0;
342
+ #{dims.reverse.map { |dim| " #{index_line(dim)}" }.join("\n")}
343
+ c[i] = a[src];
344
+ }
345
+ }
346
+ KERNEL
347
+
348
+ Ops.compiled(dialect_name: name, dialect_src: src, cpu_name: name,
349
+ cpu_src: -> { Ops.codegen.remap(dtype: dtype, dims: dims) })
350
+ end
351
+
352
+ # The stride table for an index-remapping node (:broadcast / :permute).
353
+ def remap_dims(node)
354
+ source = node.inputs.first
355
+ case node.op
356
+ when :broadcast then broadcast_dims(source.shape, node.shape)
357
+ when :permute then permute_dims(source.shape, node.attrs[:axes])
358
+ else raise Error, "#{node.op.inspect} does not remap indices"
359
+ end
360
+ end
361
+
362
+ # [output size, input-side stride] per output dim. A stretched dim has
363
+ # stride 0 — advancing along it always points at the same input element.
364
+ def broadcast_dims(in_shape, out_shape)
365
+ padded = Array.new(out_shape.length - in_shape.length, 1) + in_shape
366
+ strides = row_major_strides(padded)
367
+ out_shape.each_with_index.map do |dim, d|
368
+ [dim, padded[d] == 1 && dim != 1 ? 0 : strides[d]]
369
+ end
370
+ end
371
+
372
+ # Output dim d corresponds to input dim axes[d], so it takes that stride as is.
373
+ def permute_dims(in_shape, axes)
374
+ strides = row_major_strides(in_shape)
375
+ axes.map { |axis| [in_shape[axis], strides[axis]] }
376
+ end
377
+
378
+ def row_major_strides(shape)
379
+ stride = 1
380
+ shape.reverse.map { |dim| stride.tap { stride *= dim } }.reverse
381
+ end
382
+
383
+ # Index arithmetic for one dim, peeling dims off from the last one.
384
+ def index_line(dim)
385
+ size, stride = dim
386
+ return "rem /= #{size};" if stride.zero?
387
+
388
+ "src += (rem % #{size}) * #{stride}; rem /= #{size};"
389
+ end
390
+
391
+ def combine_expr(kind)
392
+ case kind
393
+ when :sum, :mean then "acc = acc + v;"
394
+ when :min then "acc = v < acc ? v : acc;"
395
+ when :max then "acc = v > acc ? v : acc;"
396
+ else raise Error, "unknown reduction #{kind.inspect}"
397
+ end
398
+ end
399
+
400
+ def finish_expr(kind, acc) = kind == :mean ? "acc / (#{acc})axis_len" : "acc"
401
+ end
402
+ end
403
+ end
@@ -0,0 +1,27 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Tilework
4
+ module Graph
5
+ # One graph node. It holds **no value** — only dtype and shape (the
6
+ # metadata part of a Buffer) and how it is produced. Plan#run assigns the
7
+ # actual buffer.
8
+ #
9
+ # A node only knows which op it is. How that op behaves (may it be freed,
10
+ # does it run synchronously, can it fuse, how it prints) lives in the
11
+ # Graph::Op table.
12
+ #
13
+ # order_deps are ordering edges that carry no value, added by
14
+ # Plan#compile. They express the WAR rule for mutable state ("a write runs
15
+ # after the readers").
16
+ Node = Struct.new(:id, :op, :inputs, :attrs, :dtype, :shape) do
17
+ def length = shape.inject(1, :*)
18
+ def state = attrs[:state]
19
+ def region = attrs[:region]
20
+ def spec = Op.fetch(op)
21
+ def writes_state? = spec.writes_state?
22
+ def label = spec.label_for(self)
23
+
24
+ def to_s = "##{id} #{label} #{shape.inspect} #{dtype}"
25
+ end
26
+ end
27
+ end