tilework 0.18.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE +21 -0
  3. data/lib/tilework/backend/cpu/dsl_codegen.rb +509 -0
  4. data/lib/tilework/backend/cpu/ops_codegen.rb +451 -0
  5. data/lib/tilework/backend/cpu.rb +156 -0
  6. data/lib/tilework/buffer.rb +753 -0
  7. data/lib/tilework/capabilities.rb +183 -0
  8. data/lib/tilework/compat/cuda.h +66 -0
  9. data/lib/tilework/compat/hip.h +35 -0
  10. data/lib/tilework/compat/metal.h +45 -0
  11. data/lib/tilework/conformance.rb +2296 -0
  12. data/lib/tilework/core_ext.rb +21 -0
  13. data/lib/tilework/dialect.rb +67 -0
  14. data/lib/tilework/dsl.rb +722 -0
  15. data/lib/tilework/dtype/bfloat.rb +31 -0
  16. data/lib/tilework/dtype/cast.rb +88 -0
  17. data/lib/tilework/dtype/half.rb +98 -0
  18. data/lib/tilework/graph/builder.rb +1073 -0
  19. data/lib/tilework/graph/fusion.rb +520 -0
  20. data/lib/tilework/graph/grad.rb +443 -0
  21. data/lib/tilework/graph/high_level.rb +266 -0
  22. data/lib/tilework/graph/kernels.rb +403 -0
  23. data/lib/tilework/graph/node.rb +27 -0
  24. data/lib/tilework/graph/op.rb +290 -0
  25. data/lib/tilework/graph/optim.rb +183 -0
  26. data/lib/tilework/graph/plan.rb +1086 -0
  27. data/lib/tilework/graph/pool.rb +178 -0
  28. data/lib/tilework/graph/region.rb +210 -0
  29. data/lib/tilework/graph/residency.rb +449 -0
  30. data/lib/tilework/graph/state.rb +64 -0
  31. data/lib/tilework/graph/tensor.rb +124 -0
  32. data/lib/tilework/graph.rb +213 -0
  33. data/lib/tilework/kernel.rb +163 -0
  34. data/lib/tilework/math/conv.rb +270 -0
  35. data/lib/tilework/math/fft.rb +172 -0
  36. data/lib/tilework/math/nn.rb +1103 -0
  37. data/lib/tilework/math/philox.rb +194 -0
  38. data/lib/tilework/math/random.rb +232 -0
  39. data/lib/tilework/math/tensor_core.rb +269 -0
  40. data/lib/tilework/ops.rb +682 -0
  41. data/lib/tilework/runtime/offload.rb +498 -0
  42. data/lib/tilework/runtime/submission.rb +163 -0
  43. data/lib/tilework/runtime/telemetry.rb +112 -0
  44. data/lib/tilework/scope.rb +33 -0
  45. data/lib/tilework/version.rb +5 -0
  46. data/lib/tilework.rb +345 -0
  47. metadata +92 -0
@@ -0,0 +1,194 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Tilework
4
+ # Portable random number generator (Philox 4x32-10).
5
+ #
6
+ # Native RNGs (cuRAND / MPSMatrixRandomPhilox) may both be called "Philox",
7
+ # but counter/key layout and the normal transform are implementation-
8
+ # specific, so **the same seed gives different sequences across backends**.
9
+ # This is the single definition shared by every backend that closes that gap.
10
+ #
11
+ # It uses only plain integer arithmetic, so **the same formula** can be
12
+ # written both as a dialect kernel (within the compat macros) and in plain
13
+ # Ruby. Being counter-based it is deterministic in parallel — element i
14
+ # depends on no other element, and changing the request size does not move
15
+ # the value at position i.
16
+ #
17
+ # The high half of a 32x32 -> 64-bit multiply is built from 16-bit halves
18
+ # (so it works with today's int32-only dtype set; this asks less of a backend
19
+ # than adding 64-bit integers to the compat macros).
20
+ #
21
+ # ## How closely it matches
22
+ #
23
+ # - `uniform` is **bit-identical**. Integer-to-float conversion uses only 24
24
+ # bits plus powers of two, so neither GPU float nor Ruby double arithmetic
25
+ # rounds.
26
+ # - `normal` goes through Box-Muller's sqrt/log/cos and matches within their
27
+ # precision (about 1e-6 relative). Bit identity is not guaranteed — the last
28
+ # ulp of transcendentals depends on the backend's math library.
29
+ module Philox
30
+ ROUNDS = 10
31
+ M0 = 0xD2511F53
32
+ M1 = 0xCD9E8D57
33
+ W0 = 0x9E3779B9 # golden ratio
34
+ W1 = 0xBB67AE85 # sqrt(3) - 1
35
+ MASK = 0xFFFFFFFF
36
+
37
+ module_function
38
+
39
+ # Four 32-bit randoms from counter n / key (the low and high 32 bits of the
40
+ # seed). Keep this 1:1 with the expression kernel_source generates below.
41
+ def bijection(counter, key_lo, key_hi)
42
+ c0 = counter & MASK
43
+ c1 = 0
44
+ c2 = 0
45
+ c3 = 0
46
+ k0 = key_lo & MASK
47
+ k1 = key_hi & MASK
48
+
49
+ ROUNDS.times do
50
+ hi0, lo0 = mulhilo(M0, c0)
51
+ hi1, lo1 = mulhilo(M1, c2)
52
+ c0, c1, c2, c3 = (hi1 ^ c1 ^ k0) & MASK, lo1, (hi0 ^ c3 ^ k1) & MASK, lo0
53
+ k0 = (k0 + W0) & MASK
54
+ k1 = (k1 + W1) & MASK
55
+ end
56
+ [c0, c1, c2, c3]
57
+ end
58
+
59
+ # 32x32 -> [high 32 bits, low 32 bits]. Long multiplication in 16-bit
60
+ # halves gets the high half using only 32-bit products.
61
+ def mulhilo(a, b)
62
+ a0 = a & 0xFFFF
63
+ a1 = (a >> 16) & 0xFFFF
64
+ b0 = b & 0xFFFF
65
+ b1 = (b >> 16) & 0xFFFF
66
+
67
+ t = (a0 * b0) & MASK
68
+ k = (t >> 16) & 0xFFFF
69
+ t = ((a1 * b0) + k) & MASK
70
+ w1 = t & 0xFFFF
71
+ w2 = (t >> 16) & 0xFFFF
72
+ t = ((a0 * b1) + w1) & MASK
73
+ hi = ((a1 * b1) + w2 + ((t >> 16) & 0xFFFF)) & MASK
74
+ [hi, (a * b) & MASK]
75
+ end
76
+
77
+ # 32-bit random -> (0, 1]. Only the top 24 bits are used, so neither float
78
+ # nor double rounds and GPU and Ruby agree bit for bit.
79
+ def to_unit(x) = (((x >> 8) & 0xFFFFFF) + 1) * (1.0 / (1 << 24))
80
+
81
+ # Uniform element i. One bijection serves four elements.
82
+ def uniform_at(index, key_lo, key_hi)
83
+ to_unit(bijection(index / 4, key_lo, key_hi)[index % 4])
84
+ end
85
+
86
+ # Normal element i. Box-Muller makes one value from two uniforms, so one
87
+ # bijection serves two elements.
88
+ def normal_at(index, key_lo, key_hi, mean, stddev)
89
+ out = bijection(index / 2, key_lo, key_hi)
90
+ pair = (index % 2) * 2
91
+ u1 = to_unit(out[pair])
92
+ u2 = to_unit(out[pair + 1])
93
+ mean + (stddev * Math.sqrt(-2.0 * Math.log(u1)) * Math.cos(2 * Math::PI * u2))
94
+ end
95
+
96
+ def host_values(length, kind, key_lo, key_hi, mean, stddev)
97
+ if kind == Random::UNIFORM
98
+ Array.new(length) { |i| uniform_at(i, key_lo, key_hi) }
99
+ else
100
+ Array.new(length) { |i| normal_at(i, key_lo, key_hi, mean, stddev) }
101
+ end
102
+ end
103
+
104
+ # seed (64-bit) -> two 32-bit kernel arguments. Dispatch packs scalars as
105
+ # signed 32-bit (kernel.rb#convert_args), so convert to values that fit and
106
+ # reinterpret them as GPU_UINT in the kernel.
107
+ def key_parts(seed)
108
+ seed &= 0xFFFFFFFFFFFFFFFF
109
+ [seed & MASK, (seed >> 32) & MASK]
110
+ end
111
+
112
+ def signed32(value) = value >= 0x80000000 ? value - 0x100000000 : value
113
+
114
+ # The dialect kernel; its expressions are 1:1 with the Ruby implementation above.
115
+ #
116
+ # Helpers exchange scalars only — MSL address-space qualifiers
117
+ # (`thread T*`) are not in the dialect's macro set and have no CUDA
118
+ # counterpart, so the four result words stay in locals c0..c3 in the kernel body.
119
+ def kernel_source(kind)
120
+ uniform = kind == Random::UNIFORM
121
+ <<~KERNEL
122
+ #{HELPERS}
123
+ GPU_KERNEL void philox_fill(GPU_ARGS,
124
+ device float* out,
125
+ constant int& key_lo,
126
+ constant int& key_hi,
127
+ constant float& mean,
128
+ constant float& stddev,
129
+ constant int& n) {
130
+ int i = GLOBAL_ID_X;
131
+ if (i < n) {
132
+ GPU_UINT k0 = (GPU_UINT)key_lo;
133
+ GPU_UINT k1 = (GPU_UINT)key_hi;
134
+ GPU_UINT c0 = (GPU_UINT)(i / #{uniform ? 4 : 2}), c1 = 0u, c2 = 0u, c3 = 0u;
135
+ #{ROUND_LOOP}
136
+ #{uniform ? UNIFORM_BODY : NORMAL_BODY}
137
+ }
138
+ }
139
+ KERNEL
140
+ end
141
+
142
+ HELPERS = <<~KERNEL
143
+ // High half of 32x32 -> 64 bits from 16-bit halves (same formula as philox.rb#mulhilo)
144
+ GPU_FUNC GPU_UINT philox_mulhi(GPU_UINT a, GPU_UINT b) {
145
+ GPU_UINT a0 = a & 0xFFFFu, a1 = a >> 16;
146
+ GPU_UINT b0 = b & 0xFFFFu, b1 = b >> 16;
147
+ GPU_UINT t = a0 * b0;
148
+ GPU_UINT k = t >> 16;
149
+ t = a1 * b0 + k;
150
+ GPU_UINT w1 = t & 0xFFFFu;
151
+ GPU_UINT w2 = t >> 16;
152
+ t = a0 * b1 + w1;
153
+ return a1 * b1 + w2 + (t >> 16);
154
+ }
155
+
156
+ // 32-bit random -> (0, 1]. Only the top 24 bits are used, so float does not round.
157
+ GPU_FUNC float philox_unit(GPU_UINT x) {
158
+ return (float)((x >> 8) + 1u) * (1.0f / 16777216.0f);
159
+ }
160
+ KERNEL
161
+
162
+ # Philox 4x32-10's ten rounds (same formula as philox.rb#bijection).
163
+ ROUND_LOOP = <<~KERNEL
164
+ for (int r = 0; r < #{ROUNDS}; r++) {
165
+ GPU_UINT hi0 = philox_mulhi(#{format('0x%08Xu', M0)}, c0);
166
+ GPU_UINT lo0 = #{format('0x%08Xu', M0)} * c0;
167
+ GPU_UINT hi1 = philox_mulhi(#{format('0x%08Xu', M1)}, c2);
168
+ GPU_UINT lo1 = #{format('0x%08Xu', M1)} * c2;
169
+ GPU_UINT n0 = hi1 ^ c1 ^ k0;
170
+ GPU_UINT n2 = hi0 ^ c3 ^ k1;
171
+ c0 = n0; c1 = lo1; c2 = n2; c3 = lo0;
172
+ k0 = k0 + #{format('0x%08Xu', W0)};
173
+ k1 = k1 + #{format('0x%08Xu', W1)};
174
+ }
175
+ KERNEL
176
+
177
+ UNIFORM_BODY = <<~KERNEL
178
+ int j = i % 4;
179
+ GPU_UINT w = j == 0 ? c0 : (j == 1 ? c1 : (j == 2 ? c2 : c3));
180
+ out[i] = philox_unit(w);
181
+ KERNEL
182
+
183
+ # Box-Muller makes one value from two uniforms, so one call serves two
184
+ # elements (even ones use c0/c1, odd ones c2/c3 — the same assignment as
185
+ # philox.rb#normal_at).
186
+ NORMAL_BODY = <<~KERNEL
187
+ GPU_UINT wa = (i % 2) == 0 ? c0 : c2;
188
+ GPU_UINT wb = (i % 2) == 0 ? c1 : c3;
189
+ float u1 = philox_unit(wa);
190
+ float u2 = philox_unit(wb);
191
+ out[i] = mean + stddev * sqrt(-2.0f * log(u1)) * cos(6.28318530717958648f * u2);
192
+ KERNEL
193
+ end
194
+ end
@@ -0,0 +1,232 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Tilework
4
+ # Random number generation for Tilework buffers.
5
+ #
6
+ # CUDA delegates to cuRAND (Philox, counter-based). Other backends generate
7
+ # with Ruby's Random and transfer — slower, but the result has the same
8
+ # shape, so examples and tests work on any backend (the same policy as the
9
+ # optional acceleration ABI).
10
+ #
11
+ # Generating on the host costs hundreds of ms of Ruby plus the transfer for a
12
+ # million elements. Generating on the GPU stays on the device, which changes
13
+ # the order of magnitude for large uses like weight initialization or Monte
14
+ # Carlo.
15
+ #
16
+ # ## Reproducibility
17
+ #
18
+ # The default (`mode == :native`) guarantees reproducibility **within one
19
+ # process and one backend** only. cuRAND, MPSMatrixRandomPhilox and Ruby's
20
+ # Random return different sequences for the same seed (counter/key layout
21
+ # and the normal transform are implementation-specific, so sharing the name
22
+ # "Philox" does not make them agree).
23
+ #
24
+ # For the same sequence across backends, switch to the portable path:
25
+ #
26
+ # Tilework::Random.portable! # same sequence on :cpu / :metal / :cuda
27
+ # Tilework::Random.seed!(42)
28
+ # Tilework.randn([64, 64])
29
+ #
30
+ # The portable path uses Tilework::Philox (a dialect kernel plus a Ruby
31
+ # implementation of the same formula). `random` is bit-identical and `randn`
32
+ # agrees within Box-Muller's transcendental precision (about 1e-6 relative).
33
+ # It can be slower than the native RNG, so it is not the default — the same
34
+ # idea as the optional ABI: use the fast path when there is one, align only
35
+ # when needed.
36
+ module Random
37
+ UNIFORM = 0
38
+ NORMAL = 1
39
+ BLOCK = 256
40
+ MODES = %i[native portable].freeze
41
+
42
+ class << self
43
+ # @return [:native, :portable] current generation mode
44
+ def mode = @mode || :native
45
+
46
+ # Selects native or cross-backend portable generation.
47
+ #
48
+ # @param value [:native, :portable]
49
+ # @return [:native, :portable]
50
+ # @raise [ArgumentError] for an unsupported mode
51
+ def mode=(value)
52
+ value = value.to_sym
53
+ raise ArgumentError, "unknown random mode #{value.inspect} (expected #{MODES.join(' or ')})" unless MODES.include?(value)
54
+
55
+ @mode = value
56
+ end
57
+
58
+ # Enables deterministic portable Philox generation.
59
+ # @return [:portable]
60
+ def portable! = self.mode = :portable
61
+ # Enables the fastest backend-native generator when available.
62
+ # @return [:native]
63
+ def native! = self.mode = :native
64
+ # @return [Boolean] whether portable generation is enabled
65
+ def portable? = mode == :portable
66
+
67
+ # Without an explicit seed each call returns a different sequence;
68
+ # seed: makes it deterministic (same seed, same sequence).
69
+ attr_reader :default_seed
70
+
71
+ # Sets the process-wide seed and resets the draw counter.
72
+ #
73
+ # @param value [Integer, nil] seed; `nil` restores nondeterministic defaults
74
+ # @return [Integer, nil] the configured seed
75
+ #
76
+ # Makes the whole process deterministic. It resets **the call counter as
77
+ # well as the seed** — a leftover counter would continue the sequence
78
+ # even after setting the same default_seed, which is not reproduction.
79
+ #
80
+ # Tilework::Random.seed!(42)
81
+ # a = Tilework.random(100).to_a
82
+ # Tilework::Random.seed!(42)
83
+ # a == Tilework.random(100).to_a # => true
84
+ def seed!(value)
85
+ @default_seed = value
86
+ @counter = 0
87
+ value
88
+ end
89
+
90
+ def native?
91
+ return false unless Tilework.backend
92
+ return false unless Tilework.bridge.respond_to?(:random_available?)
93
+
94
+ Tilework.bridge.random_available?
95
+ end
96
+
97
+ def library
98
+ return nil unless native?
99
+ return Tilework.bridge.random_library_name if Tilework.bridge.respond_to?(:random_library_name)
100
+
101
+ :curand
102
+ end
103
+
104
+ def next_seed
105
+ @counter = (@counter || 0) + 1
106
+ (default_seed || ::Random.new_seed) ^ (@counter * 0x9E3779B97F4A7C15)
107
+ end
108
+ end
109
+
110
+ module_function
111
+
112
+ # Fills a float32 buffer with uniformly distributed values in `(0, 1]`.
113
+ #
114
+ # @param buffer [Buffer]
115
+ # @param seed [Integer, nil]
116
+ # @return [Buffer] the supplied buffer
117
+ #
118
+ # That is how cuRAND's curandGenerateUniform is defined, so the host
119
+ # fallback matches it (Ruby's rand is [0, 1), the opposite ends).
120
+ def uniform!(buffer, seed: nil)
121
+ fill(buffer, UNIFORM, seed, 0.0, 1.0)
122
+ end
123
+
124
+ # Fills a float32 buffer with normally distributed values.
125
+ #
126
+ # @param buffer [Buffer]
127
+ # @param mean [Float]
128
+ # @param stddev [Float]
129
+ # @param seed [Integer, nil]
130
+ # @return [Buffer] the supplied buffer
131
+ def normal!(buffer, mean: 0.0, stddev: 1.0, seed: nil)
132
+ fill(buffer, NORMAL, seed, mean, stddev)
133
+ end
134
+
135
+ def fill(buffer, kind, seed, mean, stddev)
136
+ unless buffer.dtype == :float32
137
+ raise Error, "random generation only supports :float32 buffers (got #{buffer.dtype.inspect})"
138
+ end
139
+
140
+ seed ||= Random.next_seed
141
+ if Random.portable?
142
+ portable_fill(buffer, kind, seed, mean, stddev)
143
+ elsif Random.native?
144
+ Tilework.bridge.random(buffer.__handle__, buffer.length, kind, seed & 0xFFFFFFFFFFFFFFFF,
145
+ mean, stddev)
146
+ else
147
+ buffer.write(host_values(buffer.length, kind, seed, mean, stddev))
148
+ end
149
+ buffer
150
+ end
151
+
152
+ # The portable path. Backends that accept the dialect run it on the device;
153
+ # others (:cpu) fill it with a Ruby implementation of **the same formula**.
154
+ # Both give the same sequence (see Philox's comment).
155
+ def portable_fill(buffer, kind, seed, mean, stddev)
156
+ key_lo, key_hi = Philox.key_parts(seed)
157
+ unless Tilework.capabilities[:dialect]
158
+ return buffer.write(Philox.host_values(buffer.length, kind, key_lo, key_hi, mean, stddev))
159
+ end
160
+
161
+ kernel = Tilework.compile(Philox.kernel_source(kind), name: "philox_fill")
162
+ Ops.launch_kernel(kernel, grid: (buffer.length / BLOCK.to_f).ceil, block: BLOCK,
163
+ args: [buffer, Philox.signed32(key_lo), Philox.signed32(key_hi),
164
+ mean.to_f, stddev.to_f, buffer.length])
165
+ buffer
166
+ end
167
+
168
+ # Fallback for backends without a native path. It does not match cuRAND (a
169
+ # different algorithm altogether) but stays reproducible for the same seed.
170
+ def host_values(length, kind, seed, mean, stddev)
171
+ rng = ::Random.new(seed & 0x7FFFFFFF)
172
+ if kind == UNIFORM
173
+ # (0, 1] to match cuRAND; Ruby's rand is [0, 1)
174
+ Array.new(length) { 1.0 - rng.rand }
175
+ else
176
+ # Box-Muller, the same idea as cuRAND's normal generator
177
+ Array.new(length) do
178
+ u1 = 1.0 - rng.rand # avoid log(0)
179
+ u2 = rng.rand
180
+ mean + stddev * Math.sqrt(-2.0 * Math.log(u1)) * Math.cos(2 * Math::PI * u2)
181
+ end
182
+ end
183
+ end
184
+ end
185
+
186
+ class << self
187
+ # Allocates a float32 buffer of uniformly distributed values in `(0, 1]`.
188
+ #
189
+ # @param shape [Integer, Array<Integer>] positive dimensions
190
+ # @param seed [Integer, nil]
191
+ # @return [Buffer]
192
+ #
193
+ # Tilework.random(1000) # => a Buffer of length 1000
194
+ # Tilework.random([256, 128]) # => a buffer with a 2D shape
195
+ # Tilework.random(1000, seed: 42) # deterministic
196
+ def random(shape, seed: nil)
197
+ fill_new_buffer(shape) { |buffer| Random.uniform!(buffer, seed: seed) }
198
+ end
199
+
200
+ # Allocates a float32 buffer of normally distributed values.
201
+ #
202
+ # @param shape [Integer, Array<Integer>] positive dimensions
203
+ # @param mean [Float]
204
+ # @param stddev [Float]
205
+ # @param seed [Integer, nil]
206
+ # @return [Buffer]
207
+ #
208
+ # Tilework.randn([out, inp]) * Math.sqrt(2.0 / inp) # He initialization
209
+ def randn(shape, mean: 0.0, stddev: 1.0, seed: nil)
210
+ fill_new_buffer(shape) { |buffer| Random.normal!(buffer, mean: mean, stddev: stddev, seed: seed) }
211
+ end
212
+
213
+ private
214
+
215
+ def fill_new_buffer(shape)
216
+ buffer = new_random_buffer(shape)
217
+ yield buffer
218
+ rescue StandardError
219
+ buffer&.free
220
+ raise
221
+ end
222
+
223
+ def new_random_buffer(shape)
224
+ dims = shape.is_a?(Array) ? shape : [shape]
225
+ unless dims.all? { |d| d.is_a?(Integer) && d.positive? }
226
+ raise ArgumentError, "shape must be a positive Integer or an Array of them, got #{shape.inspect}"
227
+ end
228
+
229
+ Buffer.new(:float32, dims.reduce(:*), shape: dims)
230
+ end
231
+ end
232
+ end
@@ -0,0 +1,269 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Tilework
4
+ # The Tensor Core (nvcuda::wmma) path.
5
+ #
6
+ # Policy:
7
+ # - Use the fragment API (nvcuda::wmma). Hand-written mma.sync / wgmma PTX
8
+ # and CUTLASS belong behind the raw: escape hatch, not here.
9
+ # - Start with TF32: FP32 buffers go in as is, so Tensor Cores are used
10
+ # without touching Buffer::DTYPES.
11
+ #
12
+ # Callers do not decide whether to take this path; eligible? judges from
13
+ # Tilework.capabilities and the matrix shapes. Otherwise it falls through to
14
+ # the existing SIMT kernel, so users see only a speed difference.
15
+ module TensorCore
16
+ # The TF32 fragment shape. K is 8, unlike FP16's 16, so the remainder
17
+ # conditions depend on the dtype.
18
+ FRAG_M = 16
19
+ FRAG_N = 16
20
+ FRAG_K = 8
21
+
22
+ # Block tile: 4 warps (128 threads) per block in a 2x2 layout, each warp
23
+ # owning 32x32 (2x2 of 16x16 fragments) → 64x64 per block. K loads two
24
+ # fragments' worth into shared memory at a time.
25
+ #
26
+ # One tile per warp with no shared-memory reuse is dominated by global
27
+ # reads and loses to SIMT (measured 0.91x); reuse is what makes Tensor
28
+ # Cores pay.
29
+ BLOCK_M = 64
30
+ BLOCK_N = 64
31
+ BLOCK_K = 16
32
+ WARPS_M = 2
33
+ WARPS_N = 2
34
+ WARP = 32
35
+ THREADS = WARPS_M * WARPS_N * WARP # 128
36
+
37
+ class << self
38
+ # A switch to turn the path off explicitly (for performance comparisons or
39
+ # bisecting). On by default; eligible? decides from shapes and capabilities.
40
+ attr_writer :enabled
41
+
42
+ def enabled?
43
+ return false if ENV["GPU_DISABLE_TENSOR_CORE"] == "1"
44
+
45
+ @enabled.nil? ? true : @enabled
46
+ end
47
+ end
48
+
49
+ module_function
50
+
51
+ # Whether GEMM can be delegated to cuBLAS. It is an order of magnitude
52
+ # faster than our WMMA kernel (measured 11.7x) and has **no shape
53
+ # constraints** — small models that miss WMMA's multiple-of-64/16 rule
54
+ # still reach the tensor cores.
55
+ def gemm_library?
56
+ return false unless Tilework.bridge.respond_to?(:gemm_available?)
57
+
58
+ Tilework.bridge.gemm_available?
59
+ end
60
+
61
+ def gemm_library_name
62
+ return Tilework.bridge.gemm_library_name if Tilework.bridge.respond_to?(:gemm_library_name)
63
+
64
+ :cublas
65
+ end
66
+
67
+ # Input dtypes the cuBLAS path accepts and their codes for the bridge.
68
+ # **The output is always float32.**
69
+ GEMM_INPUT_CODES = { float32: 0, float16: 1, bfloat16: 2 }.freeze
70
+
71
+ # When the cuBLAS path applies: any shape (no tile-boundary constraint
72
+ # unlike our WMMA), as long as both sides share a dtype cuBLAS handles.
73
+ def gemm_eligible?(a, b, _dtype)
74
+ return false unless gemm_library?
75
+ return false unless a.dtype == b.dtype
76
+ supported = Tilework.capabilities.gemm_dtypes
77
+
78
+ GEMM_INPUT_CODES.key?(a.dtype) && supported.include?(a.dtype)
79
+ end
80
+
81
+ # Backends with an executable matrix-instruction probe (rocWMMA on HIP)
82
+ # own their native source and supported tile contract. Core only asks the
83
+ # bridge whether this dtype/shape is eligible; it never recognizes a GPU,
84
+ # an ISA name, or a backend name.
85
+ def matrix_core_eligible?(a, b, dtype)
86
+ return false unless TensorCore.enabled?
87
+ return false unless Tilework.bridge.respond_to?(:matrix_core_available?) && Tilework.bridge.matrix_core_available?
88
+ return false unless dtype == :float32 && a.dtype == :float16 && b.dtype == :float16
89
+
90
+ a.rows.positive? && a.rows % 16 == 0 && a.cols % 16 == 0 && b.cols % 16 == 0
91
+ end
92
+
93
+ def matrix_core_gemm(a, b, m, k, n)
94
+ out = Ops.allocate_output(:float32, m * n, [m, n])
95
+ Tilework.bridge.matrix_core_gemm(a.__handle__, b.__handle__, out.__handle__, m, k, n)
96
+ out
97
+ rescue StandardError
98
+ out&.free
99
+ raise
100
+ end
101
+
102
+ # C = A * B through cuBLAS. Whether TF32 is allowed follows enabled? (off
103
+ # means FP32, keeping relative error around 1e-7). fp16 input still
104
+ # accumulates in FP32 and outputs float32. The output is allocated through
105
+ # the context: a graph plan **returns intermediates to the pool instead of
106
+ # freeing them**, so an allocation bypassing the pool would add one slot per
107
+ # run (which matters in decode loops). GEMM writes all of C (beta = 0), so a
108
+ # reused, non-zeroed slot leaves no residue. Eager has no context, so it is
109
+ # the same as Buffer.new.
110
+ def gemm(a, b, m, k, n)
111
+ out = Ops.allocate_output(:float32, m * n, [m, n])
112
+ Tilework.bridge.gemm(a.__handle__, b.__handle__, out.__handle__, m, k, n,
113
+ enabled?, GEMM_INPUT_CODES.fetch(a.dtype))
114
+ out
115
+ rescue StandardError
116
+ out&.free
117
+ raise
118
+ end
119
+
120
+ # Whether `batch` GEMMs of the same shape can be issued at once.
121
+ def gemm_batched?
122
+ return false unless Tilework.bridge.respond_to?(:gemm_batched_available?)
123
+
124
+ Tilework.bridge.gemm_batched_available?
125
+ end
126
+
127
+ # A low-level passthrough to cuBLAS's strided batched GEMM.
128
+ #
129
+ # **All arguments are column-major**; nothing is reinterpreted from row-major.
130
+ # Callers who think in row-major keep the reinterpretation on their side, as
131
+ # Tilework::NN.multi_head_attention does. Moving it here would explode into
132
+ # combinations of transpose flags x which matrix.
133
+ #
134
+ # ld_* is the column-major leading dimension. To view part of a larger
135
+ # matrix as one batch element, pass **the parent's width** as ld and step
136
+ # by the stride. Buffer itself has no strides, so this is the only place
137
+ # that information can be expressed.
138
+ #
139
+ # The output is always float32, as with gemm. alpha is a scale GEMM applies
140
+ # for free; attention's 1/sqrt(dh) uses it, saving another round trip over
141
+ # the score matrix.
142
+ def gemm_batched(a, b, out, m:, k:, n:, lda:, ldb:, ldc:,
143
+ stride_a:, stride_b:, stride_c:, batch:,
144
+ trans_a: false, trans_b: false, alpha: 1.0)
145
+ unless a.dtype == b.dtype && GEMM_INPUT_CODES.key?(a.dtype)
146
+ raise ArgumentError,
147
+ "batched GEMM は両辺が同じ対応済み float dtype のときだけ通る: " \
148
+ "#{a.dtype.inspect} と #{b.dtype.inspect}"
149
+ end
150
+ unless Tilework.capabilities.gemm_dtypes.include?(a.dtype)
151
+ raise ArgumentError, "batched GEMM does not support #{a.dtype.inspect} on this backend"
152
+ end
153
+ raise ArgumentError, "出力は float32 でなければならない" unless out.dtype == :float32
154
+
155
+ Tilework.bridge.gemm_batched(a.__handle__, b.__handle__, out.__handle__,
156
+ m, k, n, lda, ldb, ldc,
157
+ stride_a, stride_b, stride_c,
158
+ trans_a, trans_b, batch, alpha.to_f,
159
+ enabled?, GEMM_INPUT_CODES.fetch(a.dtype))
160
+ out
161
+ end
162
+
163
+ # Alignment: Buffers are flat, stride-less linear memory, so shapes off the
164
+ # tile boundary never enter the WMMA path. Handling remainders with
165
+ # in-kernel branches would diverge within a warp, making fragment
166
+ # instructions undefined. Rejected shapes are computed correctly by SIMT.
167
+ def eligible?(a, b, dtype)
168
+ return false unless TensorCore.enabled?
169
+ return false unless dtype == :float32 && a.dtype == :float32 && b.dtype == :float32
170
+ return false unless Tilework.capabilities[:tf32]
171
+
172
+ a.rows % BLOCK_M == 0 && b.cols % BLOCK_N == 0 && a.cols % BLOCK_K == 0
173
+ end
174
+
175
+ # A WMMA GEMM that reuses tiles through shared memory. __syncthreads() and
176
+ # fragment instructions sit only where every thread in the block and every
177
+ # lane in the warp arrive together.
178
+ def matmul_source
179
+ <<~CUDA
180
+ #include <mma.h>
181
+ using namespace nvcuda;
182
+
183
+ #define BM #{BLOCK_M}
184
+ #define BN #{BLOCK_N}
185
+ #define BK #{BLOCK_K}
186
+ #define WM #{WARPS_M}
187
+ #define WN #{WARPS_N}
188
+ #define FM #{FRAG_M}
189
+ #define FN #{FRAG_N}
190
+ #define FK #{FRAG_K}
191
+
192
+ extern "C" __global__ void matmul_tf32(const float* a, const float* b, float* c,
193
+ int m, int k, int n) {
194
+ __shared__ float As[BM][BK];
195
+ __shared__ float Bs[BK][BN];
196
+
197
+ const int tid = threadIdx.x;
198
+ const int warp = tid / #{WARP};
199
+ const int warp_m = warp / WN; // 0..WM-1
200
+ const int warp_n = warp % WN; // 0..WN-1
201
+
202
+ const int row0 = blockIdx.y * BM; // top-left of this block's output
203
+ const int col0 = blockIdx.x * BN;
204
+
205
+ // 2x2 16x16 fragments per warp = 32x32
206
+ wmma::fragment<wmma::accumulator, FM, FN, FK, float> acc[2][2];
207
+ for (int i = 0; i < 2; i++)
208
+ for (int j = 0; j < 2; j++)
209
+ wmma::fill_fragment(acc[i][j], 0.0f);
210
+
211
+ for (int t = 0; t < k; t += BK) {
212
+ // Cooperative load into shared memory (128 threads, BM*BK = BK*BN = 1024 elements)
213
+ for (int i = tid; i < BM * BK; i += #{THREADS}) {
214
+ As[i / BK][i % BK] = a[(row0 + i / BK) * k + t + (i % BK)];
215
+ }
216
+ for (int i = tid; i < BK * BN; i += #{THREADS}) {
217
+ Bs[i / BN][i % BN] = b[(t + i / BN) * n + col0 + (i % BN)];
218
+ }
219
+ __syncthreads();
220
+
221
+ for (int kk = 0; kk < BK; kk += FK) {
222
+ wmma::fragment<wmma::matrix_a, FM, FN, FK, wmma::precision::tf32, wmma::row_major> fa[2];
223
+ wmma::fragment<wmma::matrix_b, FM, FN, FK, wmma::precision::tf32, wmma::row_major> fb[2];
224
+
225
+ for (int i = 0; i < 2; i++) {
226
+ wmma::load_matrix_sync(fa[i], &As[warp_m * 32 + i * FM][kk], BK);
227
+ // Round values read as FP32 to TF32 (10-bit mantissa, same 8-bit
228
+ // exponent as FP32) before multiply-accumulate. The exponent range
229
+ // matches FP32, so there is no FP16-style overflow/underflow.
230
+ for (int e = 0; e < fa[i].num_elements; e++) fa[i].x[e] = wmma::__float_to_tf32(fa[i].x[e]);
231
+ }
232
+ for (int j = 0; j < 2; j++) {
233
+ wmma::load_matrix_sync(fb[j], &Bs[kk][warp_n * 32 + j * FN], BN);
234
+ for (int e = 0; e < fb[j].num_elements; e++) fb[j].x[e] = wmma::__float_to_tf32(fb[j].x[e]);
235
+ }
236
+
237
+ for (int i = 0; i < 2; i++)
238
+ for (int j = 0; j < 2; j++)
239
+ wmma::mma_sync(acc[i][j], fa[i], fb[j], acc[i][j]);
240
+ }
241
+ __syncthreads();
242
+ }
243
+
244
+ for (int i = 0; i < 2; i++) {
245
+ for (int j = 0; j < 2; j++) {
246
+ int r = row0 + warp_m * 32 + i * FM;
247
+ int cc = col0 + warp_n * 32 + j * FN;
248
+ wmma::store_matrix_sync(c + r * n + cc, acc[i][j], n, wmma::mem_row_major);
249
+ }
250
+ }
251
+ }
252
+ CUDA
253
+ end
254
+
255
+ # The WMMA path. Allocated through the context for the same reason as gemm
256
+ # (only tile-aligned shapes get here, so the output is fully written).
257
+ def matmul(a, b, m, k, n)
258
+ out = Ops.allocate_output(:float32, m * n, [m, n])
259
+ # raw: true — plain CUDA including mma.h, so no compat header is prepended.
260
+ kernel = Tilework.compile(matmul_source, name: "matmul_tf32", raw: true)
261
+ Ops.launch_kernel(kernel, grid: [n / BLOCK_N, m / BLOCK_M, 1], block: [THREADS, 1, 1],
262
+ args: [a, b, out, m, k, n])
263
+ out
264
+ rescue StandardError
265
+ out&.free
266
+ raise
267
+ end
268
+ end
269
+ end