tilework 0.18.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE +21 -0
- data/lib/tilework/backend/cpu/dsl_codegen.rb +509 -0
- data/lib/tilework/backend/cpu/ops_codegen.rb +451 -0
- data/lib/tilework/backend/cpu.rb +156 -0
- data/lib/tilework/buffer.rb +753 -0
- data/lib/tilework/capabilities.rb +183 -0
- data/lib/tilework/compat/cuda.h +66 -0
- data/lib/tilework/compat/hip.h +35 -0
- data/lib/tilework/compat/metal.h +45 -0
- data/lib/tilework/conformance.rb +2296 -0
- data/lib/tilework/core_ext.rb +21 -0
- data/lib/tilework/dialect.rb +67 -0
- data/lib/tilework/dsl.rb +722 -0
- data/lib/tilework/dtype/bfloat.rb +31 -0
- data/lib/tilework/dtype/cast.rb +88 -0
- data/lib/tilework/dtype/half.rb +98 -0
- data/lib/tilework/graph/builder.rb +1073 -0
- data/lib/tilework/graph/fusion.rb +520 -0
- data/lib/tilework/graph/grad.rb +443 -0
- data/lib/tilework/graph/high_level.rb +266 -0
- data/lib/tilework/graph/kernels.rb +403 -0
- data/lib/tilework/graph/node.rb +27 -0
- data/lib/tilework/graph/op.rb +290 -0
- data/lib/tilework/graph/optim.rb +183 -0
- data/lib/tilework/graph/plan.rb +1086 -0
- data/lib/tilework/graph/pool.rb +178 -0
- data/lib/tilework/graph/region.rb +210 -0
- data/lib/tilework/graph/residency.rb +449 -0
- data/lib/tilework/graph/state.rb +64 -0
- data/lib/tilework/graph/tensor.rb +124 -0
- data/lib/tilework/graph.rb +213 -0
- data/lib/tilework/kernel.rb +163 -0
- data/lib/tilework/math/conv.rb +270 -0
- data/lib/tilework/math/fft.rb +172 -0
- data/lib/tilework/math/nn.rb +1103 -0
- data/lib/tilework/math/philox.rb +194 -0
- data/lib/tilework/math/random.rb +232 -0
- data/lib/tilework/math/tensor_core.rb +269 -0
- data/lib/tilework/ops.rb +682 -0
- data/lib/tilework/runtime/offload.rb +498 -0
- data/lib/tilework/runtime/submission.rb +163 -0
- data/lib/tilework/runtime/telemetry.rb +112 -0
- data/lib/tilework/scope.rb +33 -0
- data/lib/tilework/version.rb +5 -0
- data/lib/tilework.rb +345 -0
- metadata +92 -0
|
@@ -0,0 +1,194 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Tilework
|
|
4
|
+
# Portable random number generator (Philox 4x32-10).
|
|
5
|
+
#
|
|
6
|
+
# Native RNGs (cuRAND / MPSMatrixRandomPhilox) may both be called "Philox",
|
|
7
|
+
# but counter/key layout and the normal transform are implementation-
|
|
8
|
+
# specific, so **the same seed gives different sequences across backends**.
|
|
9
|
+
# This is the single definition shared by every backend that closes that gap.
|
|
10
|
+
#
|
|
11
|
+
# It uses only plain integer arithmetic, so **the same formula** can be
|
|
12
|
+
# written both as a dialect kernel (within the compat macros) and in plain
|
|
13
|
+
# Ruby. Being counter-based it is deterministic in parallel — element i
|
|
14
|
+
# depends on no other element, and changing the request size does not move
|
|
15
|
+
# the value at position i.
|
|
16
|
+
#
|
|
17
|
+
# The high half of a 32x32 -> 64-bit multiply is built from 16-bit halves
|
|
18
|
+
# (so it works with today's int32-only dtype set; this asks less of a backend
|
|
19
|
+
# than adding 64-bit integers to the compat macros).
|
|
20
|
+
#
|
|
21
|
+
# ## How closely it matches
|
|
22
|
+
#
|
|
23
|
+
# - `uniform` is **bit-identical**. Integer-to-float conversion uses only 24
|
|
24
|
+
# bits plus powers of two, so neither GPU float nor Ruby double arithmetic
|
|
25
|
+
# rounds.
|
|
26
|
+
# - `normal` goes through Box-Muller's sqrt/log/cos and matches within their
|
|
27
|
+
# precision (about 1e-6 relative). Bit identity is not guaranteed — the last
|
|
28
|
+
# ulp of transcendentals depends on the backend's math library.
|
|
29
|
+
module Philox
|
|
30
|
+
ROUNDS = 10
|
|
31
|
+
M0 = 0xD2511F53
|
|
32
|
+
M1 = 0xCD9E8D57
|
|
33
|
+
W0 = 0x9E3779B9 # golden ratio
|
|
34
|
+
W1 = 0xBB67AE85 # sqrt(3) - 1
|
|
35
|
+
MASK = 0xFFFFFFFF
|
|
36
|
+
|
|
37
|
+
module_function
|
|
38
|
+
|
|
39
|
+
# Four 32-bit randoms from counter n / key (the low and high 32 bits of the
|
|
40
|
+
# seed). Keep this 1:1 with the expression kernel_source generates below.
|
|
41
|
+
def bijection(counter, key_lo, key_hi)
|
|
42
|
+
c0 = counter & MASK
|
|
43
|
+
c1 = 0
|
|
44
|
+
c2 = 0
|
|
45
|
+
c3 = 0
|
|
46
|
+
k0 = key_lo & MASK
|
|
47
|
+
k1 = key_hi & MASK
|
|
48
|
+
|
|
49
|
+
ROUNDS.times do
|
|
50
|
+
hi0, lo0 = mulhilo(M0, c0)
|
|
51
|
+
hi1, lo1 = mulhilo(M1, c2)
|
|
52
|
+
c0, c1, c2, c3 = (hi1 ^ c1 ^ k0) & MASK, lo1, (hi0 ^ c3 ^ k1) & MASK, lo0
|
|
53
|
+
k0 = (k0 + W0) & MASK
|
|
54
|
+
k1 = (k1 + W1) & MASK
|
|
55
|
+
end
|
|
56
|
+
[c0, c1, c2, c3]
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
# 32x32 -> [high 32 bits, low 32 bits]. Long multiplication in 16-bit
|
|
60
|
+
# halves gets the high half using only 32-bit products.
|
|
61
|
+
def mulhilo(a, b)
|
|
62
|
+
a0 = a & 0xFFFF
|
|
63
|
+
a1 = (a >> 16) & 0xFFFF
|
|
64
|
+
b0 = b & 0xFFFF
|
|
65
|
+
b1 = (b >> 16) & 0xFFFF
|
|
66
|
+
|
|
67
|
+
t = (a0 * b0) & MASK
|
|
68
|
+
k = (t >> 16) & 0xFFFF
|
|
69
|
+
t = ((a1 * b0) + k) & MASK
|
|
70
|
+
w1 = t & 0xFFFF
|
|
71
|
+
w2 = (t >> 16) & 0xFFFF
|
|
72
|
+
t = ((a0 * b1) + w1) & MASK
|
|
73
|
+
hi = ((a1 * b1) + w2 + ((t >> 16) & 0xFFFF)) & MASK
|
|
74
|
+
[hi, (a * b) & MASK]
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
# 32-bit random -> (0, 1]. Only the top 24 bits are used, so neither float
|
|
78
|
+
# nor double rounds and GPU and Ruby agree bit for bit.
|
|
79
|
+
def to_unit(x) = (((x >> 8) & 0xFFFFFF) + 1) * (1.0 / (1 << 24))
|
|
80
|
+
|
|
81
|
+
# Uniform element i. One bijection serves four elements.
|
|
82
|
+
def uniform_at(index, key_lo, key_hi)
|
|
83
|
+
to_unit(bijection(index / 4, key_lo, key_hi)[index % 4])
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
# Normal element i. Box-Muller makes one value from two uniforms, so one
|
|
87
|
+
# bijection serves two elements.
|
|
88
|
+
def normal_at(index, key_lo, key_hi, mean, stddev)
|
|
89
|
+
out = bijection(index / 2, key_lo, key_hi)
|
|
90
|
+
pair = (index % 2) * 2
|
|
91
|
+
u1 = to_unit(out[pair])
|
|
92
|
+
u2 = to_unit(out[pair + 1])
|
|
93
|
+
mean + (stddev * Math.sqrt(-2.0 * Math.log(u1)) * Math.cos(2 * Math::PI * u2))
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
def host_values(length, kind, key_lo, key_hi, mean, stddev)
|
|
97
|
+
if kind == Random::UNIFORM
|
|
98
|
+
Array.new(length) { |i| uniform_at(i, key_lo, key_hi) }
|
|
99
|
+
else
|
|
100
|
+
Array.new(length) { |i| normal_at(i, key_lo, key_hi, mean, stddev) }
|
|
101
|
+
end
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
# seed (64-bit) -> two 32-bit kernel arguments. Dispatch packs scalars as
|
|
105
|
+
# signed 32-bit (kernel.rb#convert_args), so convert to values that fit and
|
|
106
|
+
# reinterpret them as GPU_UINT in the kernel.
|
|
107
|
+
def key_parts(seed)
|
|
108
|
+
seed &= 0xFFFFFFFFFFFFFFFF
|
|
109
|
+
[seed & MASK, (seed >> 32) & MASK]
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
def signed32(value) = value >= 0x80000000 ? value - 0x100000000 : value
|
|
113
|
+
|
|
114
|
+
# The dialect kernel; its expressions are 1:1 with the Ruby implementation above.
|
|
115
|
+
#
|
|
116
|
+
# Helpers exchange scalars only — MSL address-space qualifiers
|
|
117
|
+
# (`thread T*`) are not in the dialect's macro set and have no CUDA
|
|
118
|
+
# counterpart, so the four result words stay in locals c0..c3 in the kernel body.
|
|
119
|
+
def kernel_source(kind)
|
|
120
|
+
uniform = kind == Random::UNIFORM
|
|
121
|
+
<<~KERNEL
|
|
122
|
+
#{HELPERS}
|
|
123
|
+
GPU_KERNEL void philox_fill(GPU_ARGS,
|
|
124
|
+
device float* out,
|
|
125
|
+
constant int& key_lo,
|
|
126
|
+
constant int& key_hi,
|
|
127
|
+
constant float& mean,
|
|
128
|
+
constant float& stddev,
|
|
129
|
+
constant int& n) {
|
|
130
|
+
int i = GLOBAL_ID_X;
|
|
131
|
+
if (i < n) {
|
|
132
|
+
GPU_UINT k0 = (GPU_UINT)key_lo;
|
|
133
|
+
GPU_UINT k1 = (GPU_UINT)key_hi;
|
|
134
|
+
GPU_UINT c0 = (GPU_UINT)(i / #{uniform ? 4 : 2}), c1 = 0u, c2 = 0u, c3 = 0u;
|
|
135
|
+
#{ROUND_LOOP}
|
|
136
|
+
#{uniform ? UNIFORM_BODY : NORMAL_BODY}
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
KERNEL
|
|
140
|
+
end
|
|
141
|
+
|
|
142
|
+
HELPERS = <<~KERNEL
|
|
143
|
+
// High half of 32x32 -> 64 bits from 16-bit halves (same formula as philox.rb#mulhilo)
|
|
144
|
+
GPU_FUNC GPU_UINT philox_mulhi(GPU_UINT a, GPU_UINT b) {
|
|
145
|
+
GPU_UINT a0 = a & 0xFFFFu, a1 = a >> 16;
|
|
146
|
+
GPU_UINT b0 = b & 0xFFFFu, b1 = b >> 16;
|
|
147
|
+
GPU_UINT t = a0 * b0;
|
|
148
|
+
GPU_UINT k = t >> 16;
|
|
149
|
+
t = a1 * b0 + k;
|
|
150
|
+
GPU_UINT w1 = t & 0xFFFFu;
|
|
151
|
+
GPU_UINT w2 = t >> 16;
|
|
152
|
+
t = a0 * b1 + w1;
|
|
153
|
+
return a1 * b1 + w2 + (t >> 16);
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
// 32-bit random -> (0, 1]. Only the top 24 bits are used, so float does not round.
|
|
157
|
+
GPU_FUNC float philox_unit(GPU_UINT x) {
|
|
158
|
+
return (float)((x >> 8) + 1u) * (1.0f / 16777216.0f);
|
|
159
|
+
}
|
|
160
|
+
KERNEL
|
|
161
|
+
|
|
162
|
+
# Philox 4x32-10's ten rounds (same formula as philox.rb#bijection).
|
|
163
|
+
ROUND_LOOP = <<~KERNEL
|
|
164
|
+
for (int r = 0; r < #{ROUNDS}; r++) {
|
|
165
|
+
GPU_UINT hi0 = philox_mulhi(#{format('0x%08Xu', M0)}, c0);
|
|
166
|
+
GPU_UINT lo0 = #{format('0x%08Xu', M0)} * c0;
|
|
167
|
+
GPU_UINT hi1 = philox_mulhi(#{format('0x%08Xu', M1)}, c2);
|
|
168
|
+
GPU_UINT lo1 = #{format('0x%08Xu', M1)} * c2;
|
|
169
|
+
GPU_UINT n0 = hi1 ^ c1 ^ k0;
|
|
170
|
+
GPU_UINT n2 = hi0 ^ c3 ^ k1;
|
|
171
|
+
c0 = n0; c1 = lo1; c2 = n2; c3 = lo0;
|
|
172
|
+
k0 = k0 + #{format('0x%08Xu', W0)};
|
|
173
|
+
k1 = k1 + #{format('0x%08Xu', W1)};
|
|
174
|
+
}
|
|
175
|
+
KERNEL
|
|
176
|
+
|
|
177
|
+
UNIFORM_BODY = <<~KERNEL
|
|
178
|
+
int j = i % 4;
|
|
179
|
+
GPU_UINT w = j == 0 ? c0 : (j == 1 ? c1 : (j == 2 ? c2 : c3));
|
|
180
|
+
out[i] = philox_unit(w);
|
|
181
|
+
KERNEL
|
|
182
|
+
|
|
183
|
+
# Box-Muller makes one value from two uniforms, so one call serves two
|
|
184
|
+
# elements (even ones use c0/c1, odd ones c2/c3 — the same assignment as
|
|
185
|
+
# philox.rb#normal_at).
|
|
186
|
+
NORMAL_BODY = <<~KERNEL
|
|
187
|
+
GPU_UINT wa = (i % 2) == 0 ? c0 : c2;
|
|
188
|
+
GPU_UINT wb = (i % 2) == 0 ? c1 : c3;
|
|
189
|
+
float u1 = philox_unit(wa);
|
|
190
|
+
float u2 = philox_unit(wb);
|
|
191
|
+
out[i] = mean + stddev * sqrt(-2.0f * log(u1)) * cos(6.28318530717958648f * u2);
|
|
192
|
+
KERNEL
|
|
193
|
+
end
|
|
194
|
+
end
|
|
@@ -0,0 +1,232 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Tilework
|
|
4
|
+
# Random number generation for Tilework buffers.
|
|
5
|
+
#
|
|
6
|
+
# CUDA delegates to cuRAND (Philox, counter-based). Other backends generate
|
|
7
|
+
# with Ruby's Random and transfer — slower, but the result has the same
|
|
8
|
+
# shape, so examples and tests work on any backend (the same policy as the
|
|
9
|
+
# optional acceleration ABI).
|
|
10
|
+
#
|
|
11
|
+
# Generating on the host costs hundreds of ms of Ruby plus the transfer for a
|
|
12
|
+
# million elements. Generating on the GPU stays on the device, which changes
|
|
13
|
+
# the order of magnitude for large uses like weight initialization or Monte
|
|
14
|
+
# Carlo.
|
|
15
|
+
#
|
|
16
|
+
# ## Reproducibility
|
|
17
|
+
#
|
|
18
|
+
# The default (`mode == :native`) guarantees reproducibility **within one
|
|
19
|
+
# process and one backend** only. cuRAND, MPSMatrixRandomPhilox and Ruby's
|
|
20
|
+
# Random return different sequences for the same seed (counter/key layout
|
|
21
|
+
# and the normal transform are implementation-specific, so sharing the name
|
|
22
|
+
# "Philox" does not make them agree).
|
|
23
|
+
#
|
|
24
|
+
# For the same sequence across backends, switch to the portable path:
|
|
25
|
+
#
|
|
26
|
+
# Tilework::Random.portable! # same sequence on :cpu / :metal / :cuda
|
|
27
|
+
# Tilework::Random.seed!(42)
|
|
28
|
+
# Tilework.randn([64, 64])
|
|
29
|
+
#
|
|
30
|
+
# The portable path uses Tilework::Philox (a dialect kernel plus a Ruby
|
|
31
|
+
# implementation of the same formula). `random` is bit-identical and `randn`
|
|
32
|
+
# agrees within Box-Muller's transcendental precision (about 1e-6 relative).
|
|
33
|
+
# It can be slower than the native RNG, so it is not the default — the same
|
|
34
|
+
# idea as the optional ABI: use the fast path when there is one, align only
|
|
35
|
+
# when needed.
|
|
36
|
+
module Random
|
|
37
|
+
UNIFORM = 0
|
|
38
|
+
NORMAL = 1
|
|
39
|
+
BLOCK = 256
|
|
40
|
+
MODES = %i[native portable].freeze
|
|
41
|
+
|
|
42
|
+
class << self
|
|
43
|
+
# @return [:native, :portable] current generation mode
|
|
44
|
+
def mode = @mode || :native
|
|
45
|
+
|
|
46
|
+
# Selects native or cross-backend portable generation.
|
|
47
|
+
#
|
|
48
|
+
# @param value [:native, :portable]
|
|
49
|
+
# @return [:native, :portable]
|
|
50
|
+
# @raise [ArgumentError] for an unsupported mode
|
|
51
|
+
def mode=(value)
|
|
52
|
+
value = value.to_sym
|
|
53
|
+
raise ArgumentError, "unknown random mode #{value.inspect} (expected #{MODES.join(' or ')})" unless MODES.include?(value)
|
|
54
|
+
|
|
55
|
+
@mode = value
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
# Enables deterministic portable Philox generation.
|
|
59
|
+
# @return [:portable]
|
|
60
|
+
def portable! = self.mode = :portable
|
|
61
|
+
# Enables the fastest backend-native generator when available.
|
|
62
|
+
# @return [:native]
|
|
63
|
+
def native! = self.mode = :native
|
|
64
|
+
# @return [Boolean] whether portable generation is enabled
|
|
65
|
+
def portable? = mode == :portable
|
|
66
|
+
|
|
67
|
+
# Without an explicit seed each call returns a different sequence;
|
|
68
|
+
# seed: makes it deterministic (same seed, same sequence).
|
|
69
|
+
attr_reader :default_seed
|
|
70
|
+
|
|
71
|
+
# Sets the process-wide seed and resets the draw counter.
|
|
72
|
+
#
|
|
73
|
+
# @param value [Integer, nil] seed; `nil` restores nondeterministic defaults
|
|
74
|
+
# @return [Integer, nil] the configured seed
|
|
75
|
+
#
|
|
76
|
+
# Makes the whole process deterministic. It resets **the call counter as
|
|
77
|
+
# well as the seed** — a leftover counter would continue the sequence
|
|
78
|
+
# even after setting the same default_seed, which is not reproduction.
|
|
79
|
+
#
|
|
80
|
+
# Tilework::Random.seed!(42)
|
|
81
|
+
# a = Tilework.random(100).to_a
|
|
82
|
+
# Tilework::Random.seed!(42)
|
|
83
|
+
# a == Tilework.random(100).to_a # => true
|
|
84
|
+
def seed!(value)
|
|
85
|
+
@default_seed = value
|
|
86
|
+
@counter = 0
|
|
87
|
+
value
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
def native?
|
|
91
|
+
return false unless Tilework.backend
|
|
92
|
+
return false unless Tilework.bridge.respond_to?(:random_available?)
|
|
93
|
+
|
|
94
|
+
Tilework.bridge.random_available?
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
def library
|
|
98
|
+
return nil unless native?
|
|
99
|
+
return Tilework.bridge.random_library_name if Tilework.bridge.respond_to?(:random_library_name)
|
|
100
|
+
|
|
101
|
+
:curand
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
def next_seed
|
|
105
|
+
@counter = (@counter || 0) + 1
|
|
106
|
+
(default_seed || ::Random.new_seed) ^ (@counter * 0x9E3779B97F4A7C15)
|
|
107
|
+
end
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
module_function
|
|
111
|
+
|
|
112
|
+
# Fills a float32 buffer with uniformly distributed values in `(0, 1]`.
|
|
113
|
+
#
|
|
114
|
+
# @param buffer [Buffer]
|
|
115
|
+
# @param seed [Integer, nil]
|
|
116
|
+
# @return [Buffer] the supplied buffer
|
|
117
|
+
#
|
|
118
|
+
# That is how cuRAND's curandGenerateUniform is defined, so the host
|
|
119
|
+
# fallback matches it (Ruby's rand is [0, 1), the opposite ends).
|
|
120
|
+
def uniform!(buffer, seed: nil)
|
|
121
|
+
fill(buffer, UNIFORM, seed, 0.0, 1.0)
|
|
122
|
+
end
|
|
123
|
+
|
|
124
|
+
# Fills a float32 buffer with normally distributed values.
|
|
125
|
+
#
|
|
126
|
+
# @param buffer [Buffer]
|
|
127
|
+
# @param mean [Float]
|
|
128
|
+
# @param stddev [Float]
|
|
129
|
+
# @param seed [Integer, nil]
|
|
130
|
+
# @return [Buffer] the supplied buffer
|
|
131
|
+
def normal!(buffer, mean: 0.0, stddev: 1.0, seed: nil)
|
|
132
|
+
fill(buffer, NORMAL, seed, mean, stddev)
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
def fill(buffer, kind, seed, mean, stddev)
|
|
136
|
+
unless buffer.dtype == :float32
|
|
137
|
+
raise Error, "random generation only supports :float32 buffers (got #{buffer.dtype.inspect})"
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
seed ||= Random.next_seed
|
|
141
|
+
if Random.portable?
|
|
142
|
+
portable_fill(buffer, kind, seed, mean, stddev)
|
|
143
|
+
elsif Random.native?
|
|
144
|
+
Tilework.bridge.random(buffer.__handle__, buffer.length, kind, seed & 0xFFFFFFFFFFFFFFFF,
|
|
145
|
+
mean, stddev)
|
|
146
|
+
else
|
|
147
|
+
buffer.write(host_values(buffer.length, kind, seed, mean, stddev))
|
|
148
|
+
end
|
|
149
|
+
buffer
|
|
150
|
+
end
|
|
151
|
+
|
|
152
|
+
# The portable path. Backends that accept the dialect run it on the device;
|
|
153
|
+
# others (:cpu) fill it with a Ruby implementation of **the same formula**.
|
|
154
|
+
# Both give the same sequence (see Philox's comment).
|
|
155
|
+
def portable_fill(buffer, kind, seed, mean, stddev)
|
|
156
|
+
key_lo, key_hi = Philox.key_parts(seed)
|
|
157
|
+
unless Tilework.capabilities[:dialect]
|
|
158
|
+
return buffer.write(Philox.host_values(buffer.length, kind, key_lo, key_hi, mean, stddev))
|
|
159
|
+
end
|
|
160
|
+
|
|
161
|
+
kernel = Tilework.compile(Philox.kernel_source(kind), name: "philox_fill")
|
|
162
|
+
Ops.launch_kernel(kernel, grid: (buffer.length / BLOCK.to_f).ceil, block: BLOCK,
|
|
163
|
+
args: [buffer, Philox.signed32(key_lo), Philox.signed32(key_hi),
|
|
164
|
+
mean.to_f, stddev.to_f, buffer.length])
|
|
165
|
+
buffer
|
|
166
|
+
end
|
|
167
|
+
|
|
168
|
+
# Fallback for backends without a native path. It does not match cuRAND (a
|
|
169
|
+
# different algorithm altogether) but stays reproducible for the same seed.
|
|
170
|
+
def host_values(length, kind, seed, mean, stddev)
|
|
171
|
+
rng = ::Random.new(seed & 0x7FFFFFFF)
|
|
172
|
+
if kind == UNIFORM
|
|
173
|
+
# (0, 1] to match cuRAND; Ruby's rand is [0, 1)
|
|
174
|
+
Array.new(length) { 1.0 - rng.rand }
|
|
175
|
+
else
|
|
176
|
+
# Box-Muller, the same idea as cuRAND's normal generator
|
|
177
|
+
Array.new(length) do
|
|
178
|
+
u1 = 1.0 - rng.rand # avoid log(0)
|
|
179
|
+
u2 = rng.rand
|
|
180
|
+
mean + stddev * Math.sqrt(-2.0 * Math.log(u1)) * Math.cos(2 * Math::PI * u2)
|
|
181
|
+
end
|
|
182
|
+
end
|
|
183
|
+
end
|
|
184
|
+
end
|
|
185
|
+
|
|
186
|
+
class << self
|
|
187
|
+
# Allocates a float32 buffer of uniformly distributed values in `(0, 1]`.
|
|
188
|
+
#
|
|
189
|
+
# @param shape [Integer, Array<Integer>] positive dimensions
|
|
190
|
+
# @param seed [Integer, nil]
|
|
191
|
+
# @return [Buffer]
|
|
192
|
+
#
|
|
193
|
+
# Tilework.random(1000) # => a Buffer of length 1000
|
|
194
|
+
# Tilework.random([256, 128]) # => a buffer with a 2D shape
|
|
195
|
+
# Tilework.random(1000, seed: 42) # deterministic
|
|
196
|
+
def random(shape, seed: nil)
|
|
197
|
+
fill_new_buffer(shape) { |buffer| Random.uniform!(buffer, seed: seed) }
|
|
198
|
+
end
|
|
199
|
+
|
|
200
|
+
# Allocates a float32 buffer of normally distributed values.
|
|
201
|
+
#
|
|
202
|
+
# @param shape [Integer, Array<Integer>] positive dimensions
|
|
203
|
+
# @param mean [Float]
|
|
204
|
+
# @param stddev [Float]
|
|
205
|
+
# @param seed [Integer, nil]
|
|
206
|
+
# @return [Buffer]
|
|
207
|
+
#
|
|
208
|
+
# Tilework.randn([out, inp]) * Math.sqrt(2.0 / inp) # He initialization
|
|
209
|
+
def randn(shape, mean: 0.0, stddev: 1.0, seed: nil)
|
|
210
|
+
fill_new_buffer(shape) { |buffer| Random.normal!(buffer, mean: mean, stddev: stddev, seed: seed) }
|
|
211
|
+
end
|
|
212
|
+
|
|
213
|
+
private
|
|
214
|
+
|
|
215
|
+
def fill_new_buffer(shape)
|
|
216
|
+
buffer = new_random_buffer(shape)
|
|
217
|
+
yield buffer
|
|
218
|
+
rescue StandardError
|
|
219
|
+
buffer&.free
|
|
220
|
+
raise
|
|
221
|
+
end
|
|
222
|
+
|
|
223
|
+
def new_random_buffer(shape)
|
|
224
|
+
dims = shape.is_a?(Array) ? shape : [shape]
|
|
225
|
+
unless dims.all? { |d| d.is_a?(Integer) && d.positive? }
|
|
226
|
+
raise ArgumentError, "shape must be a positive Integer or an Array of them, got #{shape.inspect}"
|
|
227
|
+
end
|
|
228
|
+
|
|
229
|
+
Buffer.new(:float32, dims.reduce(:*), shape: dims)
|
|
230
|
+
end
|
|
231
|
+
end
|
|
232
|
+
end
|
|
@@ -0,0 +1,269 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Tilework
|
|
4
|
+
# The Tensor Core (nvcuda::wmma) path.
|
|
5
|
+
#
|
|
6
|
+
# Policy:
|
|
7
|
+
# - Use the fragment API (nvcuda::wmma). Hand-written mma.sync / wgmma PTX
|
|
8
|
+
# and CUTLASS belong behind the raw: escape hatch, not here.
|
|
9
|
+
# - Start with TF32: FP32 buffers go in as is, so Tensor Cores are used
|
|
10
|
+
# without touching Buffer::DTYPES.
|
|
11
|
+
#
|
|
12
|
+
# Callers do not decide whether to take this path; eligible? judges from
|
|
13
|
+
# Tilework.capabilities and the matrix shapes. Otherwise it falls through to
|
|
14
|
+
# the existing SIMT kernel, so users see only a speed difference.
|
|
15
|
+
module TensorCore
|
|
16
|
+
# The TF32 fragment shape. K is 8, unlike FP16's 16, so the remainder
|
|
17
|
+
# conditions depend on the dtype.
|
|
18
|
+
FRAG_M = 16
|
|
19
|
+
FRAG_N = 16
|
|
20
|
+
FRAG_K = 8
|
|
21
|
+
|
|
22
|
+
# Block tile: 4 warps (128 threads) per block in a 2x2 layout, each warp
|
|
23
|
+
# owning 32x32 (2x2 of 16x16 fragments) → 64x64 per block. K loads two
|
|
24
|
+
# fragments' worth into shared memory at a time.
|
|
25
|
+
#
|
|
26
|
+
# One tile per warp with no shared-memory reuse is dominated by global
|
|
27
|
+
# reads and loses to SIMT (measured 0.91x); reuse is what makes Tensor
|
|
28
|
+
# Cores pay.
|
|
29
|
+
BLOCK_M = 64
|
|
30
|
+
BLOCK_N = 64
|
|
31
|
+
BLOCK_K = 16
|
|
32
|
+
WARPS_M = 2
|
|
33
|
+
WARPS_N = 2
|
|
34
|
+
WARP = 32
|
|
35
|
+
THREADS = WARPS_M * WARPS_N * WARP # 128
|
|
36
|
+
|
|
37
|
+
class << self
|
|
38
|
+
# A switch to turn the path off explicitly (for performance comparisons or
|
|
39
|
+
# bisecting). On by default; eligible? decides from shapes and capabilities.
|
|
40
|
+
attr_writer :enabled
|
|
41
|
+
|
|
42
|
+
def enabled?
|
|
43
|
+
return false if ENV["GPU_DISABLE_TENSOR_CORE"] == "1"
|
|
44
|
+
|
|
45
|
+
@enabled.nil? ? true : @enabled
|
|
46
|
+
end
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
module_function
|
|
50
|
+
|
|
51
|
+
# Whether GEMM can be delegated to cuBLAS. It is an order of magnitude
|
|
52
|
+
# faster than our WMMA kernel (measured 11.7x) and has **no shape
|
|
53
|
+
# constraints** — small models that miss WMMA's multiple-of-64/16 rule
|
|
54
|
+
# still reach the tensor cores.
|
|
55
|
+
def gemm_library?
|
|
56
|
+
return false unless Tilework.bridge.respond_to?(:gemm_available?)
|
|
57
|
+
|
|
58
|
+
Tilework.bridge.gemm_available?
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
def gemm_library_name
|
|
62
|
+
return Tilework.bridge.gemm_library_name if Tilework.bridge.respond_to?(:gemm_library_name)
|
|
63
|
+
|
|
64
|
+
:cublas
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
# Input dtypes the cuBLAS path accepts and their codes for the bridge.
|
|
68
|
+
# **The output is always float32.**
|
|
69
|
+
GEMM_INPUT_CODES = { float32: 0, float16: 1, bfloat16: 2 }.freeze
|
|
70
|
+
|
|
71
|
+
# When the cuBLAS path applies: any shape (no tile-boundary constraint
|
|
72
|
+
# unlike our WMMA), as long as both sides share a dtype cuBLAS handles.
|
|
73
|
+
def gemm_eligible?(a, b, _dtype)
|
|
74
|
+
return false unless gemm_library?
|
|
75
|
+
return false unless a.dtype == b.dtype
|
|
76
|
+
supported = Tilework.capabilities.gemm_dtypes
|
|
77
|
+
|
|
78
|
+
GEMM_INPUT_CODES.key?(a.dtype) && supported.include?(a.dtype)
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
# Backends with an executable matrix-instruction probe (rocWMMA on HIP)
|
|
82
|
+
# own their native source and supported tile contract. Core only asks the
|
|
83
|
+
# bridge whether this dtype/shape is eligible; it never recognizes a GPU,
|
|
84
|
+
# an ISA name, or a backend name.
|
|
85
|
+
def matrix_core_eligible?(a, b, dtype)
|
|
86
|
+
return false unless TensorCore.enabled?
|
|
87
|
+
return false unless Tilework.bridge.respond_to?(:matrix_core_available?) && Tilework.bridge.matrix_core_available?
|
|
88
|
+
return false unless dtype == :float32 && a.dtype == :float16 && b.dtype == :float16
|
|
89
|
+
|
|
90
|
+
a.rows.positive? && a.rows % 16 == 0 && a.cols % 16 == 0 && b.cols % 16 == 0
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
def matrix_core_gemm(a, b, m, k, n)
|
|
94
|
+
out = Ops.allocate_output(:float32, m * n, [m, n])
|
|
95
|
+
Tilework.bridge.matrix_core_gemm(a.__handle__, b.__handle__, out.__handle__, m, k, n)
|
|
96
|
+
out
|
|
97
|
+
rescue StandardError
|
|
98
|
+
out&.free
|
|
99
|
+
raise
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
# C = A * B through cuBLAS. Whether TF32 is allowed follows enabled? (off
|
|
103
|
+
# means FP32, keeping relative error around 1e-7). fp16 input still
|
|
104
|
+
# accumulates in FP32 and outputs float32. The output is allocated through
|
|
105
|
+
# the context: a graph plan **returns intermediates to the pool instead of
|
|
106
|
+
# freeing them**, so an allocation bypassing the pool would add one slot per
|
|
107
|
+
# run (which matters in decode loops). GEMM writes all of C (beta = 0), so a
|
|
108
|
+
# reused, non-zeroed slot leaves no residue. Eager has no context, so it is
|
|
109
|
+
# the same as Buffer.new.
|
|
110
|
+
def gemm(a, b, m, k, n)
|
|
111
|
+
out = Ops.allocate_output(:float32, m * n, [m, n])
|
|
112
|
+
Tilework.bridge.gemm(a.__handle__, b.__handle__, out.__handle__, m, k, n,
|
|
113
|
+
enabled?, GEMM_INPUT_CODES.fetch(a.dtype))
|
|
114
|
+
out
|
|
115
|
+
rescue StandardError
|
|
116
|
+
out&.free
|
|
117
|
+
raise
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
# Whether `batch` GEMMs of the same shape can be issued at once.
|
|
121
|
+
def gemm_batched?
|
|
122
|
+
return false unless Tilework.bridge.respond_to?(:gemm_batched_available?)
|
|
123
|
+
|
|
124
|
+
Tilework.bridge.gemm_batched_available?
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
# A low-level passthrough to cuBLAS's strided batched GEMM.
|
|
128
|
+
#
|
|
129
|
+
# **All arguments are column-major**; nothing is reinterpreted from row-major.
|
|
130
|
+
# Callers who think in row-major keep the reinterpretation on their side, as
|
|
131
|
+
# Tilework::NN.multi_head_attention does. Moving it here would explode into
|
|
132
|
+
# combinations of transpose flags x which matrix.
|
|
133
|
+
#
|
|
134
|
+
# ld_* is the column-major leading dimension. To view part of a larger
|
|
135
|
+
# matrix as one batch element, pass **the parent's width** as ld and step
|
|
136
|
+
# by the stride. Buffer itself has no strides, so this is the only place
|
|
137
|
+
# that information can be expressed.
|
|
138
|
+
#
|
|
139
|
+
# The output is always float32, as with gemm. alpha is a scale GEMM applies
|
|
140
|
+
# for free; attention's 1/sqrt(dh) uses it, saving another round trip over
|
|
141
|
+
# the score matrix.
|
|
142
|
+
def gemm_batched(a, b, out, m:, k:, n:, lda:, ldb:, ldc:,
|
|
143
|
+
stride_a:, stride_b:, stride_c:, batch:,
|
|
144
|
+
trans_a: false, trans_b: false, alpha: 1.0)
|
|
145
|
+
unless a.dtype == b.dtype && GEMM_INPUT_CODES.key?(a.dtype)
|
|
146
|
+
raise ArgumentError,
|
|
147
|
+
"batched GEMM は両辺が同じ対応済み float dtype のときだけ通る: " \
|
|
148
|
+
"#{a.dtype.inspect} と #{b.dtype.inspect}"
|
|
149
|
+
end
|
|
150
|
+
unless Tilework.capabilities.gemm_dtypes.include?(a.dtype)
|
|
151
|
+
raise ArgumentError, "batched GEMM does not support #{a.dtype.inspect} on this backend"
|
|
152
|
+
end
|
|
153
|
+
raise ArgumentError, "出力は float32 でなければならない" unless out.dtype == :float32
|
|
154
|
+
|
|
155
|
+
Tilework.bridge.gemm_batched(a.__handle__, b.__handle__, out.__handle__,
|
|
156
|
+
m, k, n, lda, ldb, ldc,
|
|
157
|
+
stride_a, stride_b, stride_c,
|
|
158
|
+
trans_a, trans_b, batch, alpha.to_f,
|
|
159
|
+
enabled?, GEMM_INPUT_CODES.fetch(a.dtype))
|
|
160
|
+
out
|
|
161
|
+
end
|
|
162
|
+
|
|
163
|
+
# Alignment: Buffers are flat, stride-less linear memory, so shapes off the
|
|
164
|
+
# tile boundary never enter the WMMA path. Handling remainders with
|
|
165
|
+
# in-kernel branches would diverge within a warp, making fragment
|
|
166
|
+
# instructions undefined. Rejected shapes are computed correctly by SIMT.
|
|
167
|
+
def eligible?(a, b, dtype)
|
|
168
|
+
return false unless TensorCore.enabled?
|
|
169
|
+
return false unless dtype == :float32 && a.dtype == :float32 && b.dtype == :float32
|
|
170
|
+
return false unless Tilework.capabilities[:tf32]
|
|
171
|
+
|
|
172
|
+
a.rows % BLOCK_M == 0 && b.cols % BLOCK_N == 0 && a.cols % BLOCK_K == 0
|
|
173
|
+
end
|
|
174
|
+
|
|
175
|
+
# A WMMA GEMM that reuses tiles through shared memory. __syncthreads() and
|
|
176
|
+
# fragment instructions sit only where every thread in the block and every
|
|
177
|
+
# lane in the warp arrive together.
|
|
178
|
+
def matmul_source
|
|
179
|
+
<<~CUDA
|
|
180
|
+
#include <mma.h>
|
|
181
|
+
using namespace nvcuda;
|
|
182
|
+
|
|
183
|
+
#define BM #{BLOCK_M}
|
|
184
|
+
#define BN #{BLOCK_N}
|
|
185
|
+
#define BK #{BLOCK_K}
|
|
186
|
+
#define WM #{WARPS_M}
|
|
187
|
+
#define WN #{WARPS_N}
|
|
188
|
+
#define FM #{FRAG_M}
|
|
189
|
+
#define FN #{FRAG_N}
|
|
190
|
+
#define FK #{FRAG_K}
|
|
191
|
+
|
|
192
|
+
extern "C" __global__ void matmul_tf32(const float* a, const float* b, float* c,
|
|
193
|
+
int m, int k, int n) {
|
|
194
|
+
__shared__ float As[BM][BK];
|
|
195
|
+
__shared__ float Bs[BK][BN];
|
|
196
|
+
|
|
197
|
+
const int tid = threadIdx.x;
|
|
198
|
+
const int warp = tid / #{WARP};
|
|
199
|
+
const int warp_m = warp / WN; // 0..WM-1
|
|
200
|
+
const int warp_n = warp % WN; // 0..WN-1
|
|
201
|
+
|
|
202
|
+
const int row0 = blockIdx.y * BM; // top-left of this block's output
|
|
203
|
+
const int col0 = blockIdx.x * BN;
|
|
204
|
+
|
|
205
|
+
// 2x2 16x16 fragments per warp = 32x32
|
|
206
|
+
wmma::fragment<wmma::accumulator, FM, FN, FK, float> acc[2][2];
|
|
207
|
+
for (int i = 0; i < 2; i++)
|
|
208
|
+
for (int j = 0; j < 2; j++)
|
|
209
|
+
wmma::fill_fragment(acc[i][j], 0.0f);
|
|
210
|
+
|
|
211
|
+
for (int t = 0; t < k; t += BK) {
|
|
212
|
+
// Cooperative load into shared memory (128 threads, BM*BK = BK*BN = 1024 elements)
|
|
213
|
+
for (int i = tid; i < BM * BK; i += #{THREADS}) {
|
|
214
|
+
As[i / BK][i % BK] = a[(row0 + i / BK) * k + t + (i % BK)];
|
|
215
|
+
}
|
|
216
|
+
for (int i = tid; i < BK * BN; i += #{THREADS}) {
|
|
217
|
+
Bs[i / BN][i % BN] = b[(t + i / BN) * n + col0 + (i % BN)];
|
|
218
|
+
}
|
|
219
|
+
__syncthreads();
|
|
220
|
+
|
|
221
|
+
for (int kk = 0; kk < BK; kk += FK) {
|
|
222
|
+
wmma::fragment<wmma::matrix_a, FM, FN, FK, wmma::precision::tf32, wmma::row_major> fa[2];
|
|
223
|
+
wmma::fragment<wmma::matrix_b, FM, FN, FK, wmma::precision::tf32, wmma::row_major> fb[2];
|
|
224
|
+
|
|
225
|
+
for (int i = 0; i < 2; i++) {
|
|
226
|
+
wmma::load_matrix_sync(fa[i], &As[warp_m * 32 + i * FM][kk], BK);
|
|
227
|
+
// Round values read as FP32 to TF32 (10-bit mantissa, same 8-bit
|
|
228
|
+
// exponent as FP32) before multiply-accumulate. The exponent range
|
|
229
|
+
// matches FP32, so there is no FP16-style overflow/underflow.
|
|
230
|
+
for (int e = 0; e < fa[i].num_elements; e++) fa[i].x[e] = wmma::__float_to_tf32(fa[i].x[e]);
|
|
231
|
+
}
|
|
232
|
+
for (int j = 0; j < 2; j++) {
|
|
233
|
+
wmma::load_matrix_sync(fb[j], &Bs[kk][warp_n * 32 + j * FN], BN);
|
|
234
|
+
for (int e = 0; e < fb[j].num_elements; e++) fb[j].x[e] = wmma::__float_to_tf32(fb[j].x[e]);
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
for (int i = 0; i < 2; i++)
|
|
238
|
+
for (int j = 0; j < 2; j++)
|
|
239
|
+
wmma::mma_sync(acc[i][j], fa[i], fb[j], acc[i][j]);
|
|
240
|
+
}
|
|
241
|
+
__syncthreads();
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
for (int i = 0; i < 2; i++) {
|
|
245
|
+
for (int j = 0; j < 2; j++) {
|
|
246
|
+
int r = row0 + warp_m * 32 + i * FM;
|
|
247
|
+
int cc = col0 + warp_n * 32 + j * FN;
|
|
248
|
+
wmma::store_matrix_sync(c + r * n + cc, acc[i][j], n, wmma::mem_row_major);
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
CUDA
|
|
253
|
+
end
|
|
254
|
+
|
|
255
|
+
# The WMMA path. Allocated through the context for the same reason as gemm
|
|
256
|
+
# (only tile-aligned shapes get here, so the output is fully written).
|
|
257
|
+
def matmul(a, b, m, k, n)
|
|
258
|
+
out = Ops.allocate_output(:float32, m * n, [m, n])
|
|
259
|
+
# raw: true — plain CUDA including mma.h, so no compat header is prepended.
|
|
260
|
+
kernel = Tilework.compile(matmul_source, name: "matmul_tf32", raw: true)
|
|
261
|
+
Ops.launch_kernel(kernel, grid: [n / BLOCK_N, m / BLOCK_M, 1], block: [THREADS, 1, 1],
|
|
262
|
+
args: [a, b, out, m, k, n])
|
|
263
|
+
out
|
|
264
|
+
rescue StandardError
|
|
265
|
+
out&.free
|
|
266
|
+
raise
|
|
267
|
+
end
|
|
268
|
+
end
|
|
269
|
+
end
|