tilework 0.18.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE +21 -0
- data/lib/tilework/backend/cpu/dsl_codegen.rb +509 -0
- data/lib/tilework/backend/cpu/ops_codegen.rb +451 -0
- data/lib/tilework/backend/cpu.rb +156 -0
- data/lib/tilework/buffer.rb +753 -0
- data/lib/tilework/capabilities.rb +183 -0
- data/lib/tilework/compat/cuda.h +66 -0
- data/lib/tilework/compat/hip.h +35 -0
- data/lib/tilework/compat/metal.h +45 -0
- data/lib/tilework/conformance.rb +2296 -0
- data/lib/tilework/core_ext.rb +21 -0
- data/lib/tilework/dialect.rb +67 -0
- data/lib/tilework/dsl.rb +722 -0
- data/lib/tilework/dtype/bfloat.rb +31 -0
- data/lib/tilework/dtype/cast.rb +88 -0
- data/lib/tilework/dtype/half.rb +98 -0
- data/lib/tilework/graph/builder.rb +1073 -0
- data/lib/tilework/graph/fusion.rb +520 -0
- data/lib/tilework/graph/grad.rb +443 -0
- data/lib/tilework/graph/high_level.rb +266 -0
- data/lib/tilework/graph/kernels.rb +403 -0
- data/lib/tilework/graph/node.rb +27 -0
- data/lib/tilework/graph/op.rb +290 -0
- data/lib/tilework/graph/optim.rb +183 -0
- data/lib/tilework/graph/plan.rb +1086 -0
- data/lib/tilework/graph/pool.rb +178 -0
- data/lib/tilework/graph/region.rb +210 -0
- data/lib/tilework/graph/residency.rb +449 -0
- data/lib/tilework/graph/state.rb +64 -0
- data/lib/tilework/graph/tensor.rb +124 -0
- data/lib/tilework/graph.rb +213 -0
- data/lib/tilework/kernel.rb +163 -0
- data/lib/tilework/math/conv.rb +270 -0
- data/lib/tilework/math/fft.rb +172 -0
- data/lib/tilework/math/nn.rb +1103 -0
- data/lib/tilework/math/philox.rb +194 -0
- data/lib/tilework/math/random.rb +232 -0
- data/lib/tilework/math/tensor_core.rb +269 -0
- data/lib/tilework/ops.rb +682 -0
- data/lib/tilework/runtime/offload.rb +498 -0
- data/lib/tilework/runtime/submission.rb +163 -0
- data/lib/tilework/runtime/telemetry.rb +112 -0
- data/lib/tilework/scope.rb +33 -0
- data/lib/tilework/version.rb +5 -0
- data/lib/tilework.rb +345 -0
- metadata +92 -0
|
@@ -0,0 +1,2296 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Tilework
|
|
4
|
+
# The conformance suite.
|
|
5
|
+
#
|
|
6
|
+
# Ships the proof of the layering — "Ops, the DSL and with_gpu give the same
|
|
7
|
+
# numbers on every backend" — as library code in core rather than as a test
|
|
8
|
+
# file. A backend gem's CI gets the same checks as :cpu without writing a
|
|
9
|
+
# single assertion of its own:
|
|
10
|
+
#
|
|
11
|
+
# require "tilework"
|
|
12
|
+
# require "tilework/conformance"
|
|
13
|
+
# report = Tilework::Conformance.run!(backend: :metal)
|
|
14
|
+
# raise report.summary unless report.success?
|
|
15
|
+
#
|
|
16
|
+
# A check that did not run must never turn green by skipping. So each check
|
|
17
|
+
# records exceptions in a Result instead of swallowing them, and
|
|
18
|
+
# Report#success? is true only when **all** of these hold:
|
|
19
|
+
#
|
|
20
|
+
# 1. the backend was usable (otherwise a non-green unavailable?)
|
|
21
|
+
# 2. at least one check actually ran (zero is never green)
|
|
22
|
+
# 3. every check that ran passed
|
|
23
|
+
#
|
|
24
|
+
# 2 is needed because if success? were true with an empty CHECKS, "green with
|
|
25
|
+
# nothing checked" would be possible, defeating the suite's purpose.
|
|
26
|
+
module Conformance
|
|
27
|
+
Result = Struct.new(:name, :error) do
|
|
28
|
+
def passed? = error.nil?
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
class Report
|
|
32
|
+
attr_reader :backend, :results, :unavailable_reason
|
|
33
|
+
|
|
34
|
+
def initialize(backend, results, unavailable_reason: nil)
|
|
35
|
+
@backend = backend
|
|
36
|
+
@results = results
|
|
37
|
+
@unavailable_reason = unavailable_reason
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
# The backend itself was unusable, so not a single check could run.
|
|
41
|
+
def unavailable? = !@unavailable_reason.nil?
|
|
42
|
+
|
|
43
|
+
def failures = results.reject(&:passed?)
|
|
44
|
+
|
|
45
|
+
# Green only when at least one check actually ran and all of them passed.
|
|
46
|
+
# A missing runtime and zero checks run are never green — reporting checks
|
|
47
|
+
# that did not run as passing would defeat the suite's purpose.
|
|
48
|
+
def success?
|
|
49
|
+
return false if unavailable? || results.empty?
|
|
50
|
+
|
|
51
|
+
results.all?(&:passed?)
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
def summary
|
|
55
|
+
if unavailable?
|
|
56
|
+
return "conformance suite did not run on #{backend.inspect} " \
|
|
57
|
+
"(no execution environment): #{unavailable_reason}"
|
|
58
|
+
end
|
|
59
|
+
return "no conformance checks ran on #{backend.inspect}" if results.empty?
|
|
60
|
+
|
|
61
|
+
base = "#{results.count(&:passed?)}/#{results.length} conformance checks passed on #{backend.inspect}"
|
|
62
|
+
return base if failures.empty?
|
|
63
|
+
|
|
64
|
+
"#{base}\n#{failures.map { |r| " #{r.name}: #{r.error.message}" }.join("\n")}"
|
|
65
|
+
end
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
CHECKS = []
|
|
69
|
+
|
|
70
|
+
class << self
|
|
71
|
+
def register(name, &block)
|
|
72
|
+
CHECKS << [name, block]
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
def names = CHECKS.map(&:first)
|
|
76
|
+
|
|
77
|
+
# Selects backend and runs every registered check.
|
|
78
|
+
#
|
|
79
|
+
# Even where the backend is unusable (extension not built, no device, ...)
|
|
80
|
+
# it returns a **non-green Report** instead of raising, so the caller's
|
|
81
|
+
# boilerplate
|
|
82
|
+
#
|
|
83
|
+
# raise report.summary unless report.success?
|
|
84
|
+
#
|
|
85
|
+
# doubles as the report that the runtime is missing, keeping CI from
|
|
86
|
+
# passing while believing it ran the suite. With the backend registry,
|
|
87
|
+
# backend= raises the same DeviceError (a failed require) for an unknown
|
|
88
|
+
# backend name and an uninstalled backend gem, so both become a non-green
|
|
89
|
+
# Report here.
|
|
90
|
+
def run!(backend:)
|
|
91
|
+
begin
|
|
92
|
+
Tilework.backend = backend
|
|
93
|
+
rescue DeviceError => e
|
|
94
|
+
return Report.new(backend, [], unavailable_reason: e.message)
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
results = CHECKS.map { |name, _| Result.new(name, run_check_error(name)) }
|
|
98
|
+
Report.new(backend, results)
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
# Runs a single check (the Minitest adapter turns each into its own
|
|
102
|
+
# test_ method). Failures raise as is (to combine with Minitest's
|
|
103
|
+
# assert_raises and the like).
|
|
104
|
+
def run_check!(name)
|
|
105
|
+
_, block = CHECKS.find { |n, _| n == name }
|
|
106
|
+
raise ArgumentError, "unknown conformance check: #{name.inspect}" unless block
|
|
107
|
+
|
|
108
|
+
Tilework::Kernel.clear_cache!
|
|
109
|
+
instance_eval(&block)
|
|
110
|
+
nil
|
|
111
|
+
ensure
|
|
112
|
+
# Native modules and buffers are TypedData objects. A full backend
|
|
113
|
+
# conformance run intentionally compiles many short-lived pipelines;
|
|
114
|
+
# clear the cache on both sides of a check and collect them here so a
|
|
115
|
+
# long run does not rely on an implementation-specific GC threshold
|
|
116
|
+
# to release device modules, streams, or signal-pool entries.
|
|
117
|
+
Tilework::Kernel.clear_cache!
|
|
118
|
+
GC.start
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
private
|
|
122
|
+
|
|
123
|
+
def run_check_error(name)
|
|
124
|
+
run_check!(name)
|
|
125
|
+
nil
|
|
126
|
+
rescue StandardError => e
|
|
127
|
+
e
|
|
128
|
+
end
|
|
129
|
+
end
|
|
130
|
+
|
|
131
|
+
# --- Small helpers used by the checks ------------------------------------
|
|
132
|
+
|
|
133
|
+
def self.f32(array) = array.pack("f*").unpack("f*")
|
|
134
|
+
|
|
135
|
+
def self.assert_close(expected, actual, tol = 1e-5)
|
|
136
|
+
unless expected.length == actual.length
|
|
137
|
+
raise "length mismatch: expected #{expected.length}, got #{actual.length}"
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
expected.zip(actual).each_with_index do |(e, a), i|
|
|
141
|
+
raise "element #{i}: expected #{e.inspect}, got #{a.inspect} (tolerance #{tol})" if (e - a).abs > tol
|
|
142
|
+
end
|
|
143
|
+
end
|
|
144
|
+
|
|
145
|
+
# A comparison relative to the array's magnitude. **assert_close checks
|
|
146
|
+
# nothing on arrays of small values** — an absolute tolerance of 1e-5 on
|
|
147
|
+
# values around 1e-7, like gradients, passes everything.
|
|
148
|
+
def self.assert_close_relative(expected, actual, tol = 1e-5)
|
|
149
|
+
scale = expected.map(&:abs).max
|
|
150
|
+
# All zeros falls back to absolute comparison (relative error is undefined).
|
|
151
|
+
assert_close(expected, actual, scale.zero? ? tol : tol * scale)
|
|
152
|
+
end
|
|
153
|
+
|
|
154
|
+
def self.assert_equal_arrays(expected, actual)
|
|
155
|
+
raise "expected #{expected.inspect}, got #{actual.inspect}" unless expected == actual
|
|
156
|
+
end
|
|
157
|
+
|
|
158
|
+
# Runs with the portable RNG, then restores the original mode.
|
|
159
|
+
def self.with_portable_random
|
|
160
|
+
previous = Random.mode
|
|
161
|
+
Random.portable!
|
|
162
|
+
yield
|
|
163
|
+
ensure
|
|
164
|
+
Random.mode = previous
|
|
165
|
+
end
|
|
166
|
+
|
|
167
|
+
# An n x n Buffer that is not a view. Building from a nested array returns
|
|
168
|
+
# an owning buffer (reshape gives a view, which is no use for paging checks).
|
|
169
|
+
def self.square(n) = Tilework[Array.new(n) { f32(Array.new(n) { rand }) }]
|
|
170
|
+
|
|
171
|
+
def self.assert(cond, message)
|
|
172
|
+
raise message unless cond
|
|
173
|
+
end
|
|
174
|
+
|
|
175
|
+
# --- Ops (operators) -----------------------------------------------------
|
|
176
|
+
|
|
177
|
+
register(:ops_buffer_addition) do
|
|
178
|
+
a = Conformance.f32(Array.new(64) { rand })
|
|
179
|
+
b = Conformance.f32(Array.new(64) { rand })
|
|
180
|
+
c = Tilework[a] + Tilework[b]
|
|
181
|
+
Conformance.assert_close(a.zip(b).map { |x, y| x + y }, c.to_a)
|
|
182
|
+
end
|
|
183
|
+
|
|
184
|
+
register(:bfloat16_storage_ops_and_graph_forward) do
|
|
185
|
+
values = [1.0, -2.0, 4.0, 8.0]
|
|
186
|
+
input = Tilework[values, dtype: :bfloat16]
|
|
187
|
+
Conformance.assert(input.bytesize == 8, "bfloat16 must use two bytes per element")
|
|
188
|
+
Conformance.assert_equal_arrays(values, input.to_a)
|
|
189
|
+
doubled = input * 2.0
|
|
190
|
+
Conformance.assert(doubled.dtype == :bfloat16, "bfloat16 scalar arithmetic must preserve storage dtype")
|
|
191
|
+
Conformance.assert_equal_arrays(values.map { |value| value * 2.0 }, doubled.to_a)
|
|
192
|
+
total = input.sum(device: true)
|
|
193
|
+
Conformance.assert(total.dtype == :float32, "bfloat16 reductions must accumulate into float32")
|
|
194
|
+
Conformance.assert_close([values.sum], total.to_a)
|
|
195
|
+
|
|
196
|
+
matrix = Tilework[[[1.0, 2.0], [3.0, 4.0]], dtype: :bfloat16]
|
|
197
|
+
plan = Graph.build do |g|
|
|
198
|
+
x = g.input(:x, [2, 2], dtype: :bfloat16)
|
|
199
|
+
g.output(x.matmul(g.param(matrix), out_dtype: :bfloat16) + 1.0)
|
|
200
|
+
end.compile
|
|
201
|
+
output = plan.run(x: matrix)
|
|
202
|
+
Conformance.assert(output.dtype == :bfloat16, "graph output must preserve requested bfloat16 dtype")
|
|
203
|
+
Conformance.assert_equal_arrays([8.0, 11.0, 16.0, 23.0], output.to_a)
|
|
204
|
+
|
|
205
|
+
probs = NN.softmax(matrix)
|
|
206
|
+
Conformance.assert(probs.dtype == :bfloat16, "NN forward must preserve bfloat16 storage")
|
|
207
|
+
probs.to_a.each_slice(2) do |row|
|
|
208
|
+
Conformance.assert_close([1.0], [row.sum], 0.01)
|
|
209
|
+
end
|
|
210
|
+
|
|
211
|
+
cache = Buffer.new(:bfloat16, 4, shape: [2, 2])
|
|
212
|
+
state_plan = Graph.build do |g|
|
|
213
|
+
state = g.state(cache, name: :bf16_cache)
|
|
214
|
+
value = g.input(:value, [2, 2], dtype: :bfloat16)
|
|
215
|
+
g.output(g.read(g.write_into(state, value: value)))
|
|
216
|
+
end.compile
|
|
217
|
+
Conformance.assert_equal_arrays(matrix.to_a, state_plan.run(value: matrix).to_a)
|
|
218
|
+
Conformance.assert_equal_arrays(matrix.to_a, cache.to_a)
|
|
219
|
+
end
|
|
220
|
+
|
|
221
|
+
register(:ops_scalar_multiplication_and_coerce) do
|
|
222
|
+
a = Conformance.f32(Array.new(64) { rand })
|
|
223
|
+
buf = Tilework[a]
|
|
224
|
+
Conformance.assert_close(a.map { |x| x * 2.0 }, (buf * 2.0).to_a)
|
|
225
|
+
Conformance.assert_close(a.map { |x| x * 2.0 }, (2.0 * buf).to_a)
|
|
226
|
+
Conformance.assert_close(a.map { |x| 10.0 - x }, (10.0 - buf).to_a)
|
|
227
|
+
end
|
|
228
|
+
|
|
229
|
+
register(:ops_int32_arithmetic_is_exact) do
|
|
230
|
+
a = Array.new(64) { rand(-1000..1000) }
|
|
231
|
+
b = Array.new(64) { rand(1..100) }
|
|
232
|
+
da = Tilework[a]
|
|
233
|
+
db = Tilework[b]
|
|
234
|
+
Conformance.assert_equal_arrays(a.zip(b).map { |x, y| x - y }, (da - db).to_a)
|
|
235
|
+
Conformance.assert_equal_arrays(a.map { |x| x * 3 }, (da * 3).to_a)
|
|
236
|
+
end
|
|
237
|
+
|
|
238
|
+
register(:ops_unary_math) do
|
|
239
|
+
a = Conformance.f32(Array.new(64) { (rand * 4) + 0.1 })
|
|
240
|
+
buf = Tilework[a]
|
|
241
|
+
Conformance.assert_close(a.map { |x| Math.exp(x) }, buf.exp.to_a, 1e-3)
|
|
242
|
+
Conformance.assert_close(a.map { |x| Math.log(x) }, buf.log.to_a)
|
|
243
|
+
Conformance.assert_close(a.map { |x| Math.sqrt(x) }, buf.sqrt.to_a)
|
|
244
|
+
end
|
|
245
|
+
|
|
246
|
+
register(:ops_abs_and_clamp) do
|
|
247
|
+
a = Conformance.f32(Array.new(64) { (rand * 2) - 1 })
|
|
248
|
+
buf = Tilework[a]
|
|
249
|
+
Conformance.assert_close(a.map(&:abs), buf.abs.to_a)
|
|
250
|
+
Conformance.assert_close(a.map { |x| x.clamp(-0.5, 0.5) }, buf.clamp(-0.5, 0.5).to_a)
|
|
251
|
+
end
|
|
252
|
+
|
|
253
|
+
register(:ops_comparison_masks) do
|
|
254
|
+
a = Conformance.f32(Array.new(64) { rand })
|
|
255
|
+
b = Conformance.f32(Array.new(64) { rand })
|
|
256
|
+
da = Tilework[a]
|
|
257
|
+
Conformance.assert_equal_arrays(a.zip(b).map { |x, y| x > y ? 1 : 0 }, (da > Tilework[b]).to_a)
|
|
258
|
+
Conformance.assert_equal_arrays(a.map { |x| x <= 0.5 ? 1 : 0 }, (da <= 0.5).to_a)
|
|
259
|
+
end
|
|
260
|
+
|
|
261
|
+
register(:ops_elementwise_maximum_minimum) do
|
|
262
|
+
a = Conformance.f32(Array.new(64) { rand })
|
|
263
|
+
b = Conformance.f32(Array.new(64) { rand })
|
|
264
|
+
Conformance.assert_close(a.zip(b).map(&:max), Tilework[a].maximum(Tilework[b]).to_a)
|
|
265
|
+
Conformance.assert_close(a.map { |x| [x, 0.5].min }, Tilework[a].minimum(0.5).to_a)
|
|
266
|
+
end
|
|
267
|
+
|
|
268
|
+
register(:ops_min_max_mean_reductions) do
|
|
269
|
+
a = Conformance.f32(Array.new(500) { (rand * 200) - 100 })
|
|
270
|
+
buf = Tilework[a]
|
|
271
|
+
Conformance.assert_equal_arrays(a.min, buf.min)
|
|
272
|
+
Conformance.assert_equal_arrays(a.max, buf.max)
|
|
273
|
+
Conformance.assert_close([a.sum / a.length], [buf.mean], (a.sum.abs / a.length * 1e-4) + 1e-4)
|
|
274
|
+
end
|
|
275
|
+
|
|
276
|
+
register(:ops_sum_reduction) do
|
|
277
|
+
a = Conformance.f32(Array.new(2_000) { rand })
|
|
278
|
+
Conformance.assert_close([a.sum], [Tilework[a].sum], a.sum.abs * 1e-4)
|
|
279
|
+
ints = Array.new(2_000) { rand(-100..100) }
|
|
280
|
+
Conformance.assert_equal_arrays(ints.sum, Tilework[ints].sum)
|
|
281
|
+
end
|
|
282
|
+
|
|
283
|
+
register(:ops_argmax_argmin) do
|
|
284
|
+
a = Conformance.f32(Array.new(500) { rand })
|
|
285
|
+
buf = Tilework[a]
|
|
286
|
+
Conformance.assert_equal_arrays(a.index(a.max), buf.argmax)
|
|
287
|
+
Conformance.assert_equal_arrays(a.index(a.min), buf.argmin)
|
|
288
|
+
end
|
|
289
|
+
|
|
290
|
+
# device: true folds all the way without reaching the host; the same input
|
|
291
|
+
# gives the same answer (the element count exceeds one threadgroup = 256,
|
|
292
|
+
# so real backends take two or more passes).
|
|
293
|
+
register(:ops_device_resident_reductions_match_the_host_values) do
|
|
294
|
+
a = Conformance.f32(Array.new(5_000) { (rand * 200) - 100 })
|
|
295
|
+
buf = Tilework[a]
|
|
296
|
+
tolerance = (a.sum.abs * 1e-4) + 1e-3
|
|
297
|
+
Conformance.assert_close([buf.sum], buf.sum(device: true).to_a, tolerance)
|
|
298
|
+
Conformance.assert_close([buf.min], buf.min(device: true).to_a)
|
|
299
|
+
Conformance.assert_close([buf.max], buf.max(device: true).to_a)
|
|
300
|
+
Conformance.assert_close([buf.mean], buf.mean(device: true).to_a, (tolerance / a.length) + 1e-4)
|
|
301
|
+
|
|
302
|
+
ints = Array.new(5_000) { rand(-100..100) }
|
|
303
|
+
device_sum = Tilework[ints].sum(device: true)
|
|
304
|
+
Conformance.assert(device_sum.dtype == :int32, "an int32 sum must stay int32 on the device")
|
|
305
|
+
Conformance.assert_equal_arrays([ints.sum], device_sum.to_a)
|
|
306
|
+
end
|
|
307
|
+
|
|
308
|
+
# Even in later passes carrying indices, ties keep the smaller index (the
|
|
309
|
+
# same strict comparison as host folding decides winners).
|
|
310
|
+
register(:ops_device_resident_arg_reductions_match_the_host_index) do
|
|
311
|
+
a = Conformance.f32(Array.new(5_000) { rand })
|
|
312
|
+
a[1_000] = 2.0
|
|
313
|
+
a[4_000] = 2.0 # the max appears twice; the smaller index is the answer
|
|
314
|
+
a[2_500] = -1.0
|
|
315
|
+
buf = Tilework[a]
|
|
316
|
+
|
|
317
|
+
argmax = buf.argmax(device: true)
|
|
318
|
+
Conformance.assert(argmax.dtype == :int32, "argmax returns an index, so the buffer is int32")
|
|
319
|
+
Conformance.assert_equal_arrays([buf.argmax], argmax.to_a)
|
|
320
|
+
Conformance.assert_equal_arrays([1_000], argmax.to_a)
|
|
321
|
+
Conformance.assert_equal_arrays([buf.argmin], buf.argmin(device: true).to_a)
|
|
322
|
+
Conformance.assert_equal_arrays([2_500], buf.argmin(device: true).to_a)
|
|
323
|
+
end
|
|
324
|
+
|
|
325
|
+
register(:ops_matmul) do
|
|
326
|
+
m, k, n = 12, 9, 7
|
|
327
|
+
a_rows = Array.new(m) { Conformance.f32(Array.new(k) { (rand * 2) - 1 }) }
|
|
328
|
+
b_rows = Array.new(k) { Conformance.f32(Array.new(n) { (rand * 2) - 1 }) }
|
|
329
|
+
da = Tilework::Buffer.from_matrix(a_rows)
|
|
330
|
+
db = Tilework::Buffer.from_matrix(b_rows)
|
|
331
|
+
c = da.matmul(db)
|
|
332
|
+
Conformance.assert_equal_arrays([m, n], c.shape)
|
|
333
|
+
expected = Array.new(m) { |i| Array.new(n) { |j| (0...k).sum { |t| a_rows[i][t] * b_rows[t][j] } } }.flatten
|
|
334
|
+
Conformance.assert_close(expected, c.to_a, 1e-2)
|
|
335
|
+
end
|
|
336
|
+
|
|
337
|
+
register(:ops_matmul_int32_is_exact) do
|
|
338
|
+
a = Tilework::Buffer.from_matrix([[1, 2, 3], [4, 5, 6]])
|
|
339
|
+
b = Tilework::Buffer.from_matrix([[7, 8], [9, 10], [11, 12]])
|
|
340
|
+
Conformance.assert_equal_arrays([58, 64, 139, 154], a.matmul(b).to_a)
|
|
341
|
+
end
|
|
342
|
+
|
|
343
|
+
# --- NN (fused kernels) --------------------------------------------------
|
|
344
|
+
|
|
345
|
+
def self.nn_matrix(rows, cols, values)
|
|
346
|
+
buf = Buffer.new(:float32, rows * cols, shape: [rows, cols])
|
|
347
|
+
buf.write(values)
|
|
348
|
+
buf
|
|
349
|
+
end
|
|
350
|
+
|
|
351
|
+
register(:nn_softmax_matches_the_closed_form) do
|
|
352
|
+
out = NN.softmax(Conformance.nn_matrix(1, 3, [1.0, 2.0, 3.0])).to_a
|
|
353
|
+
exps = [1.0, 2.0, 3.0].map { |v| Math.exp(v - 3.0) }
|
|
354
|
+
Conformance.assert_close(exps.map { |e| e / exps.sum }, out)
|
|
355
|
+
end
|
|
356
|
+
|
|
357
|
+
register(:nn_softmax_survives_large_logits) do
|
|
358
|
+
out = NN.softmax(Conformance.nn_matrix(1, 4, [1000.0, 1001.0, 1002.0, 1003.0])).to_a
|
|
359
|
+
Conformance.assert(out.none? { |v| v.nan? || v.infinite? }, "large logits overflowed")
|
|
360
|
+
Conformance.assert_close([1.0], [out.sum])
|
|
361
|
+
end
|
|
362
|
+
|
|
363
|
+
register(:nn_layernorm_gives_zero_mean_and_unit_variance) do
|
|
364
|
+
out = NN.layernorm(Conformance.nn_matrix(1, 4, [1.0, 2.0, 3.0, 4.0])).to_a
|
|
365
|
+
Conformance.assert_close([0.0], [out.sum / out.length], 1e-5)
|
|
366
|
+
Conformance.assert_close([1.0], [out.sum { |v| v * v } / out.length], 1e-3)
|
|
367
|
+
end
|
|
368
|
+
|
|
369
|
+
register(:nn_layernorm_applies_gamma_and_beta) do
|
|
370
|
+
x = Conformance.nn_matrix(1, 4, [1.0, 2.0, 3.0, 4.0])
|
|
371
|
+
gamma = Buffer.new(:float32, 4)
|
|
372
|
+
gamma.write([2.0, 2.0, 2.0, 2.0])
|
|
373
|
+
beta = Buffer.new(:float32, 4)
|
|
374
|
+
beta.write([1.0, 1.0, 1.0, 1.0])
|
|
375
|
+
plain = NN.layernorm(x).to_a
|
|
376
|
+
scaled = NN.layernorm(x, gamma: gamma, beta: beta).to_a
|
|
377
|
+
Conformance.assert_close(plain.map { |v| (v * 2.0) + 1.0 }, scaled, 1e-5)
|
|
378
|
+
end
|
|
379
|
+
|
|
380
|
+
register(:nn_rmsnorm_divides_by_the_root_mean_square_without_centering) do
|
|
381
|
+
x = Conformance.nn_matrix(2, 4, [3.0, 3.0, 3.0, 3.0, 1.0, -2.0, 3.0, -4.0])
|
|
382
|
+
got = NN.rmsnorm(x, eps: 0.0).to_a
|
|
383
|
+
want = [3.0, 3.0, 3.0, 3.0, 1.0, -2.0, 3.0, -4.0].each_slice(4).flat_map do |row|
|
|
384
|
+
rms = Math.sqrt(row.sum { |v| v * v } / 4.0)
|
|
385
|
+
row.map { |v| v / rms }
|
|
386
|
+
end
|
|
387
|
+
Conformance.assert_close(want, got, 1e-5)
|
|
388
|
+
end
|
|
389
|
+
|
|
390
|
+
register(:nn_groupnorm_normalizes_within_each_group) do
|
|
391
|
+
n, c, h, w = 1, 4, 2, 2
|
|
392
|
+
x = Buffer.new(:float32, n * c * h * w, shape: [n, c, h, w])
|
|
393
|
+
x.write((0...16).map { |i| (i * 3 % 7).to_f })
|
|
394
|
+
out = NN.groupnorm(x, groups: 2).to_a
|
|
395
|
+
out.each_slice(8) do |group|
|
|
396
|
+
Conformance.assert_close([0.0], [group.sum / 8.0], 1e-5)
|
|
397
|
+
Conformance.assert_close([1.0], [group.sum { |v| v * v } / 8.0], 1e-3)
|
|
398
|
+
end
|
|
399
|
+
end
|
|
400
|
+
|
|
401
|
+
register(:nn_relu_clamps_negatives_to_zero) do
|
|
402
|
+
x = Buffer.new(:float32, 4)
|
|
403
|
+
x.write([-2.0, -0.5, 0.5, 2.0])
|
|
404
|
+
Conformance.assert_close([0.0, 0.0, 0.5, 2.0], NN.relu(x).to_a)
|
|
405
|
+
end
|
|
406
|
+
|
|
407
|
+
register(:nn_sigmoid_and_silu_match_their_definitions) do
|
|
408
|
+
x = Buffer.new(:float32, 3)
|
|
409
|
+
x.write([-1.0, 0.0, 2.0])
|
|
410
|
+
want = [-1.0, 0.0, 2.0].map { |v| 1.0 / (1.0 + Math.exp(-v)) }
|
|
411
|
+
Conformance.assert_close(want, NN.sigmoid(x).to_a)
|
|
412
|
+
Conformance.assert_close([-1.0, 0.0, 2.0].each_with_index.map { |v, i| v * want[i] }, NN.silu(x).to_a)
|
|
413
|
+
end
|
|
414
|
+
|
|
415
|
+
register(:nn_gelu_variants_agree_closely_and_pass_through_zero) do
|
|
416
|
+
x = Buffer.new(:float32, 5)
|
|
417
|
+
x.write([-2.0, -0.5, 0.0, 0.5, 2.0])
|
|
418
|
+
approx = NN.gelu(x).to_a
|
|
419
|
+
exact = NN.gelu(x, exact: true).to_a
|
|
420
|
+
Conformance.assert_close([0.0], [approx[2]], 1e-9)
|
|
421
|
+
Conformance.assert_close(exact, approx, 2e-3) # the known gap of the tanh approximation
|
|
422
|
+
end
|
|
423
|
+
|
|
424
|
+
# Losses end in a reduction, so check that the host Float and the shape-[1]
|
|
425
|
+
# Buffer left on the device agree (training loops use the latter).
|
|
426
|
+
register(:nn_mse_matches_the_closed_form) do
|
|
427
|
+
pred = Conformance.nn_matrix(2, 3, [0.5, -1.0, 2.0, 0.0, 1.0, -0.5])
|
|
428
|
+
target = Conformance.nn_matrix(2, 3, [0.0, 0.0, 1.0, 1.0, 1.0, 1.0])
|
|
429
|
+
errors = [0.25, 1.0, 1.0, 1.0, 0.0, 2.25]
|
|
430
|
+
|
|
431
|
+
Conformance.assert_close(errors, NN.mse(pred, target, reduction: :none).to_a)
|
|
432
|
+
Conformance.assert_close([errors.sum / 6.0], [NN.mse(pred, target)])
|
|
433
|
+
Conformance.assert_close([errors.sum], [NN.mse(pred, target, reduction: :sum)])
|
|
434
|
+
Conformance.assert_close([errors.sum / 6.0], NN.mse(pred, target, device: true).to_a, 1e-5)
|
|
435
|
+
end
|
|
436
|
+
|
|
437
|
+
# Cross entropy **takes logits**. A naive log(softmax(x)) hits -inf where a
|
|
438
|
+
# probability underflows to 0, so this pins the log-sum-exp computation
|
|
439
|
+
# (the large-logits check below is the core of it).
|
|
440
|
+
register(:nn_cross_entropy_matches_the_log_sum_exp_form) do
|
|
441
|
+
rows = [[0.1, 0.5, -0.3], [2.0, 0.0, 1.0]]
|
|
442
|
+
logits = Conformance.nn_matrix(2, 3, rows.flatten)
|
|
443
|
+
labels = Buffer.new(:int32, 2, shape: [2])
|
|
444
|
+
labels.write([1, 0])
|
|
445
|
+
|
|
446
|
+
want = rows.each_with_index.map do |row, i|
|
|
447
|
+
m = row.max
|
|
448
|
+
Math.log(row.sum { |v| Math.exp(v - m) }) - (row[[1, 0][i]] - m)
|
|
449
|
+
end
|
|
450
|
+
Conformance.assert_close(want, NN.cross_entropy(logits, labels, reduction: :none).to_a, 1e-5)
|
|
451
|
+
Conformance.assert_close([want.sum / 2.0], [NN.cross_entropy(logits, labels)], 1e-5)
|
|
452
|
+
Conformance.assert_close([want.sum], [NN.cross_entropy(logits, labels, reduction: :sum)], 1e-5)
|
|
453
|
+
Conformance.assert_close([want.sum / 2.0], NN.cross_entropy(logits, labels, device: true).to_a, 1e-5)
|
|
454
|
+
end
|
|
455
|
+
|
|
456
|
+
register(:nn_cross_entropy_survives_large_logits) do
|
|
457
|
+
logits = Conformance.nn_matrix(1, 4, [1000.0, 1001.0, 1002.0, 1003.0])
|
|
458
|
+
labels = Buffer.new(:int32, 1, shape: [1])
|
|
459
|
+
labels.write([3])
|
|
460
|
+
value = NN.cross_entropy(logits, labels)
|
|
461
|
+
Conformance.assert(!value.nan? && !value.infinite?, "large logits overflowed (got #{value})")
|
|
462
|
+
# The correct class is the largest, so the loss settles at log(1 + e^-1 + e^-2 + e^-3).
|
|
463
|
+
want = Math.log([0, 1, 2, 3].sum { |k| Math.exp(-k) })
|
|
464
|
+
Conformance.assert_close([want], [value], 1e-4)
|
|
465
|
+
end
|
|
466
|
+
|
|
467
|
+
register(:nn_transpose_of_a_non_square_matrix_fills_every_element) do
|
|
468
|
+
out = NN.transpose(Conformance.nn_matrix(2, 3, [1.0, 2.0, 3.0, 4.0, 5.0, 6.0]))
|
|
469
|
+
Conformance.assert_equal_arrays([3, 2], out.shape)
|
|
470
|
+
Conformance.assert_equal_arrays([1.0, 4.0, 2.0, 5.0, 3.0, 6.0], out.to_a)
|
|
471
|
+
end
|
|
472
|
+
|
|
473
|
+
register(:nn_multi_head_attention_matches_a_hand_written_reference) do
|
|
474
|
+
heads, dh, n, m = 2, 3, 4, 5
|
|
475
|
+
d = heads * dh
|
|
476
|
+
qa = (0...(n * d)).map { |i| Math.sin(i * 0.7) }
|
|
477
|
+
ka = (0...(m * d)).map { |i| Math.cos(i * 0.3) }
|
|
478
|
+
va = (0...(m * d)).map { |i| Math.sin(i * 0.11) - 0.2 }
|
|
479
|
+
|
|
480
|
+
got = NN.multi_head_attention(Conformance.nn_matrix(n, d, qa), Conformance.nn_matrix(m, d, ka),
|
|
481
|
+
Conformance.nn_matrix(m, d, va), heads: heads).to_a
|
|
482
|
+
|
|
483
|
+
scale = 1.0 / Math.sqrt(dh)
|
|
484
|
+
want = Array.new(n * d, 0.0)
|
|
485
|
+
heads.times do |h|
|
|
486
|
+
(0...n).each do |r|
|
|
487
|
+
s = (0...m).map { |c| (0...dh).sum { |j| qa[(r * d) + (h * dh) + j] * ka[(c * d) + (h * dh) + j] } * scale }
|
|
488
|
+
e = s.map { |x| Math.exp(x - s.max) }
|
|
489
|
+
(0...dh).each do |j|
|
|
490
|
+
want[(r * d) + (h * dh) + j] = (0...m).sum { |c| (e[c] / e.sum) * va[(c * d) + (h * dh) + j] }
|
|
491
|
+
end
|
|
492
|
+
end
|
|
493
|
+
end
|
|
494
|
+
Conformance.assert_close(want, got, 1e-4)
|
|
495
|
+
end
|
|
496
|
+
|
|
497
|
+
# --- DSL (Ruby blocks) / with_gpu ----------------------------------------
|
|
498
|
+
#
|
|
499
|
+
# Each check builds its Tilework.kernel {} inside the check, so every run
|
|
500
|
+
# uses a fresh DSLKernel and no generated source is carried across backend
|
|
501
|
+
# switches.
|
|
502
|
+
|
|
503
|
+
register(:dsl_vec_add_matches_reference) do
|
|
504
|
+
vec_add = Tilework.kernel("conformance_vec_add") do |a, b, c, n|
|
|
505
|
+
i = global_id_x
|
|
506
|
+
c[i] = a[i] + b[i] if i < n
|
|
507
|
+
end
|
|
508
|
+
n = 500
|
|
509
|
+
a = Conformance.f32(Array.new(n) { rand })
|
|
510
|
+
b = Conformance.f32(Array.new(n) { rand })
|
|
511
|
+
c = Buffer.new(:float32, n)
|
|
512
|
+
vec_add.(Tilework[a], Tilework[b], c, n)
|
|
513
|
+
Conformance.assert_close(a.zip(b).map { |x, y| x + y }, c.to_a)
|
|
514
|
+
end
|
|
515
|
+
|
|
516
|
+
register(:dsl_specializes_for_int_buffers) do
|
|
517
|
+
vec_add = Tilework.kernel("conformance_vec_add_int") do |a, b, c, n|
|
|
518
|
+
i = global_id_x
|
|
519
|
+
c[i] = a[i] + b[i] if i < n
|
|
520
|
+
end
|
|
521
|
+
n = 200
|
|
522
|
+
a = Array.new(n) { rand(-100..100) }
|
|
523
|
+
b = Array.new(n) { rand(-100..100) }
|
|
524
|
+
c = Buffer.new(:int32, n)
|
|
525
|
+
vec_add.(Tilework[a], Tilework[b], c, n)
|
|
526
|
+
Conformance.assert_equal_arrays(a.zip(b).map(&:sum), c.to_a)
|
|
527
|
+
end
|
|
528
|
+
|
|
529
|
+
register(:dsl_kernel_with_loop_computes_row_sums) do
|
|
530
|
+
rows, cols = 8, 10
|
|
531
|
+
m = Conformance.f32(Array.new(rows * cols) { rand })
|
|
532
|
+
row_sum = Tilework.kernel("conformance_row_sum", out: 1) do |mat, out, rows, cols|
|
|
533
|
+
r = global_id_x
|
|
534
|
+
if r < rows
|
|
535
|
+
acc = 0.0
|
|
536
|
+
cols.times { |c| acc = acc + mat[(r * cols) + c] }
|
|
537
|
+
out[r] = acc
|
|
538
|
+
end
|
|
539
|
+
end
|
|
540
|
+
out = row_sum.(Tilework[m], rows, cols, over: rows)
|
|
541
|
+
Conformance.assert_close(Array.new(rows) { |r| m[r * cols, cols].sum }, out.to_a, 1e-3)
|
|
542
|
+
end
|
|
543
|
+
|
|
544
|
+
register(:dsl_2d_launch_transpose) do
|
|
545
|
+
size = 8
|
|
546
|
+
m = Conformance.f32(Array.new(size * size) { rand })
|
|
547
|
+
transpose = Tilework.kernel("conformance_transpose") do |src, dst, size|
|
|
548
|
+
x = global_id_x
|
|
549
|
+
y = global_id_y
|
|
550
|
+
dst[(x * size) + y] = src[(y * size) + x] if x < size && y < size
|
|
551
|
+
end
|
|
552
|
+
dst = Buffer.new(:float32, size * size)
|
|
553
|
+
transpose.launch(grid: [1, 1], block: [size, size], args: [Tilework[m], dst, size])
|
|
554
|
+
expected = Array.new(size * size) { |i| x, y = i.divmod(size); m[(y * size) + x] }
|
|
555
|
+
Conformance.assert_close(expected, dst.to_a)
|
|
556
|
+
end
|
|
557
|
+
|
|
558
|
+
register(:dsl_2d_indexing_transpose_with_nested_arrays) do
|
|
559
|
+
rows, cols = 4, 5
|
|
560
|
+
m = Array.new(rows) { |r| Array.new(cols) { |c| ((r * cols) + c).to_f } }
|
|
561
|
+
transpose = Tilework.kernel("conformance_transpose2d", out: 1) do |mat, out|
|
|
562
|
+
r = global_id_y
|
|
563
|
+
c = global_id_x
|
|
564
|
+
out[r, c] = mat[c, r] if c < mat.rows && r < mat.cols
|
|
565
|
+
end
|
|
566
|
+
result = Tilework.run(m) { |mat| transpose.(mat, over: [mat.rows, mat.cols]) }
|
|
567
|
+
expected = cols.times.map { |c| rows.times.map { |r| m[r][c] } }
|
|
568
|
+
Conformance.assert_equal_arrays(expected, result)
|
|
569
|
+
end
|
|
570
|
+
|
|
571
|
+
register(:dsl_unroll_expands_at_transpile_time) do
|
|
572
|
+
weighted = Tilework.kernel("conformance_weighted", out: 1) do |a, out, n|
|
|
573
|
+
i = global_id_x
|
|
574
|
+
if i < n
|
|
575
|
+
total = 0.0
|
|
576
|
+
unroll([1.0, 2.0, 3.0]) { |w| total = total + (a[i] * w) }
|
|
577
|
+
out[i] = total
|
|
578
|
+
end
|
|
579
|
+
end
|
|
580
|
+
a = Buffer.from([1.0, 2.0, 3.0, 4.0])
|
|
581
|
+
out = weighted.(a, 4)
|
|
582
|
+
Conformance.assert_close([6.0, 12.0, 18.0, 24.0], out.to_a)
|
|
583
|
+
end
|
|
584
|
+
|
|
585
|
+
register(:dsl_math_functions) do
|
|
586
|
+
sqrt_k = Tilework.kernel("conformance_sqrt") do |a, out, n|
|
|
587
|
+
i = global_id_x
|
|
588
|
+
out[i] = Math.sqrt(a[i]) if i < n
|
|
589
|
+
end
|
|
590
|
+
a = Buffer.from([4.0, 9.0, 16.0])
|
|
591
|
+
out = Buffer.new(:float32, 3)
|
|
592
|
+
sqrt_k.(a, out, 3)
|
|
593
|
+
Conformance.assert_close([2.0, 3.0, 4.0], out.to_a)
|
|
594
|
+
end
|
|
595
|
+
|
|
596
|
+
# An end-to-end check that with_gpu (Tilework.run) bridges Array <-> Buffer
|
|
597
|
+
# conversion, DSL kernels and operator chains.
|
|
598
|
+
register(:with_gpu_and_dsl_end_to_end) do
|
|
599
|
+
vec_add = Tilework.kernel("conformance_with_gpu_vec_add") do |a, b, c, n|
|
|
600
|
+
i = global_id_x
|
|
601
|
+
c[i] = a[i] + b[i] if i < n
|
|
602
|
+
end
|
|
603
|
+
n = 200
|
|
604
|
+
xs = Conformance.f32(Array.new(n) { rand })
|
|
605
|
+
ys = Conformance.f32(Array.new(n) { rand })
|
|
606
|
+
|
|
607
|
+
result = Tilework.run(xs, ys) do |x, y|
|
|
608
|
+
c = Buffer.new(:float32, n)
|
|
609
|
+
vec_add.(x, y, c, n)
|
|
610
|
+
(c * 2.0).clamp(0.0, 1.5)
|
|
611
|
+
end
|
|
612
|
+
|
|
613
|
+
expected = xs.zip(ys).map { |a, b| ((a + b) * 2.0).clamp(0.0, 1.5) }
|
|
614
|
+
Conformance.assert_close(expected, result, 1e-5)
|
|
615
|
+
end
|
|
616
|
+
# --- Random (the portable path) ------------------------------------------
|
|
617
|
+
#
|
|
618
|
+
# Native RNG (cuRAND / MPS) sequences differ per backend, so only the
|
|
619
|
+
# portable path (Random.portable!) can be checked here. **The expected
|
|
620
|
+
# values are hard-coded** — rather than using :cpu as the oracle, this pins
|
|
621
|
+
# that every backend produces these exact numbers.
|
|
622
|
+
PORTABLE_UNIFORM_SEED_42 = [
|
|
623
|
+
0.6129599213600159, 0.4685865640640259, 0.0732317566871643,
|
|
624
|
+
0.34086155891418457, 0.9877187013626099, 0.3270633816719055,
|
|
625
|
+
].freeze
|
|
626
|
+
|
|
627
|
+
PORTABLE_NORMAL_SEED_42 = [
|
|
628
|
+
-0.9701898097991943, -1.2356207370758057, -0.07318136096000671, -1.106665015220642,
|
|
629
|
+
].freeze
|
|
630
|
+
|
|
631
|
+
# Uniform is bit-identical (integer-to-float conversion uses only 24 bits and
|
|
632
|
+
# powers of two, so no backend rounds).
|
|
633
|
+
register(:random_portable_uniform_is_bit_identical) do
|
|
634
|
+
Conformance.with_portable_random do
|
|
635
|
+
got = Tilework.random(6, seed: 42).to_a
|
|
636
|
+
Conformance.assert_equal_arrays(Conformance.f32(Conformance::PORTABLE_UNIFORM_SEED_42), got)
|
|
637
|
+
end
|
|
638
|
+
end
|
|
639
|
+
|
|
640
|
+
# Normal goes through Box-Muller's sqrt/log/cos and agrees within their
|
|
641
|
+
# precision (no bit identity — the last ulp depends on the math library).
|
|
642
|
+
register(:random_portable_normal_matches_across_backends) do
|
|
643
|
+
Conformance.with_portable_random do
|
|
644
|
+
got = Tilework.randn(4, seed: 42).to_a
|
|
645
|
+
Conformance.assert_close(Conformance::PORTABLE_NORMAL_SEED_42, got, 1e-5)
|
|
646
|
+
end
|
|
647
|
+
end
|
|
648
|
+
|
|
649
|
+
# Counter-based, so element i does not depend on the request size.
|
|
650
|
+
register(:random_portable_is_independent_of_the_request_size) do
|
|
651
|
+
Conformance.with_portable_random do
|
|
652
|
+
long = Tilework.random(64, seed: 7).to_a
|
|
653
|
+
short = Tilework.random(5, seed: 7).to_a
|
|
654
|
+
Conformance.assert_equal_arrays(long.first(5), short)
|
|
655
|
+
end
|
|
656
|
+
end
|
|
657
|
+
|
|
658
|
+
register(:random_native_is_reproducible_within_a_process) do
|
|
659
|
+
a = Tilework.random(32, seed: 1234).to_a
|
|
660
|
+
b = Tilework.random(32, seed: 1234).to_a
|
|
661
|
+
Conformance.assert_equal_arrays(a, b)
|
|
662
|
+
Conformance.assert(a.all? { |x| x > 0.0 && x <= 1.0 }, "uniform values must lie in (0, 1]")
|
|
663
|
+
end
|
|
664
|
+
|
|
665
|
+
# Mixing async launches with host writes does not change a seed's result.
|
|
666
|
+
# On unified-memory backends buffer_write is a raw memcpy, so racing an
|
|
667
|
+
# in-flight kernel changes results intermittently.
|
|
668
|
+
register(:random_survives_interleaved_async_work_and_host_writes) do
|
|
669
|
+
run = lambda do
|
|
670
|
+
noise = Tilework.random(512, seed: 4321)
|
|
671
|
+
chain = noise
|
|
672
|
+
3.times do |k|
|
|
673
|
+
# Interleave a host write (Tilework[...] is buffer_write)
|
|
674
|
+
weights = Tilework[Array.new(512) { |i| ((i + k) % 7) * 0.25 }]
|
|
675
|
+
chain = ((chain * 1.5) + weights).clamp(-4.0, 4.0)
|
|
676
|
+
end
|
|
677
|
+
chain.to_a
|
|
678
|
+
end
|
|
679
|
+
Conformance.assert_equal_arrays(run.call, run.call)
|
|
680
|
+
end
|
|
681
|
+
|
|
682
|
+
# --- Graph layer (Tensor / Graph / Plan) ---
|
|
683
|
+
|
|
684
|
+
register(:graph_matches_the_eager_path) do
|
|
685
|
+
a = Tilework[Conformance.f32([1.0, 2.0, 3.0, 4.0, 5.0, 6.0])].reshape(2, 3)
|
|
686
|
+
w = Tilework[Conformance.f32([1.0, 0.0, 0.0, 1.0, 1.0, 1.0])].reshape(3, 2)
|
|
687
|
+
plan = Tilework::Graph.build do |g|
|
|
688
|
+
x = g.input(:x, [2, 3])
|
|
689
|
+
g.output(((x.matmul(g.param(w)) * 2.0) + 1.0).tanh)
|
|
690
|
+
end.compile
|
|
691
|
+
expected = ((a.matmul(w) * 2.0) + 1.0).tanh.to_a
|
|
692
|
+
Conformance.assert_close(expected, plan.run(x: a).to_a)
|
|
693
|
+
end
|
|
694
|
+
|
|
695
|
+
# Async launch + buffer reuse is bit-identical to waiting per node and
|
|
696
|
+
# allocating every time. :cpu dispatches synchronously, so this matters only
|
|
697
|
+
# on hardware.
|
|
698
|
+
register(:graph_async_and_pooled_execution_matches_the_serial_one) do
|
|
699
|
+
values = Conformance.f32(Array.new(512) { rand })
|
|
700
|
+
build = lambda do
|
|
701
|
+
plan = Tilework::Graph.build do |g|
|
|
702
|
+
v = g.input(:x, [512])
|
|
703
|
+
5.times { v = ((v * 1.5) + 0.25).tanh }
|
|
704
|
+
g.output(v)
|
|
705
|
+
end.compile
|
|
706
|
+
plan.run(x: Tilework[values]).to_a
|
|
707
|
+
end
|
|
708
|
+
|
|
709
|
+
async = Tilework::Graph.async
|
|
710
|
+
pool = Tilework::Graph.buffer_pool
|
|
711
|
+
begin
|
|
712
|
+
Tilework::Graph.async = true
|
|
713
|
+
Tilework::Graph.buffer_pool = true
|
|
714
|
+
pipelined = build.call
|
|
715
|
+
Tilework::Graph.async = false
|
|
716
|
+
Tilework::Graph.buffer_pool = false
|
|
717
|
+
serial = build.call
|
|
718
|
+
ensure
|
|
719
|
+
Tilework::Graph.async = async
|
|
720
|
+
Tilework::Graph.buffer_pool = pool
|
|
721
|
+
end
|
|
722
|
+
Conformance.assert_equal_arrays(serial, pipelined)
|
|
723
|
+
end
|
|
724
|
+
|
|
725
|
+
# Reductions are shape-[1] values inside a graph; also covers a plan with two outputs.
|
|
726
|
+
register(:graph_reductions_match_the_eager_path) do
|
|
727
|
+
values = Conformance.f32(Array.new(1_000) { (rand * 2) - 1 })
|
|
728
|
+
buf = Tilework[values]
|
|
729
|
+
plan = Tilework::Graph.build do |g|
|
|
730
|
+
x = g.input(:x, [1_000])
|
|
731
|
+
g.output((x * x).mean)
|
|
732
|
+
g.output(x.argmax)
|
|
733
|
+
end.compile
|
|
734
|
+
|
|
735
|
+
mean, argmax = plan.run(x: buf)
|
|
736
|
+
Conformance.assert_close([(buf * buf).mean], mean.to_a, 1e-4)
|
|
737
|
+
Conformance.assert_equal_arrays([buf.argmax], argmax.to_a)
|
|
738
|
+
end
|
|
739
|
+
|
|
740
|
+
def self.nd(shape, values)
|
|
741
|
+
buffer = Buffer.new(:float32, values.length, shape: shape)
|
|
742
|
+
buffer.write(Conformance.f32(values))
|
|
743
|
+
buffer
|
|
744
|
+
end
|
|
745
|
+
|
|
746
|
+
# Batched matmul becomes one strided batched GEMM when available and per-batch
|
|
747
|
+
# 2D matmuls otherwise; this pins that **the path does not change values**.
|
|
748
|
+
# A wrong column-major reinterpretation (swapped transposes or ld) breaks
|
|
749
|
+
# only the delegated path, so m / k / n are all different to rule out shapes
|
|
750
|
+
# matching by accident.
|
|
751
|
+
register(:graph_batched_matmul_matches_per_batch_matmuls) do
|
|
752
|
+
batch = 2
|
|
753
|
+
m = 2
|
|
754
|
+
k = 3
|
|
755
|
+
n = 4
|
|
756
|
+
a = Conformance.nd([batch, m, k], Array.new(batch * m * k) { |i| ((i % 5) - 2) * 0.5 })
|
|
757
|
+
b = Conformance.nd([batch, k, n], Array.new(batch * k * n) { |i| ((i % 7) - 3) * 0.25 })
|
|
758
|
+
|
|
759
|
+
plan = Tilework::Graph.build do |g|
|
|
760
|
+
g.output(g.input(:a, [batch, m, k]).matmul(g.param(b)))
|
|
761
|
+
end.compile
|
|
762
|
+
|
|
763
|
+
expected = batch.times.flat_map do |i|
|
|
764
|
+
a.slice(i).reshape(m, k).matmul(b.slice(i).reshape(k, n)).to_a
|
|
765
|
+
end
|
|
766
|
+
result = plan.run(a: a)
|
|
767
|
+
Conformance.assert(result.shape == [batch, m, n],
|
|
768
|
+
"batched matmul should be #{[batch, m, n].inspect}, got #{result.shape.inspect}")
|
|
769
|
+
Conformance.assert_close(expected, result.to_a, 1e-4)
|
|
770
|
+
end
|
|
771
|
+
|
|
772
|
+
# A broadcast gives the same answer as materializing the stretched side
|
|
773
|
+
# first. The oracle is a same-size constant rearranged on the host, so value
|
|
774
|
+
# mix-ups (rows and columns swapped, a misplaced stride 0) always show here.
|
|
775
|
+
# The arithmetic is identical, so bit identity is required.
|
|
776
|
+
register(:graph_broadcast_matches_an_explicitly_expanded_operand) do
|
|
777
|
+
rows = 4
|
|
778
|
+
cols = 3
|
|
779
|
+
x = Conformance.nd([rows, cols], Array.new(rows * cols) { |i| ((i % 7) - 3) * 0.5 })
|
|
780
|
+
row_vector = Conformance.f32([0.25, -1.5, 2.0])
|
|
781
|
+
col_values = Conformance.f32([1.0, -2.0, 0.5, 3.0])
|
|
782
|
+
tiled_rows = Conformance.nd([rows, cols], row_vector * rows)
|
|
783
|
+
tiled_cols = Conformance.nd([rows, cols], col_values.flat_map { |v| [v] * cols })
|
|
784
|
+
|
|
785
|
+
broadcast = Tilework::Graph.build do |g|
|
|
786
|
+
t = g.input(:x, [rows, cols])
|
|
787
|
+
g.output((t + g.param(Tilework[row_vector])) * g.param(Conformance.nd([rows, 1], col_values)))
|
|
788
|
+
end.compile.run(x: x)
|
|
789
|
+
|
|
790
|
+
expanded = Tilework::Graph.build do |g|
|
|
791
|
+
t = g.input(:x, [rows, cols])
|
|
792
|
+
g.output((t + g.param(tiled_rows)) * g.param(tiled_cols))
|
|
793
|
+
end.compile.run(x: x)
|
|
794
|
+
|
|
795
|
+
Conformance.assert(broadcast.shape == [rows, cols],
|
|
796
|
+
"broadcast should be #{[rows, cols].inspect}, got #{broadcast.shape.inspect}")
|
|
797
|
+
Conformance.assert_equal_arrays(expanded.to_a, broadcast.to_a)
|
|
798
|
+
end
|
|
799
|
+
|
|
800
|
+
# Select by condition. An inverted mask or a scalar argument in the wrong
|
|
801
|
+
# position shows up here as swapped values.
|
|
802
|
+
register(:graph_where_selects_elementwise) do
|
|
803
|
+
values = Conformance.f32(Array.new(64) { |i| ((i % 9) - 4) * 0.5 })
|
|
804
|
+
x = Tilework[values]
|
|
805
|
+
plan = Tilework::Graph.build do |g|
|
|
806
|
+
t = g.input(:x, [64])
|
|
807
|
+
g.output(g.where(t > 0, t, 0.0)) # tensor / scalar values
|
|
808
|
+
g.output(g.where(t.eq(0.0), t + 1.0, t * 10.0)) # tensors on both sides
|
|
809
|
+
g.output(g.where(t > 0, 1, 0)) # scalars on both sides (an int32 mask)
|
|
810
|
+
end.compile
|
|
811
|
+
|
|
812
|
+
relu, patched, mask = plan.run(x: x)
|
|
813
|
+
Conformance.assert_close(values.map { |v| v.positive? ? v : 0.0 }, relu.to_a)
|
|
814
|
+
Conformance.assert_close(values.map { |v| v.zero? ? v + 1.0 : v * 10.0 }, patched.to_a, 1e-4)
|
|
815
|
+
Conformance.assert_equal_arrays(values.map { |v| v.positive? ? 1 : 0 }, mask.to_a)
|
|
816
|
+
end
|
|
817
|
+
|
|
818
|
+
# Casting truncates toward zero (like a C cast). floor would be off by one
|
|
819
|
+
# on the negative side, so negative values are always included.
|
|
820
|
+
register(:graph_cast_truncates_toward_zero) do
|
|
821
|
+
values = Conformance.f32([-2.75, -1.5, -0.5, 0.0, 0.5, 1.5, 2.75])
|
|
822
|
+
plan = Tilework::Graph.build do |g|
|
|
823
|
+
t = g.input(:x, [values.length])
|
|
824
|
+
g.output(t.cast(:int32))
|
|
825
|
+
g.output(t.cast(:int32).cast(:float32))
|
|
826
|
+
end.compile
|
|
827
|
+
|
|
828
|
+
ints, back = plan.run(x: Tilework[values])
|
|
829
|
+
Conformance.assert_equal_arrays(values.map(&:to_i), ints.to_a)
|
|
830
|
+
Conformance.assert_close(values.map { |v| v.to_i.to_f }, back.to_a)
|
|
831
|
+
end
|
|
832
|
+
|
|
833
|
+
register(:graph_iota_writes_the_index_along_its_axis) do
|
|
834
|
+
plan = Tilework::Graph.build do |g|
|
|
835
|
+
g.output(g.iota([2, 3], axis: 0))
|
|
836
|
+
g.output(g.iota([2, 3], axis: 1))
|
|
837
|
+
g.output(g.iota([2, 2, 2], axis: 1, dtype: :float32))
|
|
838
|
+
end.compile
|
|
839
|
+
|
|
840
|
+
rows, cols, middle = plan.run
|
|
841
|
+
Conformance.assert_equal_arrays([0, 0, 0, 1, 1, 1], rows.to_a)
|
|
842
|
+
Conformance.assert_equal_arrays([0, 1, 2, 0, 1, 2], cols.to_a)
|
|
843
|
+
Conformance.assert_close([0.0, 0.0, 1.0, 1.0, 0.0, 0.0, 1.0, 1.0], middle.to_a)
|
|
844
|
+
end
|
|
845
|
+
|
|
846
|
+
# reshape is a view, so the source buffer must live until the view's readers
|
|
847
|
+
# finish. Miscounting the lifetime returns memory still being read to the
|
|
848
|
+
# pool and the next writer — which shows when the same plan reruns, so run
|
|
849
|
+
# three times and expect the same values each time.
|
|
850
|
+
register(:graph_reshape_is_a_view_whose_source_outlives_its_readers) do
|
|
851
|
+
x = Conformance.nd([2, 2], [1.0, 2.0, 3.0, 4.0])
|
|
852
|
+
y = Tilework[Conformance.f32([10.0, 20.0, 30.0, 40.0])]
|
|
853
|
+
plan = Tilework::Graph.build do |g|
|
|
854
|
+
a = g.input(:x, [2, 2]) * 2.0
|
|
855
|
+
g.output((a.reshape(4) + g.param(y)) * 0.5)
|
|
856
|
+
g.output(a.reshape(1, 4))
|
|
857
|
+
end.compile
|
|
858
|
+
|
|
859
|
+
3.times do
|
|
860
|
+
summed, viewed = plan.run(x: x)
|
|
861
|
+
Conformance.assert_close([6.0, 12.0, 18.0, 24.0], summed.to_a)
|
|
862
|
+
Conformance.assert_close([2.0, 4.0, 6.0, 8.0], viewed.to_a)
|
|
863
|
+
Conformance.assert(viewed.shape == [1, 4], "reshape should be [1, 4], got #{viewed.shape.inspect}")
|
|
864
|
+
end
|
|
865
|
+
end
|
|
866
|
+
|
|
867
|
+
# Axis reductions. One thread folds in order along the axis, a different
|
|
868
|
+
# order from the tree-folded whole-buffer reduction — so the reference is a
|
|
869
|
+
# host fold in the same order, and the tolerance covers the accumulator gap
|
|
870
|
+
# (float on the dialect side, Ruby double on :cpu).
|
|
871
|
+
register(:graph_axis_reductions_match_a_host_computation) do
|
|
872
|
+
batch = 2
|
|
873
|
+
rows = 3
|
|
874
|
+
cols = 4
|
|
875
|
+
values = Array.new(batch * rows * cols) { |i| ((i % 11) - 5) * 0.375 }
|
|
876
|
+
x = Conformance.nd([batch, rows, cols], values)
|
|
877
|
+
flat = Conformance.f32(values)
|
|
878
|
+
|
|
879
|
+
plan = Tilework::Graph.build do |g|
|
|
880
|
+
t = g.input(:x, [batch, rows, cols])
|
|
881
|
+
g.output(t.sum(axis: 2))
|
|
882
|
+
g.output(t.max(axis: 1))
|
|
883
|
+
g.output(t.mean(axis: 0))
|
|
884
|
+
g.output(t.argmax(axis: 2))
|
|
885
|
+
end.compile
|
|
886
|
+
sums, maxes, means, positions = plan.run(x: x)
|
|
887
|
+
|
|
888
|
+
# The host-side answer, folding as [outer, axis, inner].
|
|
889
|
+
fold = lambda do |axis, &combine|
|
|
890
|
+
axis_len = [batch, rows, cols][axis]
|
|
891
|
+
inner = [batch, rows, cols].drop(axis + 1).inject(1, :*)
|
|
892
|
+
outer = [batch, rows, cols].take(axis).inject(1, :*)
|
|
893
|
+
(0...(outer * inner)).map do |i|
|
|
894
|
+
base = ((i / inner) * axis_len * inner) + (i % inner)
|
|
895
|
+
combine.call((0...axis_len).map { |t| flat[base + (t * inner)] })
|
|
896
|
+
end
|
|
897
|
+
end
|
|
898
|
+
|
|
899
|
+
Conformance.assert_close(fold.call(2) { |v| v.sum }, sums.to_a, 1e-4)
|
|
900
|
+
Conformance.assert_close(fold.call(1) { |v| v.max }, maxes.to_a, 1e-6)
|
|
901
|
+
Conformance.assert_close(fold.call(0) { |v| v.sum / v.length }, means.to_a, 1e-4)
|
|
902
|
+
Conformance.assert_equal_arrays(fold.call(2) { |v| v.index(v.max) }, positions.to_a)
|
|
903
|
+
Conformance.assert(sums.shape == [batch, rows],
|
|
904
|
+
"sum(axis: 2) should be #{[batch, rows].inspect}, got #{sums.shape.inspect}")
|
|
905
|
+
end
|
|
906
|
+
|
|
907
|
+
# Row-wise fusion agrees with the unfused path. The fold order and the
|
|
908
|
+
# write-back rounding match the axis-reduction kernel, so the tolerance is
|
|
909
|
+
# only for the same reason as elementwise fusion (FMA contraction).
|
|
910
|
+
register(:graph_row_wise_fusion_matches_the_unfused_path) do
|
|
911
|
+
rows = 256
|
|
912
|
+
cols = 8
|
|
913
|
+
x = Conformance.nd([rows, cols], Array.new(rows * cols) { |i| ((i % 13) - 6) * 0.5 })
|
|
914
|
+
build = lambda do
|
|
915
|
+
plan = Tilework::Graph.build do |g|
|
|
916
|
+
t = g.input(:x, [rows, cols])
|
|
917
|
+
# softmax in primitives; two reductions in a row.
|
|
918
|
+
e = (t - t.max(axis: -1, keepdims: true)).exp
|
|
919
|
+
g.output(e / e.sum(axis: -1, keepdims: true))
|
|
920
|
+
# The folded value itself is an output too (a reduction that is an output cannot join a cluster).
|
|
921
|
+
g.output(t.mean(axis: -1, keepdims: true))
|
|
922
|
+
end.compile
|
|
923
|
+
[plan, plan.run(x: x).map(&:to_a)]
|
|
924
|
+
end
|
|
925
|
+
|
|
926
|
+
previous = Tilework::Graph.row_fusion
|
|
927
|
+
begin
|
|
928
|
+
Tilework::Graph.row_fusion = true
|
|
929
|
+
fused_plan, fused = build.call
|
|
930
|
+
Tilework::Graph.row_fusion = false
|
|
931
|
+
plain_plan, plain = build.call
|
|
932
|
+
ensure
|
|
933
|
+
Tilework::Graph.row_fusion = previous
|
|
934
|
+
end
|
|
935
|
+
|
|
936
|
+
Conformance.assert(fused_plan.fusion_stats[:nodes] < plain_plan.fusion_stats[:nodes],
|
|
937
|
+
"row fusion should launch fewer kernels than the unfused path")
|
|
938
|
+
fused.zip(plain).each { |a, b| Conformance.assert_close(b, a, 1e-6) }
|
|
939
|
+
end
|
|
940
|
+
|
|
941
|
+
register(:graph_permute_reorders_axes) do
|
|
942
|
+
values = Array.new(24) { |i| i.to_f }
|
|
943
|
+
x = Conformance.nd([2, 3, 4], values)
|
|
944
|
+
plan = Tilework::Graph.build do |g|
|
|
945
|
+
t = g.input(:x, [2, 3, 4])
|
|
946
|
+
g.output(t.permute(1, 0, 2))
|
|
947
|
+
g.output(t.permute(2, 1, 0))
|
|
948
|
+
end.compile
|
|
949
|
+
|
|
950
|
+
swapped, reversed = plan.run(x: x)
|
|
951
|
+
Conformance.assert(swapped.shape == [3, 2, 4], "permute(1, 0, 2) should be [3, 2, 4]")
|
|
952
|
+
Conformance.assert(reversed.shape == [4, 3, 2], "permute(2, 1, 0) should be [4, 3, 2]")
|
|
953
|
+
at = ->(i, j, k) { values[(i * 12) + (j * 4) + k] }
|
|
954
|
+
Conformance.assert_close((0...3).flat_map { |j| (0...2).flat_map { |i| (0...4).map { |k| at.(i, j, k) } } },
|
|
955
|
+
swapped.to_a)
|
|
956
|
+
Conformance.assert_close((0...4).flat_map { |k| (0...3).flat_map { |j| (0...2).map { |i| at.(i, j, k) } } },
|
|
957
|
+
reversed.to_a)
|
|
958
|
+
end
|
|
959
|
+
|
|
960
|
+
# Table lookup. Out-of-range indices are clamped to the ends — what an
|
|
961
|
+
# out-of-range read does on a GPU is undefined, so clamping was chosen over
|
|
962
|
+
# breaking silently.
|
|
963
|
+
register(:graph_gather_reads_along_an_axis) do
|
|
964
|
+
table = Conformance.nd([4, 2], [10.0, 11.0, 20.0, 21.0, 30.0, 31.0, 40.0, 41.0])
|
|
965
|
+
ids = Buffer.new(:int32, 4, shape: [4])
|
|
966
|
+
ids.write([2, 0, 3, 99])
|
|
967
|
+
plan = Tilework::Graph.build do |g|
|
|
968
|
+
g.output(g.gather(g.param(table), g.param(ids)))
|
|
969
|
+
end.compile
|
|
970
|
+
|
|
971
|
+
rows = plan.run
|
|
972
|
+
Conformance.assert(rows.shape == [4, 2], "gather should be [4, 2], got #{rows.shape.inspect}")
|
|
973
|
+
Conformance.assert_close([30.0, 31.0, 10.0, 11.0, 40.0, 41.0, 40.0, 41.0], rows.to_a)
|
|
974
|
+
end
|
|
975
|
+
|
|
976
|
+
# Border fill, with different before/after amounts per dim, covering inner and outer dims.
|
|
977
|
+
register(:graph_pad_fills_the_border) do
|
|
978
|
+
x = Conformance.nd([2, 3], [1.0, 2.0, 3.0, 4.0, 5.0, 6.0])
|
|
979
|
+
plan = Tilework::Graph.build do |g|
|
|
980
|
+
g.output(g.pad(g.input(:x, [2, 3]), before: [1, 1], after: [0, 2], value: -1.0))
|
|
981
|
+
end.compile
|
|
982
|
+
|
|
983
|
+
padded = plan.run(x: x)
|
|
984
|
+
Conformance.assert(padded.shape == [3, 6], "pad should be [3, 6], got #{padded.shape.inspect}")
|
|
985
|
+
Conformance.assert_close([-1.0, -1.0, -1.0, -1.0, -1.0, -1.0,
|
|
986
|
+
-1.0, 1.0, 2.0, 3.0, -1.0, -1.0,
|
|
987
|
+
-1.0, 4.0, 5.0, 6.0, -1.0, -1.0], padded.to_a)
|
|
988
|
+
end
|
|
989
|
+
|
|
990
|
+
# Concatenating along an inner axis scatters pieces, so it takes a different path from the outermost dim.
|
|
991
|
+
register(:graph_concat_joins_along_an_inner_axis) do
|
|
992
|
+
a = Conformance.nd([2, 2], [1.0, 2.0, 3.0, 4.0])
|
|
993
|
+
b = Conformance.nd([2, 1], [5.0, 6.0])
|
|
994
|
+
plan = Tilework::Graph.build do |g|
|
|
995
|
+
g.output(g.concat(g.input(:a, [2, 2]), g.param(b), axis: 1))
|
|
996
|
+
end.compile
|
|
997
|
+
|
|
998
|
+
Conformance.assert_close([1.0, 2.0, 5.0, 3.0, 4.0, 6.0], plan.run(a: a).to_a)
|
|
999
|
+
end
|
|
1000
|
+
|
|
1001
|
+
# An outermost-dim slice is a view (no kernel). Returning the source to the
|
|
1002
|
+
# pool early corrupts values, so like reshape it is rerun to check.
|
|
1003
|
+
register(:graph_slice_is_a_view_of_the_outermost_dimension) do
|
|
1004
|
+
x = Conformance.nd([3, 2], [1.0, 2.0, 3.0, 4.0, 5.0, 6.0])
|
|
1005
|
+
plan = Tilework::Graph.build do |g|
|
|
1006
|
+
rows = g.input(:x, [3, 2]) * 2.0
|
|
1007
|
+
g.output(rows.slice(1..2) + 1.0)
|
|
1008
|
+
g.output(rows.slice(0))
|
|
1009
|
+
end.compile
|
|
1010
|
+
|
|
1011
|
+
3.times do
|
|
1012
|
+
tail, head = plan.run(x: x)
|
|
1013
|
+
Conformance.assert(tail.shape == [2, 2], "slice(1..2) should be [2, 2], got #{tail.shape.inspect}")
|
|
1014
|
+
Conformance.assert_close([7.0, 9.0, 11.0, 13.0], tail.to_a)
|
|
1015
|
+
Conformance.assert_close([2.0, 4.0], head.to_a)
|
|
1016
|
+
end
|
|
1017
|
+
end
|
|
1018
|
+
|
|
1019
|
+
# Convolution. Delegated to cuDNN / MPSGraph when present, otherwise (:cpu)
|
|
1020
|
+
# expanded into im2col + matmul at record time. It is compared with a
|
|
1021
|
+
# direct convolution on the host, so both paths face the same check.
|
|
1022
|
+
register(:graph_conv2d_matches_a_direct_convolution) do
|
|
1023
|
+
n = 2
|
|
1024
|
+
cin = 2
|
|
1025
|
+
h = 5
|
|
1026
|
+
w = 4
|
|
1027
|
+
cout = 3
|
|
1028
|
+
kh = 3
|
|
1029
|
+
kw = 2
|
|
1030
|
+
pad = [1, 0]
|
|
1031
|
+
stride = [2, 1]
|
|
1032
|
+
x = Conformance.nd([n, cin, h, w], Array.new(n * cin * h * w) { |i| ((i % 11) - 5) * 0.25 })
|
|
1033
|
+
weight = Conformance.nd([cout, cin, kh, kw], Array.new(cout * cin * kh * kw) { |i| ((i % 7) - 3) * 0.125 })
|
|
1034
|
+
|
|
1035
|
+
plan = Tilework::Graph.build do |g|
|
|
1036
|
+
g.output(Conv.forward(g.input(:x, [n, cin, h, w]), g.param(weight), pad: pad, stride: stride))
|
|
1037
|
+
end.compile
|
|
1038
|
+
out = plan.run(x: x)
|
|
1039
|
+
|
|
1040
|
+
oh = Conv.output_size(h, kh, pad[0], stride[0])
|
|
1041
|
+
ow = Conv.output_size(w, kw, pad[1], stride[1])
|
|
1042
|
+
Conformance.assert(out.shape == [n, cout, oh, ow],
|
|
1043
|
+
"conv2d should be #{[n, cout, oh, ow].inspect}, got #{out.shape.inspect}")
|
|
1044
|
+
|
|
1045
|
+
xs = x.to_a
|
|
1046
|
+
ws = weight.to_a
|
|
1047
|
+
expected = (0...n).flat_map do |b|
|
|
1048
|
+
(0...cout).flat_map do |co|
|
|
1049
|
+
(0...oh).flat_map do |y|
|
|
1050
|
+
(0...ow).map do |t|
|
|
1051
|
+
(0...cin).sum do |ci|
|
|
1052
|
+
(0...kh).sum do |ky|
|
|
1053
|
+
(0...kw).sum do |kx|
|
|
1054
|
+
iy = (y * stride[0]) + ky - pad[0]
|
|
1055
|
+
ix = (t * stride[1]) + kx - pad[1]
|
|
1056
|
+
next 0.0 if iy.negative? || iy >= h || ix.negative? || ix >= w
|
|
1057
|
+
|
|
1058
|
+
xs[(((b * cin) + ci) * h * w) + (iy * w) + ix] *
|
|
1059
|
+
ws[(((co * cin) + ci) * kh * kw) + (ky * kw) + kx]
|
|
1060
|
+
end
|
|
1061
|
+
end
|
|
1062
|
+
end
|
|
1063
|
+
end
|
|
1064
|
+
end
|
|
1065
|
+
end
|
|
1066
|
+
end
|
|
1067
|
+
Conformance.assert_close(expected, out.to_a, 1e-4)
|
|
1068
|
+
end
|
|
1069
|
+
|
|
1070
|
+
# The same check with dilation; the only difference is reading taps with
|
|
1071
|
+
# (dilation - 1) gaps — Conv.output_size computes the effective kernel width.
|
|
1072
|
+
register(:graph_conv2d_dilation_matches_a_direct_convolution) do
|
|
1073
|
+
n = 1
|
|
1074
|
+
cin = 2
|
|
1075
|
+
h = 8
|
|
1076
|
+
w = 7
|
|
1077
|
+
cout = 3
|
|
1078
|
+
kh = 3
|
|
1079
|
+
kw = 3
|
|
1080
|
+
pad = [2, 1]
|
|
1081
|
+
stride = [1, 2]
|
|
1082
|
+
dilation = [2, 3]
|
|
1083
|
+
x = Conformance.nd([n, cin, h, w], Array.new(n * cin * h * w) { |i| ((i % 11) - 5) * 0.25 })
|
|
1084
|
+
weight = Conformance.nd([cout, cin, kh, kw], Array.new(cout * cin * kh * kw) { |i| ((i % 7) - 3) * 0.125 })
|
|
1085
|
+
|
|
1086
|
+
plan = Tilework::Graph.build do |g|
|
|
1087
|
+
g.output(Conv.forward(g.input(:x, [n, cin, h, w]), g.param(weight),
|
|
1088
|
+
pad: pad, stride: stride, dilation: dilation))
|
|
1089
|
+
end.compile
|
|
1090
|
+
out = plan.run(x: x)
|
|
1091
|
+
|
|
1092
|
+
oh = Conv.output_size(h, kh, pad[0], stride[0], dilation[0])
|
|
1093
|
+
ow = Conv.output_size(w, kw, pad[1], stride[1], dilation[1])
|
|
1094
|
+
Conformance.assert(out.shape == [n, cout, oh, ow],
|
|
1095
|
+
"dilated conv2d should be #{[n, cout, oh, ow].inspect}, got #{out.shape.inspect}")
|
|
1096
|
+
|
|
1097
|
+
xs = x.to_a
|
|
1098
|
+
ws = weight.to_a
|
|
1099
|
+
expected = (0...n).flat_map do |b|
|
|
1100
|
+
(0...cout).flat_map do |co|
|
|
1101
|
+
(0...oh).flat_map do |y|
|
|
1102
|
+
(0...ow).map do |t|
|
|
1103
|
+
(0...cin).sum do |ci|
|
|
1104
|
+
(0...kh).sum do |ky|
|
|
1105
|
+
(0...kw).sum do |kx|
|
|
1106
|
+
iy = (y * stride[0]) + (ky * dilation[0]) - pad[0]
|
|
1107
|
+
ix = (t * stride[1]) + (kx * dilation[1]) - pad[1]
|
|
1108
|
+
next 0.0 if iy.negative? || iy >= h || ix.negative? || ix >= w
|
|
1109
|
+
|
|
1110
|
+
xs[(((b * cin) + ci) * h * w) + (iy * w) + ix] *
|
|
1111
|
+
ws[(((co * cin) + ci) * kh * kw) + (ky * kw) + kx]
|
|
1112
|
+
end
|
|
1113
|
+
end
|
|
1114
|
+
end
|
|
1115
|
+
end
|
|
1116
|
+
end
|
|
1117
|
+
end
|
|
1118
|
+
end
|
|
1119
|
+
Conformance.assert_close(expected, out.to_a, 1e-4)
|
|
1120
|
+
end
|
|
1121
|
+
|
|
1122
|
+
# groups > 1 (including depthwise). Each output channel sees only its own
|
|
1123
|
+
# group's input channel range, and the reference sums only that range.
|
|
1124
|
+
register(:graph_conv2d_groups_matches_a_direct_convolution) do
|
|
1125
|
+
n = 1
|
|
1126
|
+
cin = 4
|
|
1127
|
+
h = 6
|
|
1128
|
+
w = 5
|
|
1129
|
+
cout = 4
|
|
1130
|
+
kh = 3
|
|
1131
|
+
kw = 3
|
|
1132
|
+
pad = [1, 1]
|
|
1133
|
+
stride = [1, 1]
|
|
1134
|
+
groups = 4 # depthwise: one input channel each
|
|
1135
|
+
cin_per_group = cin / groups
|
|
1136
|
+
cout_per_group = cout / groups
|
|
1137
|
+
x = Conformance.nd([n, cin, h, w], Array.new(n * cin * h * w) { |i| ((i % 11) - 5) * 0.25 })
|
|
1138
|
+
weight = Conformance.nd([cout, cin_per_group, kh, kw],
|
|
1139
|
+
Array.new(cout * cin_per_group * kh * kw) { |i| ((i % 7) - 3) * 0.125 })
|
|
1140
|
+
|
|
1141
|
+
plan = Tilework::Graph.build do |g|
|
|
1142
|
+
g.output(Conv.forward(g.input(:x, [n, cin, h, w]), g.param(weight),
|
|
1143
|
+
pad: pad, stride: stride, groups: groups))
|
|
1144
|
+
end.compile
|
|
1145
|
+
out = plan.run(x: x)
|
|
1146
|
+
|
|
1147
|
+
oh = Conv.output_size(h, kh, pad[0], stride[0])
|
|
1148
|
+
ow = Conv.output_size(w, kw, pad[1], stride[1])
|
|
1149
|
+
Conformance.assert(out.shape == [n, cout, oh, ow],
|
|
1150
|
+
"grouped conv2d should be #{[n, cout, oh, ow].inspect}, got #{out.shape.inspect}")
|
|
1151
|
+
|
|
1152
|
+
xs = x.to_a
|
|
1153
|
+
ws = weight.to_a
|
|
1154
|
+
expected = (0...n).flat_map do |b|
|
|
1155
|
+
(0...cout).flat_map do |co|
|
|
1156
|
+
gi = co / cout_per_group
|
|
1157
|
+
(0...oh).flat_map do |y|
|
|
1158
|
+
(0...ow).map do |t|
|
|
1159
|
+
(0...cin_per_group).sum do |ci|
|
|
1160
|
+
(0...kh).sum do |ky|
|
|
1161
|
+
(0...kw).sum do |kx|
|
|
1162
|
+
iy = (y * stride[0]) + ky - pad[0]
|
|
1163
|
+
ix = (t * stride[1]) + kx - pad[1]
|
|
1164
|
+
next 0.0 if iy.negative? || iy >= h || ix.negative? || ix >= w
|
|
1165
|
+
|
|
1166
|
+
xs[(((b * cin) + (gi * cin_per_group) + ci) * h * w) + (iy * w) + ix] *
|
|
1167
|
+
ws[(((co * cin_per_group) + ci) * kh * kw) + (ky * kw) + kx]
|
|
1168
|
+
end
|
|
1169
|
+
end
|
|
1170
|
+
end
|
|
1171
|
+
end
|
|
1172
|
+
end
|
|
1173
|
+
end
|
|
1174
|
+
end
|
|
1175
|
+
Conformance.assert_close(expected, out.to_a, 1e-4)
|
|
1176
|
+
end
|
|
1177
|
+
|
|
1178
|
+
# General groups (not depthwise: groups == 2 with several channels per
|
|
1179
|
+
# group) are covered by the same form of check.
|
|
1180
|
+
register(:graph_conv2d_general_groups_matches_a_direct_convolution) do
|
|
1181
|
+
n = 1
|
|
1182
|
+
cin = 4
|
|
1183
|
+
h = 6
|
|
1184
|
+
w = 6
|
|
1185
|
+
cout = 6
|
|
1186
|
+
kh = 3
|
|
1187
|
+
kw = 3
|
|
1188
|
+
pad = [1, 1]
|
|
1189
|
+
stride = [1, 1]
|
|
1190
|
+
groups = 2
|
|
1191
|
+
cin_per_group = cin / groups
|
|
1192
|
+
cout_per_group = cout / groups
|
|
1193
|
+
x = Conformance.nd([n, cin, h, w], Array.new(n * cin * h * w) { |i| ((i % 13) - 6) * 0.2 })
|
|
1194
|
+
weight = Conformance.nd([cout, cin_per_group, kh, kw],
|
|
1195
|
+
Array.new(cout * cin_per_group * kh * kw) { |i| ((i % 9) - 4) * 0.1 })
|
|
1196
|
+
|
|
1197
|
+
plan = Tilework::Graph.build do |g|
|
|
1198
|
+
g.output(Conv.forward(g.input(:x, [n, cin, h, w]), g.param(weight),
|
|
1199
|
+
pad: pad, stride: stride, groups: groups))
|
|
1200
|
+
end.compile
|
|
1201
|
+
out = plan.run(x: x)
|
|
1202
|
+
|
|
1203
|
+
oh = Conv.output_size(h, kh, pad[0], stride[0])
|
|
1204
|
+
ow = Conv.output_size(w, kw, pad[1], stride[1])
|
|
1205
|
+
|
|
1206
|
+
xs = x.to_a
|
|
1207
|
+
ws = weight.to_a
|
|
1208
|
+
expected = (0...n).flat_map do |b|
|
|
1209
|
+
(0...cout).flat_map do |co|
|
|
1210
|
+
gi = co / cout_per_group
|
|
1211
|
+
(0...oh).flat_map do |y|
|
|
1212
|
+
(0...ow).map do |t|
|
|
1213
|
+
(0...cin_per_group).sum do |ci|
|
|
1214
|
+
(0...kh).sum do |ky|
|
|
1215
|
+
(0...kw).sum do |kx|
|
|
1216
|
+
iy = (y * stride[0]) + ky - pad[0]
|
|
1217
|
+
ix = (t * stride[1]) + kx - pad[1]
|
|
1218
|
+
next 0.0 if iy.negative? || iy >= h || ix.negative? || ix >= w
|
|
1219
|
+
|
|
1220
|
+
xs[(((b * cin) + (gi * cin_per_group) + ci) * h * w) + (iy * w) + ix] *
|
|
1221
|
+
ws[(((co * cin_per_group) + ci) * kh * kw) + (ky * kw) + kx]
|
|
1222
|
+
end
|
|
1223
|
+
end
|
|
1224
|
+
end
|
|
1225
|
+
end
|
|
1226
|
+
end
|
|
1227
|
+
end
|
|
1228
|
+
end
|
|
1229
|
+
Conformance.assert_close(expected, out.to_a, 1e-4)
|
|
1230
|
+
end
|
|
1231
|
+
|
|
1232
|
+
# High-level ops agree with their decompositions (Tier 1 in docs/graph-ir.md).
|
|
1233
|
+
#
|
|
1234
|
+
# Bit identity is not available — the decomposition reduces in a different
|
|
1235
|
+
# order and accumulates in a different type than the hand-written kernel.
|
|
1236
|
+
# Measured on :cpu the gap is about 1e-7; the tolerance is 1e-5 (for
|
|
1237
|
+
# backend exp / tanh differences and FMA contraction). A wrong axis, a
|
|
1238
|
+
# dropped gamma and a missing square are all caught. Differences like moving
|
|
1239
|
+
# eps (1e-6) outside the square root fall below that, so **those are not checked**.
|
|
1240
|
+
register(:graph_high_level_ops_match_their_decomposition) do
|
|
1241
|
+
rows = 3
|
|
1242
|
+
cols = 8
|
|
1243
|
+
values = Array.new(rows * cols) { |i| ((i % 13) - 6) * 0.5 }
|
|
1244
|
+
x = Conformance.nd([rows, cols], values)
|
|
1245
|
+
gamma = Tilework[Conformance.f32(Array.new(cols) { |i| 0.5 + (i * 0.25) })]
|
|
1246
|
+
beta = Tilework[Conformance.f32(Array.new(cols) { |i| (i % 3) - 1.0 })]
|
|
1247
|
+
kv = Conformance.nd([5, cols], Array.new(5 * cols) { |i| ((i % 7) - 3) * 0.25 })
|
|
1248
|
+
target = Conformance.nd([rows, cols], Array.new(rows * cols) { |i| ((i % 5) - 2) * 0.5 })
|
|
1249
|
+
labels = Buffer.new(:int32, rows, shape: [rows])
|
|
1250
|
+
labels.write(Array.new(rows) { |i| (i * 3) % cols })
|
|
1251
|
+
|
|
1252
|
+
build = lambda do
|
|
1253
|
+
plan = Tilework::Graph.build do |g|
|
|
1254
|
+
t = g.input(:x, [rows, cols])
|
|
1255
|
+
g.output(NN.softmax(t))
|
|
1256
|
+
g.output(NN.layernorm(t, gamma: g.param(gamma), beta: g.param(beta)))
|
|
1257
|
+
g.output(NN.layernorm(t)) # the path without affine transform
|
|
1258
|
+
g.output(NN.rmsnorm(t, gamma: g.param(gamma)))
|
|
1259
|
+
g.output(NN.relu(t))
|
|
1260
|
+
g.output(NN.sigmoid(t))
|
|
1261
|
+
g.output(NN.silu(t))
|
|
1262
|
+
g.output(NN.gelu(t))
|
|
1263
|
+
g.output(NN.transpose(t))
|
|
1264
|
+
# Needs head splitting, axis permutation and batched matmul all at once.
|
|
1265
|
+
g.output(NN.multi_head_attention(t, g.param(kv), g.param(kv), heads: 2))
|
|
1266
|
+
# A loss. It ends in a reduction, so a decomposition with the wrong axis
|
|
1267
|
+
# or divisor shows a difference once folded into one number.
|
|
1268
|
+
g.output(NN.mse(t, g.param(target)))
|
|
1269
|
+
g.output(NN.cross_entropy(t, g.param(labels)))
|
|
1270
|
+
g.output(NN.cross_entropy(t, g.param(labels), reduction: :none))
|
|
1271
|
+
end.compile
|
|
1272
|
+
[plan, plan.run(x: x).map(&:to_a)]
|
|
1273
|
+
end
|
|
1274
|
+
|
|
1275
|
+
previous = Tilework::Graph.lower_high_level
|
|
1276
|
+
begin
|
|
1277
|
+
Tilework::Graph.lower_high_level = false
|
|
1278
|
+
fused_plan, fused = build.call
|
|
1279
|
+
Tilework::Graph.lower_high_level = true
|
|
1280
|
+
lowered_plan, lowered = build.call
|
|
1281
|
+
ensure
|
|
1282
|
+
Tilework::Graph.lower_high_level = previous
|
|
1283
|
+
end
|
|
1284
|
+
|
|
1285
|
+
Conformance.assert(fused_plan.nodes.count { |n| n.op == :nn } == 13,
|
|
1286
|
+
"the default path should record 13 :nn nodes")
|
|
1287
|
+
Conformance.assert(lowered_plan.nodes.none? { |n| n.op == :nn },
|
|
1288
|
+
"the lowered path should have no :nn node left")
|
|
1289
|
+
names = %w[softmax layernorm layernorm_plain rmsnorm relu sigmoid silu gelu
|
|
1290
|
+
transpose multi_head_attention mse cross_entropy cross_entropy_rows]
|
|
1291
|
+
names.each_with_index do |name, i|
|
|
1292
|
+
begin
|
|
1293
|
+
Conformance.assert_close(fused[i], lowered[i], 1e-5)
|
|
1294
|
+
rescue RuntimeError => e
|
|
1295
|
+
raise "#{name}: the decomposition disagrees with the kernel (#{e.message})"
|
|
1296
|
+
end
|
|
1297
|
+
end
|
|
1298
|
+
end
|
|
1299
|
+
|
|
1300
|
+
# softmax written in primitives alone agrees with the hand-written fused
|
|
1301
|
+
# kernel (NN.softmax) — the smallest evidence of Tier 1 in docs/graph-ir.md.
|
|
1302
|
+
register(:graph_softmax_in_primitives_matches_the_nn_kernel) do
|
|
1303
|
+
rows = 4
|
|
1304
|
+
cols = 8
|
|
1305
|
+
x = Conformance.nd([rows, cols], Array.new(rows * cols) { |i| ((i % 13) - 6) * 0.5 })
|
|
1306
|
+
plan = Tilework::Graph.build do |g|
|
|
1307
|
+
t = g.input(:x, [rows, cols])
|
|
1308
|
+
e = (t - t.max(axis: -1, keepdims: true)).exp
|
|
1309
|
+
g.output(e / e.sum(axis: -1, keepdims: true))
|
|
1310
|
+
end.compile
|
|
1311
|
+
|
|
1312
|
+
Conformance.assert_close(NN.softmax(x).to_a, plan.run(x: x).to_a, 1e-5)
|
|
1313
|
+
end
|
|
1314
|
+
|
|
1315
|
+
# Concatenation just lays pieces along the outermost dim. If a piece were not
|
|
1316
|
+
# contiguous in the output its contents would swap, so values themselves are checked.
|
|
1317
|
+
register(:graph_concat_joins_along_the_outermost_dimension) do
|
|
1318
|
+
a = Tilework[Conformance.f32([1.0, 2.0, 3.0, 4.0])].reshape(2, 2)
|
|
1319
|
+
b = Tilework[Conformance.f32([5.0, 6.0])].reshape(1, 2)
|
|
1320
|
+
plan = Tilework::Graph.build do |g|
|
|
1321
|
+
g.output(g.concat(g.input(:a, [2, 2]), g.param(b)))
|
|
1322
|
+
end.compile
|
|
1323
|
+
|
|
1324
|
+
Conformance.assert_equal_arrays([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], plan.run(a: a).to_a)
|
|
1325
|
+
end
|
|
1326
|
+
|
|
1327
|
+
# Row-wise reductions look only at the last dim, so 3D and up give the same
|
|
1328
|
+
# answer as the 2D with leading dims folded; the dims must also come back intact.
|
|
1329
|
+
register(:nn_row_wise_ops_treat_leading_dims_as_rows) do
|
|
1330
|
+
values = Array.new(12) { |i| ((i % 4) - 1.5) * 0.75 }
|
|
1331
|
+
cube = Conformance.nd([3, 2, 2], values)
|
|
1332
|
+
flat = Conformance.nn_matrix(6, 2, Conformance.f32(values))
|
|
1333
|
+
|
|
1334
|
+
out = NN.softmax(cube)
|
|
1335
|
+
Conformance.assert(out.shape == [3, 2, 2], "softmax should keep #{cube.shape.inspect}, got #{out.shape.inspect}")
|
|
1336
|
+
Conformance.assert_equal_arrays(NN.softmax(flat).to_a, out.to_a)
|
|
1337
|
+
Conformance.assert_equal_arrays(NN.rmsnorm(flat).to_a, NN.rmsnorm(cube).to_a)
|
|
1338
|
+
end
|
|
1339
|
+
|
|
1340
|
+
# Fused elementwise ops agree with unfused execution.
|
|
1341
|
+
#
|
|
1342
|
+
# Fusion wraps each expression-tree node in a cast to its output type,
|
|
1343
|
+
# reproducing each node's write-back exactly. So **the arithmetic is
|
|
1344
|
+
# identical**, and :cpu agrees bit for bit (graph_cpu_test.rb checks that).
|
|
1345
|
+
#
|
|
1346
|
+
# The tolerance here exists for one reason only: a shader compiler may
|
|
1347
|
+
# contract `a * b + c` within one expression into a single FMA, which
|
|
1348
|
+
# differs in the low bits from two unfused kernels (the contracted one is
|
|
1349
|
+
# actually more accurate). There is no portable way to forbid contraction,
|
|
1350
|
+
# so the tolerance passes identical arithmetic but fails a wrong order or
|
|
1351
|
+
# truncation.
|
|
1352
|
+
#
|
|
1353
|
+
# int32 and comparisons are mixed in because integer truncation is easy to
|
|
1354
|
+
# lose in fusion (unnoticeable when looking at floats only).
|
|
1355
|
+
register(:graph_fused_elementwise_matches_the_unfused_path) do
|
|
1356
|
+
values = Conformance.f32(Array.new(256) { |i| ((i % 17) - 8) * 0.375 })
|
|
1357
|
+
bias = Tilework[Conformance.f32(Array.new(16) { |i| ((i % 5) - 2) * 0.5 })]
|
|
1358
|
+
build = lambda do
|
|
1359
|
+
plan = Tilework::Graph.build do |g|
|
|
1360
|
+
x = g.input(:x, [256])
|
|
1361
|
+
v = x
|
|
1362
|
+
4.times { v = ((v * 1.5) + 0.25).tanh.abs }
|
|
1363
|
+
# Mix a round trip through integers with a 0/1 mask
|
|
1364
|
+
ints = g.input(:i, [256], dtype: :int32)
|
|
1365
|
+
# cast and where are expression-tree nodes too, so the same property
|
|
1366
|
+
# (fusion does not change values) is pinned for them here.
|
|
1367
|
+
v = g.where(v > 0.5, v * 2.0, v.cast(:int32).cast(:float32))
|
|
1368
|
+
v += ((ints * 3) / 2).eq(4)
|
|
1369
|
+
# Fused, a broadcast folds into **an index**; unfused, it writes out the
|
|
1370
|
+
# stretched values. The values read are the same, so these must agree too.
|
|
1371
|
+
v = v.reshape(16, 16) + g.param(bias)
|
|
1372
|
+
g.output(v.reshape(256))
|
|
1373
|
+
end.compile
|
|
1374
|
+
plan.run(x: Tilework[values], i: Tilework::Buffer.from(Array.new(256) { |i| i % 5 }))
|
|
1375
|
+
end
|
|
1376
|
+
|
|
1377
|
+
previous = Tilework::Graph.fusion
|
|
1378
|
+
begin
|
|
1379
|
+
Tilework::Graph.fusion = true
|
|
1380
|
+
fused = build.call.to_a
|
|
1381
|
+
Tilework::Graph.fusion = false
|
|
1382
|
+
plain = build.call.to_a
|
|
1383
|
+
ensure
|
|
1384
|
+
Tilework::Graph.fusion = previous
|
|
1385
|
+
end
|
|
1386
|
+
Conformance.assert_close(plain, fused, 1e-6)
|
|
1387
|
+
end
|
|
1388
|
+
|
|
1389
|
+
# NN's fused kernels run the same function in a graph (no graph-specific
|
|
1390
|
+
# implementation), so this requires bit identity, not closeness.
|
|
1391
|
+
register(:graph_nn_matches_the_eager_path) do
|
|
1392
|
+
x = Tilework[Conformance.f32(Array.new(24) { |i| ((i % 7) - 3) * 0.5 })].reshape(4, 6)
|
|
1393
|
+
gamma = Tilework[Conformance.f32(Array.new(6) { |i| 1.0 + (i * 0.1) })]
|
|
1394
|
+
|
|
1395
|
+
plan = Tilework::Graph.build do |g|
|
|
1396
|
+
t = g.input(:x, [4, 6])
|
|
1397
|
+
# gamma can be passed as a Buffer (lifted like g.param).
|
|
1398
|
+
h = Tilework::NN.rmsnorm(t, gamma: gamma)
|
|
1399
|
+
g.output(Tilework::NN.softmax(Tilework::NN.gelu(h) + 1.0))
|
|
1400
|
+
g.output(Tilework::NN.transpose(h))
|
|
1401
|
+
end.compile
|
|
1402
|
+
|
|
1403
|
+
probs, transposed = plan.run(x: x)
|
|
1404
|
+
expected = Tilework::NN.rmsnorm(x, gamma: gamma)
|
|
1405
|
+
Conformance.assert_equal_arrays(Tilework::NN.softmax(Tilework::NN.gelu(expected) + 1.0).to_a, probs.to_a)
|
|
1406
|
+
Conformance.assert_equal_arrays(Tilework::NN.transpose(expected).to_a, transposed.to_a)
|
|
1407
|
+
end
|
|
1408
|
+
|
|
1409
|
+
# With gemm_library, attention is delegated to a vendor library, so inside a
|
|
1410
|
+
# graph it waits on both sides (like matmul). Whether those waits are right
|
|
1411
|
+
# is checked by agreeing with eager while running asynchronously.
|
|
1412
|
+
register(:graph_attention_matches_the_eager_path) do
|
|
1413
|
+
heads = 2
|
|
1414
|
+
n = 3
|
|
1415
|
+
d = 4
|
|
1416
|
+
q = Conformance.nn_matrix(n, d, Conformance.f32(Array.new(n * d) { |i| ((i % 5) - 2) * 0.25 }))
|
|
1417
|
+
k = Conformance.nn_matrix(n, d, Conformance.f32(Array.new(n * d) { |i| ((i % 3) - 1) * 0.5 }))
|
|
1418
|
+
v = Conformance.nn_matrix(n, d, Conformance.f32(Array.new(n * d) { |i| (i % 4) * 0.125 }))
|
|
1419
|
+
|
|
1420
|
+
plan = Tilework::Graph.build do |g|
|
|
1421
|
+
qt = g.input(:q, [n, d])
|
|
1422
|
+
g.output(Tilework::NN.multi_head_attention(qt, g.param(k), g.param(v), heads: heads))
|
|
1423
|
+
end.compile
|
|
1424
|
+
|
|
1425
|
+
expected = Tilework::NN.multi_head_attention(q, k, v, heads: heads).to_a
|
|
1426
|
+
Conformance.assert_equal_arrays(expected, plan.run(q: q).to_a)
|
|
1427
|
+
end
|
|
1428
|
+
|
|
1429
|
+
# RNG inside a graph **depends only on the seed**. The key comes from (plan
|
|
1430
|
+
# seed, the node's static index), so it depends neither on execution order
|
|
1431
|
+
# nor on run count — the basis for not making RNG mutable state; breaking it
|
|
1432
|
+
# brings back bugs where async execution changes the sequence.
|
|
1433
|
+
register(:graph_random_is_pure_in_the_plan_seed) do
|
|
1434
|
+
plan = Tilework::Graph.build do |g|
|
|
1435
|
+
g.output(g.random([64]))
|
|
1436
|
+
g.output(g.randn([64]))
|
|
1437
|
+
end.compile
|
|
1438
|
+
|
|
1439
|
+
first = plan.run(seed: 1234).map(&:to_a)
|
|
1440
|
+
again = plan.run(seed: 1234).map(&:to_a)
|
|
1441
|
+
advanced = plan.run(seed: plan.next_seed(1234)).map(&:to_a)
|
|
1442
|
+
|
|
1443
|
+
Conformance.assert_equal_arrays(first[0], again[0])
|
|
1444
|
+
Conformance.assert_equal_arrays(first[1], again[1])
|
|
1445
|
+
Conformance.assert(first[0] != first[1], "two draws in one plan must not share a sequence")
|
|
1446
|
+
Conformance.assert(first[0] != advanced[0], "advancing the seed must change the sequence")
|
|
1447
|
+
end
|
|
1448
|
+
|
|
1449
|
+
# Placement (writing a write_into value straight into the state's view) is
|
|
1450
|
+
# bit-identical to producing the value and copying it. A mismatch means the
|
|
1451
|
+
# write position or the write/read order is broken.
|
|
1452
|
+
register(:graph_placed_writes_match_the_copying_path) do
|
|
1453
|
+
weight = Tilework[Conformance.f32([2.0, 0.0, 0.0, 3.0])].reshape(2, 2)
|
|
1454
|
+
run = lambda do
|
|
1455
|
+
cache = Tilework[Conformance.f32(Array.new(8, 0.0))].reshape(4, 2)
|
|
1456
|
+
plan = Tilework::Graph.build do |g|
|
|
1457
|
+
kv0 = g.state(cache, name: :kv)
|
|
1458
|
+
k = g.input(:k, [1, 2])
|
|
1459
|
+
kv1 = g.write_into(kv0, at: g.index(:pos), value: k.matmul(g.param(weight)))
|
|
1460
|
+
g.output(g.read(kv1) * 1.0)
|
|
1461
|
+
end.compile
|
|
1462
|
+
3.times { |step| plan.run(k: Tilework[Conformance.f32([step + 1.0, 1.0])].reshape(1, 2), pos: step) }
|
|
1463
|
+
cache.to_a
|
|
1464
|
+
end
|
|
1465
|
+
|
|
1466
|
+
previous = Tilework::Graph.placement
|
|
1467
|
+
begin
|
|
1468
|
+
Tilework::Graph.placement = true
|
|
1469
|
+
placed = run.call
|
|
1470
|
+
Tilework::Graph.placement = false
|
|
1471
|
+
copied = run.call
|
|
1472
|
+
ensure
|
|
1473
|
+
Tilework::Graph.placement = previous
|
|
1474
|
+
end
|
|
1475
|
+
Conformance.assert_equal_arrays(copied, placed)
|
|
1476
|
+
end
|
|
1477
|
+
|
|
1478
|
+
# Mutable-state ordering. Version edges give only RAW, so check numerically
|
|
1479
|
+
# that version N+1's write does not run while version N's readers still read.
|
|
1480
|
+
register(:graph_state_write_waits_for_the_readers_of_the_old_version) do
|
|
1481
|
+
cache = Tilework[Conformance.f32([1.0, 1.0, 2.0, 2.0])].reshape(2, 2)
|
|
1482
|
+
plan = Tilework::Graph.build do |g|
|
|
1483
|
+
s0 = g.state(cache, name: :s)
|
|
1484
|
+
stale = g.read(s0, 0)
|
|
1485
|
+
g.write_into(s0, at: 0, value: g.param(Tilework[Conformance.f32([9.0, 9.0])].reshape(1, 2)))
|
|
1486
|
+
g.output(stale * 1.0)
|
|
1487
|
+
end.compile
|
|
1488
|
+
Conformance.assert_close([1.0, 1.0], plan.run.to_a)
|
|
1489
|
+
Conformance.assert_close([9.0, 9.0, 2.0, 2.0], cache.to_a)
|
|
1490
|
+
end
|
|
1491
|
+
|
|
1492
|
+
# A write position decided at run time (a KV-cache append), advanced by
|
|
1493
|
+
# rerunning the same plan.
|
|
1494
|
+
register(:graph_state_appends_across_runs) do
|
|
1495
|
+
cache = Tilework::Buffer.new(:float32, 8, shape: [4, 2])
|
|
1496
|
+
cache.write([0.0] * 8)
|
|
1497
|
+
plan = Tilework::Graph.build do |g|
|
|
1498
|
+
kv0 = g.state(cache, name: :kv)
|
|
1499
|
+
k = g.input(:k, [1, 2])
|
|
1500
|
+
g.output(g.read(g.write_into(kv0, at: g.index(:pos), value: k)) * 1.0)
|
|
1501
|
+
end.compile
|
|
1502
|
+
3.times do |step|
|
|
1503
|
+
plan.run(k: Tilework[Conformance.f32([step + 1.0, (step + 1) * 10.0])].reshape(1, 2), pos: step)
|
|
1504
|
+
end
|
|
1505
|
+
Conformance.assert_close([1.0, 10.0, 2.0, 20.0, 3.0, 30.0, 0.0, 0.0], cache.to_a)
|
|
1506
|
+
end
|
|
1507
|
+
|
|
1508
|
+
# Two writes to disjoint regions need not serialize (rejoined with g.merge).
|
|
1509
|
+
register(:graph_state_disjoint_writes_fill_the_buffer) do
|
|
1510
|
+
out = Tilework::Buffer.new(:float32, 8, shape: [2, 4])
|
|
1511
|
+
out.write([0.0] * 8)
|
|
1512
|
+
plan = Tilework::Graph.build do |g|
|
|
1513
|
+
s0 = g.state(out, name: :out)
|
|
1514
|
+
left = g.write_into(s0, at: s0.axis(1, 0...2), value: g.input(:l, [2, 2]))
|
|
1515
|
+
right = g.write_into(s0, at: s0.axis(1, 2...4), value: g.input(:r, [2, 2]))
|
|
1516
|
+
g.output(g.read(g.merge(left, right)))
|
|
1517
|
+
end.compile
|
|
1518
|
+
l = Tilework[Conformance.f32([1.0, 2.0, 5.0, 6.0])].reshape(2, 2)
|
|
1519
|
+
r = Tilework[Conformance.f32([3.0, 4.0, 7.0, 8.0])].reshape(2, 2)
|
|
1520
|
+
Conformance.assert_close([1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0], plan.run(l: l, r: r).to_a)
|
|
1521
|
+
end
|
|
1522
|
+
|
|
1523
|
+
# strict_state = :copy materializes reads, making the graph a pure DAG. The
|
|
1524
|
+
# default view path producing the same values verifies the mutable-state
|
|
1525
|
+
# machinery itself (an oracle inside the graph layer).
|
|
1526
|
+
register(:graph_state_view_reads_agree_with_materialized_reads) do
|
|
1527
|
+
build = lambda do
|
|
1528
|
+
cache = Tilework[Conformance.f32([1.0, 2.0, 3.0, 4.0, 5.0, 6.0])].reshape(3, 2)
|
|
1529
|
+
plan = Tilework::Graph.build do |g|
|
|
1530
|
+
s0 = g.state(cache, name: :kv)
|
|
1531
|
+
head = g.read(s0, 0..1)
|
|
1532
|
+
s1 = g.write_into(s0, at: 2, value: g.param(Tilework[Conformance.f32([7.0, 8.0])].reshape(1, 2)))
|
|
1533
|
+
g.output((head * 2.0) + g.read(s1, 1..2))
|
|
1534
|
+
end.compile
|
|
1535
|
+
plan.run.to_a
|
|
1536
|
+
end
|
|
1537
|
+
|
|
1538
|
+
previous = Tilework::Graph.strict_state
|
|
1539
|
+
begin
|
|
1540
|
+
Tilework::Graph.strict_state = :view
|
|
1541
|
+
view = build.call
|
|
1542
|
+
Tilework::Graph.strict_state = :copy
|
|
1543
|
+
copied = build.call
|
|
1544
|
+
ensure
|
|
1545
|
+
Tilework::Graph.strict_state = previous
|
|
1546
|
+
end
|
|
1547
|
+
Conformance.assert_close(copied, view)
|
|
1548
|
+
end
|
|
1549
|
+
# --- Automatic offload (#17 Phase 4 / docs/graph-offload.md) -------------
|
|
1550
|
+
#
|
|
1551
|
+
# Paging must not change values, so it is held to bit identity at any
|
|
1552
|
+
# budget. Without a staging ABI Offload.transport falls back to :posix, so
|
|
1553
|
+
# the policy itself is checkable on :cpu.
|
|
1554
|
+
register(:graph_offload_matches_the_resident_path) do
|
|
1555
|
+
# Bind **non-view** Buffers: reshape returns a view, and views cannot be
|
|
1556
|
+
# paged because restore! allocates a new handle (they are just counted as
|
|
1557
|
+
# pinned, but then this check would page nothing).
|
|
1558
|
+
weights = Array.new(4) { Conformance.square(4) }
|
|
1559
|
+
x = Conformance.square(4)
|
|
1560
|
+
build = lambda do |vram|
|
|
1561
|
+
Tilework::Graph.build do |g|
|
|
1562
|
+
value = g.input(:x, [4, 4])
|
|
1563
|
+
weights.each { |w| value = value.matmul(g.param(w)) }
|
|
1564
|
+
g.output(value)
|
|
1565
|
+
end.compile(vram: vram)
|
|
1566
|
+
end
|
|
1567
|
+
|
|
1568
|
+
resident = build.call(nil).run(x: x).to_a
|
|
1569
|
+
# Squeeze the budget to "two weights + reserve". One node needs one weight
|
|
1570
|
+
# at a time, so it passes, but four do not fit, so paging always happens.
|
|
1571
|
+
probe = build.call(1 << 30).residency
|
|
1572
|
+
slot = probe.weights / weights.length
|
|
1573
|
+
paged = build.call(probe.reserved + (2 * slot))
|
|
1574
|
+
begin
|
|
1575
|
+
Conformance.assert(paged.residency.floor.positive?, "the budget must force paging")
|
|
1576
|
+
Conformance.assert_equal_arrays(resident, paged.run(x: x).to_a)
|
|
1577
|
+
Conformance.assert(paged.offload_stats[:fetches].positive?, "no weight was ever paged in")
|
|
1578
|
+
ensure
|
|
1579
|
+
paged.free_offload!
|
|
1580
|
+
end
|
|
1581
|
+
end
|
|
1582
|
+
|
|
1583
|
+
# Rerunning the same plan does not change values. Eviction is buffer_free,
|
|
1584
|
+
# so releasing VRAM a not-yet-executed kernel reads breaks things as
|
|
1585
|
+
# "occasionally different values" (the same hazard that makes Pool return
|
|
1586
|
+
# rather than free). One run cannot hit it, so run three times.
|
|
1587
|
+
register(:graph_offload_is_stable_across_runs) do
|
|
1588
|
+
weights = Array.new(3) { Conformance.square(4) }
|
|
1589
|
+
x = Conformance.square(4)
|
|
1590
|
+
build = lambda do |vram|
|
|
1591
|
+
Tilework::Graph.build do |g|
|
|
1592
|
+
value = g.input(:x, [4, 4])
|
|
1593
|
+
weights.each { |w| value = (value.matmul(g.param(w)) * 1.5) }
|
|
1594
|
+
g.output(value)
|
|
1595
|
+
end.compile(vram: vram)
|
|
1596
|
+
end
|
|
1597
|
+
|
|
1598
|
+
want = build.call(nil).run(x: x).to_a
|
|
1599
|
+
probe = build.call(1 << 30).residency
|
|
1600
|
+
plan = build.call(probe.reserved + (probe.weights / weights.length))
|
|
1601
|
+
begin
|
|
1602
|
+
3.times { |i| Conformance.assert_equal_arrays(want, plan.run(x: x).to_a) && i }
|
|
1603
|
+
ensure
|
|
1604
|
+
plan.free_offload!
|
|
1605
|
+
end
|
|
1606
|
+
end
|
|
1607
|
+
|
|
1608
|
+
# --- Backward (#22) ------------------------------------------------------
|
|
1609
|
+
|
|
1610
|
+
# Gradients are correct, checked against **finite differences** — backward
|
|
1611
|
+
# easily returns plausible but wrong values, and forward agreement
|
|
1612
|
+
# guarantees nothing.
|
|
1613
|
+
#
|
|
1614
|
+
# It checks **directional derivatives**, not per-element values. float32
|
|
1615
|
+
# central differences agree only to about 1e-3 per element (cancellation),
|
|
1616
|
+
# so a per-element tolerance would constrain nothing. <grad, v> against
|
|
1617
|
+
# (f(x+hv) - f(x-hv)) / 2h for a direction v averages the error down to
|
|
1618
|
+
# about 1e-4. The tolerance is 3e-2 (measured on :cpu: 1e-5 to 1e-4).
|
|
1619
|
+
#
|
|
1620
|
+
# v is fixed (no random draw) — a failure that cannot be reproduced is useless.
|
|
1621
|
+
register(:graph_gradients_match_finite_differences) do
|
|
1622
|
+
rows, cols, out = 3, 4, 2
|
|
1623
|
+
# Fixed values that **stay away from 0 and where's thresholds** — abs and
|
|
1624
|
+
# where have kinks where the derivative jumps, and differences straddling
|
|
1625
|
+
# one disagree with the analytic value (a limit of finite differences,
|
|
1626
|
+
# not a vjp error).
|
|
1627
|
+
x = Array.new(rows * cols) { |i| Math.sin((i + 0.6) * 1.7) * 0.8 }
|
|
1628
|
+
w = Array.new(cols * out) { |i| Math.cos(i * 2.3) * 0.6 }
|
|
1629
|
+
dx = Array.new(rows * cols) { |i| Math.cos(i * 0.9) * 0.5 }
|
|
1630
|
+
dw = Array.new(cols * out) { |i| Math.sin(i * 1.3) * 0.5 }
|
|
1631
|
+
# gamma (the norms' affine scale) is differentiated too. Hand-written vjps
|
|
1632
|
+
# use a separate expression for it, so checking x alone tests only half.
|
|
1633
|
+
gm = Array.new(cols) { |i| 0.5 + (i * 0.25) }
|
|
1634
|
+
dg = Array.new(cols) { |i| Math.cos((i + 1) * 1.1) * 0.5 }
|
|
1635
|
+
labels = Buffer.new(:int32, rows, shape: [rows])
|
|
1636
|
+
labels.write(Array.new(rows) { |i| i % out })
|
|
1637
|
+
|
|
1638
|
+
# One plan per expression. Summing everything into one loss lets one wrong
|
|
1639
|
+
# vjp dilute into the other terms and fall under the tolerance.
|
|
1640
|
+
cases = {
|
|
1641
|
+
arithmetic: ->(_g, xt, wt, _gt) { (((xt * xt) / (xt + 3.0)) - (2.0 / (xt + 3.0)) - (xt * 0.5)).sum + (wt * wt).sum },
|
|
1642
|
+
unary: ->(_g, xt, _wt, _gt) { ((((xt * xt) + 1.5).sqrt.log + xt.sin + xt.cos.exp + xt.tanh).mean) },
|
|
1643
|
+
abs_where: ->(g, xt, _wt, _gt) { (xt.abs + g.where(xt > 0.5, xt * 2.0, xt * 0.25)).sum },
|
|
1644
|
+
matmul: ->(_g, xt, wt, _gt) { (xt.matmul(wt) * 1.5).sum },
|
|
1645
|
+
broadcast: ->(_g, xt, wt, _gt) { (xt + wt.permute(1, 0).slice(0..0).reshape(cols)).sum },
|
|
1646
|
+
rowwise: ->(_g, xt, _wt, gt) { NN.silu(Tilework::NN.rmsnorm(xt, gamma: gt) * 1.5).sum },
|
|
1647
|
+
# layernorm's rows always sum to 0, so summing directly makes the
|
|
1648
|
+
# derivative 0 too (checking nothing). Weight by x before folding.
|
|
1649
|
+
layernorm: ->(_g, xt, _wt, gt) { (NN.layernorm(xt, gamma: gt, beta: gt * 0.5) * xt).sum },
|
|
1650
|
+
softmax: ->(_g, xt, _wt, _gt) { (NN.softmax(xt) * NN.softmax(xt)).sum },
|
|
1651
|
+
gelu: ->(_g, xt, _wt, _gt) { NN.gelu(xt).sum },
|
|
1652
|
+
cross_entropy: ->(g, xt, wt, _gt) { NN.cross_entropy(xt.matmul(wt), g.param(labels), reduction: :sum) * 5.0 },
|
|
1653
|
+
# The reduction's scale differs between :mean and :sum, so run both.
|
|
1654
|
+
cross_entropy_mean: ->(g, xt, wt, _gt) { NN.cross_entropy(xt.matmul(wt), g.param(labels)) * 20.0 },
|
|
1655
|
+
mse: ->(_g, xt, _wt, _gt) { NN.mse(xt, xt * 2.0) },
|
|
1656
|
+
max_reduce: ->(_g, xt, _wt, _gt) { xt.max(axis: -1).sum },
|
|
1657
|
+
# Shape ops. permute needs 3D for an inverse-permutation mistake to show
|
|
1658
|
+
# (a 2D transpose is its own inverse, so a mix-up gives the same answer).
|
|
1659
|
+
# iota weights each position differently so ordering differences reach the value.
|
|
1660
|
+
layout: lambda { |g, xt, _wt, _gt|
|
|
1661
|
+
t = xt.reshape(rows, 2, 2).permute(1, 2, 0)
|
|
1662
|
+
weighted = (t * g.iota([2, 2, rows], axis: 2).cast(:float32)).sum
|
|
1663
|
+
pieces = g.concat(xt.permute(1, 0).slice(0..1) / 3.0, xt.permute(1, 0).slice(1..2))
|
|
1664
|
+
weighted + pieces.reshape(4 * rows).sum
|
|
1665
|
+
},
|
|
1666
|
+
pad_concat: lambda { |g, xt, _wt, _gt|
|
|
1667
|
+
padded = g.pad(xt, before: [1, 2], after: [2, 1], value: 0.0)
|
|
1668
|
+
# A linear expression would make the derivative the sum of the
|
|
1669
|
+
# direction vector (≈0), so square before folding.
|
|
1670
|
+
joined = g.concat(padded, padded, axis: 1)
|
|
1671
|
+
(joined * joined).sum
|
|
1672
|
+
},
|
|
1673
|
+
}
|
|
1674
|
+
|
|
1675
|
+
# h is 2e-3: smaller suffers cancellation, larger truncation error, so for
|
|
1676
|
+
# float32 this is about the floor.
|
|
1677
|
+
step = 2e-3
|
|
1678
|
+
cases.each do |name, body|
|
|
1679
|
+
build = lambda do |with_grad|
|
|
1680
|
+
Tilework::Graph.build(differentiable: true) do |g|
|
|
1681
|
+
xt = g.input(:x, [rows, cols])
|
|
1682
|
+
wt = g.input(:w, [cols, out])
|
|
1683
|
+
gt = g.input(:g, [cols])
|
|
1684
|
+
loss = body.call(g, xt, wt, gt)
|
|
1685
|
+
g.output(loss)
|
|
1686
|
+
g.grad(loss, wrt: [xt, wt, gt]).each { |grad| g.output(grad) } if with_grad
|
|
1687
|
+
end.compile
|
|
1688
|
+
end
|
|
1689
|
+
|
|
1690
|
+
plan = build.call(true)
|
|
1691
|
+
plain = build.call(false)
|
|
1692
|
+
bind = lambda do
|
|
1693
|
+
{ x: Conformance.nd([rows, cols], x), w: Conformance.nd([cols, out], w), g: Conformance.nd([cols], gm) }
|
|
1694
|
+
end
|
|
1695
|
+
grads = plan.run(**bind.call).drop(1).map(&:to_a)
|
|
1696
|
+
|
|
1697
|
+
[[:x, x, dx, [rows, cols], 0], [:w, w, dw, [cols, out], 1],
|
|
1698
|
+
[:g, gm, dg, [cols], 2]].each do |arg, base, dir, shape, index|
|
|
1699
|
+
shifted = lambda do |sign|
|
|
1700
|
+
bindings = bind.call
|
|
1701
|
+
bindings[arg] = Conformance.nd(shape, base.each_with_index.map { |v, i| v + (sign * step * dir[i]) })
|
|
1702
|
+
plain.run(**bindings).to_a[0]
|
|
1703
|
+
end
|
|
1704
|
+
numeric = (shifted.call(1.0) - shifted.call(-1.0)) / (2.0 * step)
|
|
1705
|
+
analytic = grads[index].each_with_index.sum { |v, i| v * dir[i] }
|
|
1706
|
+
# An expression with a near-zero derivative checks nothing, so first
|
|
1707
|
+
# check that it moves (expressions not using :w excepted).
|
|
1708
|
+
next if numeric.abs < 1e-6 && analytic.abs < 1e-6
|
|
1709
|
+
|
|
1710
|
+
Conformance.assert(numeric.abs > 1e-2,
|
|
1711
|
+
"#{name} d/d#{arg}: the directional derivative is #{numeric}, too small to check")
|
|
1712
|
+
scale = [numeric.abs, analytic.abs].max
|
|
1713
|
+
Conformance.assert((numeric - analytic).abs / scale < 3e-2,
|
|
1714
|
+
"#{name} d/d#{arg}: finite difference #{numeric}, analytic #{analytic}")
|
|
1715
|
+
end
|
|
1716
|
+
end
|
|
1717
|
+
end
|
|
1718
|
+
|
|
1719
|
+
|
|
1720
|
+
|
|
1721
|
+
# Checkpointing lowers the peak without changing gradients (#22 Step 5).
|
|
1722
|
+
#
|
|
1723
|
+
# The pool's allocated is the number of slots alive at once, so it stands in
|
|
1724
|
+
# for the peak. If it does not drop, the scheduler has hoisted the copies
|
|
1725
|
+
# and nothing is saved.
|
|
1726
|
+
register(:graph_checkpointing_keeps_the_gradients_and_lowers_the_peak) do
|
|
1727
|
+
n, d, layers = 8, 16, 4
|
|
1728
|
+
x = Conformance.nd([n, d], Array.new(n * d) { |i| Math.sin((i + 0.3) * 0.7) * 0.5 })
|
|
1729
|
+
weights = Array.new(layers) do |l|
|
|
1730
|
+
Conformance.nd([d, d], Array.new(d * d) { |i| Math.cos((i + l) * 0.31) * 0.25 })
|
|
1731
|
+
end
|
|
1732
|
+
|
|
1733
|
+
build = lambda do |use_checkpoint|
|
|
1734
|
+
Tilework::Graph.build(differentiable: true) do |g|
|
|
1735
|
+
xt = g.input(:x, [n, d])
|
|
1736
|
+
h = xt
|
|
1737
|
+
weights.each do |w|
|
|
1738
|
+
wt = g.param(w)
|
|
1739
|
+
h = if use_checkpoint
|
|
1740
|
+
g.checkpoint { NN.gelu(h.matmul(wt)) }
|
|
1741
|
+
else
|
|
1742
|
+
NN.gelu(h.matmul(wt))
|
|
1743
|
+
end
|
|
1744
|
+
end
|
|
1745
|
+
loss = (h * h).sum
|
|
1746
|
+
g.output(loss)
|
|
1747
|
+
g.grad(loss, wrt: [xt]).each { |grad| g.output(grad) }
|
|
1748
|
+
end.compile
|
|
1749
|
+
end
|
|
1750
|
+
|
|
1751
|
+
plain = build.call(false)
|
|
1752
|
+
checkpointed = build.call(true)
|
|
1753
|
+
want = plain.run(x: x).map(&:to_a)
|
|
1754
|
+
got = checkpointed.run(x: x).map(&:to_a)
|
|
1755
|
+
# Bit identity is not available. checkpoint changes the fusion structure,
|
|
1756
|
+
# and inside a fused expression a shader compiler may fold a*b+c into an
|
|
1757
|
+
# FMA (measured on :metal: 4.5e-7 relative to the array max).
|
|
1758
|
+
# graph_grad_cpu_test.rb checks bit identity on :cpu.
|
|
1759
|
+
want.each_with_index { |values, i| Conformance.assert_close_relative(values, got[i], 1e-5) }
|
|
1760
|
+
|
|
1761
|
+
# Run once more until the pool stabilizes before comparing.
|
|
1762
|
+
plain.run(x: x)
|
|
1763
|
+
checkpointed.run(x: x)
|
|
1764
|
+
peak = plain.pool_stats[:allocated]
|
|
1765
|
+
lower = checkpointed.pool_stats[:allocated]
|
|
1766
|
+
Conformance.assert(lower < peak,
|
|
1767
|
+
"checkpointing did not lower the peak (#{lower} slots against #{peak})")
|
|
1768
|
+
end
|
|
1769
|
+
|
|
1770
|
+
# Optimizers apply the updates they promise (#22 Step 3).
|
|
1771
|
+
#
|
|
1772
|
+
# The MLP check below sees training progress, but that **passes even with a
|
|
1773
|
+
# slightly wrong formula** (the loss drops as long as the gradient points
|
|
1774
|
+
# the right way). Here one step runs and the update itself is compared with
|
|
1775
|
+
# the closed form.
|
|
1776
|
+
#
|
|
1777
|
+
# Adam's first step starts from m = v = 0, so after bias correction
|
|
1778
|
+
# m̂/√v̂ = g/|g| — **it moves by lr regardless of the gradient's magnitude**.
|
|
1779
|
+
# Dropping the correction breaks this, so mishandling t fails here.
|
|
1780
|
+
register(:graph_optimizers_apply_the_updates_they_promise) do
|
|
1781
|
+
values = Conformance.f32([0.5, -1.5, 2.0, -0.25])
|
|
1782
|
+
target = Conformance.f32([0.0, 0.0, 0.0, 0.0])
|
|
1783
|
+
lr = 0.1
|
|
1784
|
+
|
|
1785
|
+
# loss = Σ(p - t)², so the gradient is 2(p - t) = 2p.
|
|
1786
|
+
want_grad = values.map { |v| 2.0 * v }
|
|
1787
|
+
|
|
1788
|
+
sgd_param = Tilework[values.dup]
|
|
1789
|
+
sgd = Tilework::Graph::SGD.new({ w: sgd_param }, lr: lr)
|
|
1790
|
+
sgd_plan = Tilework::Graph.build(differentiable: true) do |g|
|
|
1791
|
+
p = sgd.attach(g)[:w]
|
|
1792
|
+
loss = ((p - g.param(Tilework[target])) * (p - g.param(Tilework[target]))).sum
|
|
1793
|
+
sgd.step(loss)
|
|
1794
|
+
g.output(loss)
|
|
1795
|
+
end.compile
|
|
1796
|
+
sgd_plan.run
|
|
1797
|
+
Conformance.assert_close(values.each_with_index.map { |v, i| v - (lr * want_grad[i]) },
|
|
1798
|
+
sgd_param.to_a, 1e-5)
|
|
1799
|
+
|
|
1800
|
+
adam_param = Tilework[values.dup]
|
|
1801
|
+
adam = Tilework::Graph::Adam.new({ w: adam_param }, lr: lr)
|
|
1802
|
+
adam_plan = Tilework::Graph.build(differentiable: true) do |g|
|
|
1803
|
+
p = adam.attach(g)[:w]
|
|
1804
|
+
loss = ((p - g.param(Tilework[target])) * (p - g.param(Tilework[target]))).sum
|
|
1805
|
+
adam.step(loss)
|
|
1806
|
+
g.output(loss)
|
|
1807
|
+
end.compile
|
|
1808
|
+
begin
|
|
1809
|
+
adam_plan.run
|
|
1810
|
+
# Only the sign matters on the first step (slightly less, by eps).
|
|
1811
|
+
want = values.each_with_index.map { |v, i| v - (lr * (want_grad[i].negative? ? -1.0 : 1.0)) }
|
|
1812
|
+
Conformance.assert_close(want, adam_param.to_a, 1e-4)
|
|
1813
|
+
ensure
|
|
1814
|
+
adam.free!
|
|
1815
|
+
sgd.free!
|
|
1816
|
+
end
|
|
1817
|
+
end
|
|
1818
|
+
|
|
1819
|
+
# gather's gradient (scatter-add) agrees with the form written as a matmul
|
|
1820
|
+
# against a match matrix, and gives the same value every run. It scans the
|
|
1821
|
+
# indices per output element without atomics, so the fold order is fixed
|
|
1822
|
+
# and bit identity can be required.
|
|
1823
|
+
#
|
|
1824
|
+
# It also checks that duplicate indices **accumulate** (not overwrite).
|
|
1825
|
+
register(:graph_scatter_add_matches_a_matmul_reference) do
|
|
1826
|
+
rows, inner, count = 5, 3, 6
|
|
1827
|
+
table = Conformance.nd([rows, inner], Array.new(rows * inner) { |i| Math.sin(i * 1.4) })
|
|
1828
|
+
ids = Buffer.new(:int32, count, shape: [count])
|
|
1829
|
+
# Duplicates (2 three times), a row never drawn (1), and out-of-range (clamped to the ends).
|
|
1830
|
+
ids.write([0, 2, 2, 2, 4, 9])
|
|
1831
|
+
|
|
1832
|
+
plan = Tilework::Graph.build(differentiable: true) do |g|
|
|
1833
|
+
t = g.input(:t, [rows, inner])
|
|
1834
|
+
picked = g.gather(t, g.param(ids))
|
|
1835
|
+
g.output(g.grad((picked * picked).sum, wrt: [t])[0])
|
|
1836
|
+
end.compile
|
|
1837
|
+
|
|
1838
|
+
# Reference: build mask[j, i] = (ids[i] == j) and multiply dy by it.
|
|
1839
|
+
reference = Tilework::Graph.build(differentiable: true) do |g|
|
|
1840
|
+
t = g.input(:t, [rows, inner])
|
|
1841
|
+
picked = g.gather(t, g.param(ids))
|
|
1842
|
+
dy = 2.0 * picked
|
|
1843
|
+
index = g.param(ids).reshape(1, count)
|
|
1844
|
+
clamped = g.where(index > (rows - 1), rows - 1, g.where(index < 0, 0, index))
|
|
1845
|
+
mask = g.where(g.iota([rows, count], axis: 0).eq(clamped), 1.0, 0.0)
|
|
1846
|
+
g.output(mask.matmul(dy))
|
|
1847
|
+
end.compile
|
|
1848
|
+
|
|
1849
|
+
got = plan.run(t: table).to_a
|
|
1850
|
+
Conformance.assert_equal_arrays(reference.run(t: table).to_a, got)
|
|
1851
|
+
2.times { Conformance.assert_equal_arrays(got, plan.run(t: table).to_a) }
|
|
1852
|
+
end
|
|
1853
|
+
|
|
1854
|
+
# Convolution gradients. **The only gradient taking a different path per
|
|
1855
|
+
# backend**, so both are checked against finite differences — the delegated
|
|
1856
|
+
# backward kernels (:conv2d_backward_data / _filter) with cuDNN / MPSGraph,
|
|
1857
|
+
# otherwise the primitive vjps of the im2col decomposition (where the pad and
|
|
1858
|
+
# gather rows come in).
|
|
1859
|
+
register(:graph_convolution_gradients_match_finite_differences) do
|
|
1860
|
+
n, cin, h, w = 1, 2, 5, 5
|
|
1861
|
+
cout, kh, kw = 2, 3, 3
|
|
1862
|
+
x = Array.new(n * cin * h * w) { |i| Math.sin((i + 0.4) * 1.3) * 0.7 }
|
|
1863
|
+
k = Array.new(cout * cin * kh * kw) { |i| Math.cos((i + 0.2) * 0.9) * 0.4 }
|
|
1864
|
+
dx = Array.new(x.length) { |i| Math.cos(i * 1.7) * 0.5 }
|
|
1865
|
+
dk = Array.new(k.length) { |i| Math.sin(i * 2.1) * 0.5 }
|
|
1866
|
+
|
|
1867
|
+
build = lambda do |with_grad|
|
|
1868
|
+
Tilework::Graph.build(differentiable: true) do |g|
|
|
1869
|
+
xt = g.input(:x, [n, cin, h, w])
|
|
1870
|
+
kt = g.input(:k, [cout, cin, kh, kw])
|
|
1871
|
+
y = Tilework::Conv.forward(xt, kt, pad: 1, stride: 1)
|
|
1872
|
+
loss = (y * y).sum
|
|
1873
|
+
g.output(loss)
|
|
1874
|
+
g.grad(loss, wrt: [xt, kt]).each { |grad| g.output(grad) } if with_grad
|
|
1875
|
+
end.compile
|
|
1876
|
+
end
|
|
1877
|
+
|
|
1878
|
+
plan = build.call(true)
|
|
1879
|
+
plain = build.call(false)
|
|
1880
|
+
bind = lambda do
|
|
1881
|
+
{ x: Conformance.nd([n, cin, h, w], x), k: Conformance.nd([cout, cin, kh, kw], k) }
|
|
1882
|
+
end
|
|
1883
|
+
grads = plan.run(**bind.call).drop(1).map(&:to_a)
|
|
1884
|
+
|
|
1885
|
+
step = 2e-3
|
|
1886
|
+
[[:x, x, dx, [n, cin, h, w], 0], [:k, k, dk, [cout, cin, kh, kw], 1]].each do |arg, base, dir, shape, index|
|
|
1887
|
+
shifted = lambda do |sign|
|
|
1888
|
+
bindings = bind.call
|
|
1889
|
+
bindings[arg] = Conformance.nd(shape, base.each_with_index.map { |v, i| v + (sign * step * dir[i]) })
|
|
1890
|
+
plain.run(**bindings).to_a[0]
|
|
1891
|
+
end
|
|
1892
|
+
numeric = (shifted.call(1.0) - shifted.call(-1.0)) / (2.0 * step)
|
|
1893
|
+
analytic = grads[index].each_with_index.sum { |v, i| v * dir[i] }
|
|
1894
|
+
Conformance.assert(numeric.abs > 1e-2, "conv d/d#{arg}: derivative #{numeric} is too small to check")
|
|
1895
|
+
Conformance.assert((numeric - analytic).abs / [numeric.abs, analytic.abs].max < 3e-2,
|
|
1896
|
+
"conv d/d#{arg}: finite difference #{numeric}, analytic #{analytic}")
|
|
1897
|
+
end
|
|
1898
|
+
end
|
|
1899
|
+
|
|
1900
|
+
# The same finite-difference check with dilation.
|
|
1901
|
+
register(:graph_dilated_convolution_gradients_match_finite_differences) do
|
|
1902
|
+
n, cin, h, w = 1, 2, 6, 6
|
|
1903
|
+
cout, kh, kw = 2, 3, 3
|
|
1904
|
+
dilation = [2, 2]
|
|
1905
|
+
x = Array.new(n * cin * h * w) { |i| Math.sin((i + 0.4) * 1.3) * 0.7 }
|
|
1906
|
+
k = Array.new(cout * cin * kh * kw) { |i| Math.cos((i + 0.2) * 0.9) * 0.4 }
|
|
1907
|
+
dx = Array.new(x.length) { |i| Math.cos(i * 1.7) * 0.5 }
|
|
1908
|
+
dk = Array.new(k.length) { |i| Math.sin(i * 2.1) * 0.5 }
|
|
1909
|
+
|
|
1910
|
+
build = lambda do |with_grad|
|
|
1911
|
+
Tilework::Graph.build(differentiable: true) do |g|
|
|
1912
|
+
xt = g.input(:x, [n, cin, h, w])
|
|
1913
|
+
kt = g.input(:k, [cout, cin, kh, kw])
|
|
1914
|
+
y = Tilework::Conv.forward(xt, kt, pad: 2, stride: 1, dilation: dilation)
|
|
1915
|
+
loss = (y * y).sum
|
|
1916
|
+
g.output(loss)
|
|
1917
|
+
g.grad(loss, wrt: [xt, kt]).each { |grad| g.output(grad) } if with_grad
|
|
1918
|
+
end.compile
|
|
1919
|
+
end
|
|
1920
|
+
|
|
1921
|
+
plan = build.call(true)
|
|
1922
|
+
plain = build.call(false)
|
|
1923
|
+
bind = lambda do
|
|
1924
|
+
{ x: Conformance.nd([n, cin, h, w], x), k: Conformance.nd([cout, cin, kh, kw], k) }
|
|
1925
|
+
end
|
|
1926
|
+
grads = plan.run(**bind.call).drop(1).map(&:to_a)
|
|
1927
|
+
|
|
1928
|
+
step = 2e-3
|
|
1929
|
+
[[:x, x, dx, [n, cin, h, w], 0], [:k, k, dk, [cout, cin, kh, kw], 1]].each do |arg, base, dir, shape, index|
|
|
1930
|
+
shifted = lambda do |sign|
|
|
1931
|
+
bindings = bind.call
|
|
1932
|
+
bindings[arg] = Conformance.nd(shape, base.each_with_index.map { |v, i| v + (sign * step * dir[i]) })
|
|
1933
|
+
plain.run(**bindings).to_a[0]
|
|
1934
|
+
end
|
|
1935
|
+
numeric = (shifted.call(1.0) - shifted.call(-1.0)) / (2.0 * step)
|
|
1936
|
+
analytic = grads[index].each_with_index.sum { |v, i| v * dir[i] }
|
|
1937
|
+
Conformance.assert(numeric.abs > 1e-2, "dilated conv d/d#{arg}: derivative #{numeric} is too small to check")
|
|
1938
|
+
Conformance.assert((numeric - analytic).abs / [numeric.abs, analytic.abs].max < 3e-2,
|
|
1939
|
+
"dilated conv d/d#{arg}: finite difference #{numeric}, analytic #{analytic}")
|
|
1940
|
+
end
|
|
1941
|
+
end
|
|
1942
|
+
|
|
1943
|
+
# The same finite-difference check with groups (depthwise: groups == cin == cout).
|
|
1944
|
+
register(:graph_grouped_convolution_gradients_match_finite_differences) do
|
|
1945
|
+
n, cin, h, w = 1, 4, 5, 5
|
|
1946
|
+
cout, kh, kw = 4, 3, 3
|
|
1947
|
+
groups = 4
|
|
1948
|
+
cin_per_group = cin / groups
|
|
1949
|
+
x = Array.new(n * cin * h * w) { |i| Math.sin((i + 0.4) * 1.3) * 0.7 }
|
|
1950
|
+
k = Array.new(cout * cin_per_group * kh * kw) { |i| Math.cos((i + 0.2) * 0.9) * 0.4 }
|
|
1951
|
+
dx = Array.new(x.length) { |i| Math.cos(i * 1.7) * 0.5 }
|
|
1952
|
+
dk = Array.new(k.length) { |i| Math.sin(i * 2.1) * 0.5 }
|
|
1953
|
+
|
|
1954
|
+
build = lambda do |with_grad|
|
|
1955
|
+
Tilework::Graph.build(differentiable: true) do |g|
|
|
1956
|
+
xt = g.input(:x, [n, cin, h, w])
|
|
1957
|
+
kt = g.input(:k, [cout, cin_per_group, kh, kw])
|
|
1958
|
+
y = Tilework::Conv.forward(xt, kt, pad: 1, stride: 1, groups: groups)
|
|
1959
|
+
loss = (y * y).sum
|
|
1960
|
+
g.output(loss)
|
|
1961
|
+
g.grad(loss, wrt: [xt, kt]).each { |grad| g.output(grad) } if with_grad
|
|
1962
|
+
end.compile
|
|
1963
|
+
end
|
|
1964
|
+
|
|
1965
|
+
plan = build.call(true)
|
|
1966
|
+
plain = build.call(false)
|
|
1967
|
+
bind = lambda do
|
|
1968
|
+
{ x: Conformance.nd([n, cin, h, w], x), k: Conformance.nd([cout, cin_per_group, kh, kw], k) }
|
|
1969
|
+
end
|
|
1970
|
+
grads = plan.run(**bind.call).drop(1).map(&:to_a)
|
|
1971
|
+
|
|
1972
|
+
step = 2e-3
|
|
1973
|
+
[[:x, x, dx, [n, cin, h, w], 0],
|
|
1974
|
+
[:k, k, dk, [cout, cin_per_group, kh, kw], 1]].each do |arg, base, dir, shape, index|
|
|
1975
|
+
shifted = lambda do |sign|
|
|
1976
|
+
bindings = bind.call
|
|
1977
|
+
bindings[arg] = Conformance.nd(shape, base.each_with_index.map { |v, i| v + (sign * step * dir[i]) })
|
|
1978
|
+
plain.run(**bindings).to_a[0]
|
|
1979
|
+
end
|
|
1980
|
+
numeric = (shifted.call(1.0) - shifted.call(-1.0)) / (2.0 * step)
|
|
1981
|
+
analytic = grads[index].each_with_index.sum { |v, i| v * dir[i] }
|
|
1982
|
+
Conformance.assert(numeric.abs > 1e-2, "grouped conv d/d#{arg}: derivative #{numeric} is too small to check")
|
|
1983
|
+
Conformance.assert((numeric - analytic).abs / [numeric.abs, analytic.abs].max < 3e-2,
|
|
1984
|
+
"grouped conv d/d#{arg}: finite difference #{numeric}, analytic #{analytic}")
|
|
1985
|
+
end
|
|
1986
|
+
end
|
|
1987
|
+
|
|
1988
|
+
# Hand-written vjps agree with differentiating the decomposition.
|
|
1989
|
+
#
|
|
1990
|
+
# Finite differences can only say one of the two is wrong, not which, so the
|
|
1991
|
+
# fast path needs another reference. The decomposition's gradient follows
|
|
1992
|
+
# automatically from primitive vjps, so it is the oracle — the same trick as
|
|
1993
|
+
# comparing high-level ops' forward pass with their decompositions.
|
|
1994
|
+
#
|
|
1995
|
+
# Bit identity is not available (different fold order and accumulator
|
|
1996
|
+
# type); the tolerance is the forward pass's 1e-5.
|
|
1997
|
+
register(:graph_hand_written_vjps_match_the_decomposed_ones) do
|
|
1998
|
+
rows, cols = 3, 4
|
|
1999
|
+
x = Conformance.nd([rows, cols], Array.new(rows * cols) { |i| Math.sin((i + 0.6) * 1.7) * 0.8 })
|
|
2000
|
+
gamma = Tilework[Conformance.f32(Array.new(cols) { |i| 0.5 + (i * 0.25) })]
|
|
2001
|
+
beta = Tilework[Conformance.f32(Array.new(cols) { |i| (i % 3) - 1.0 })]
|
|
2002
|
+
labels = Buffer.new(:int32, rows, shape: [rows])
|
|
2003
|
+
labels.write(Array.new(rows) { |i| i % cols })
|
|
2004
|
+
|
|
2005
|
+
build = lambda do
|
|
2006
|
+
Tilework::Graph.build(differentiable: true) do |g|
|
|
2007
|
+
t = g.input(:x, [rows, cols])
|
|
2008
|
+
gm = g.param(gamma)
|
|
2009
|
+
bt = g.param(beta)
|
|
2010
|
+
# For each of the four functions, fold to a scalar and take gradients
|
|
2011
|
+
# with respect to x and gamma. Weighting before folding keeps the
|
|
2012
|
+
# derivative from vanishing for expressions whose row sums are
|
|
2013
|
+
# constant (layernorm).
|
|
2014
|
+
losses = [
|
|
2015
|
+
(NN.softmax(t) * t).sum,
|
|
2016
|
+
(NN.layernorm(t, gamma: gm, beta: bt) * t).sum,
|
|
2017
|
+
(NN.rmsnorm(t, gamma: gm) * t).sum,
|
|
2018
|
+
NN.cross_entropy(t, g.param(labels), reduction: :sum),
|
|
2019
|
+
NN.cross_entropy(t, g.param(labels)),
|
|
2020
|
+
]
|
|
2021
|
+
losses.each do |loss|
|
|
2022
|
+
g.output(loss)
|
|
2023
|
+
g.grad(loss, wrt: [t, gm, bt]).each { |grad| g.output(grad) }
|
|
2024
|
+
end
|
|
2025
|
+
end.compile
|
|
2026
|
+
end
|
|
2027
|
+
|
|
2028
|
+
previous = Tilework::Graph.lower_high_level
|
|
2029
|
+
begin
|
|
2030
|
+
Tilework::Graph.lower_high_level = false
|
|
2031
|
+
fast_plan = build.call
|
|
2032
|
+
fast = fast_plan.run(x: x).map(&:to_a)
|
|
2033
|
+
Tilework::Graph.lower_high_level = true
|
|
2034
|
+
lowered_plan = build.call
|
|
2035
|
+
lowered = lowered_plan.run(x: x).map(&:to_a)
|
|
2036
|
+
ensure
|
|
2037
|
+
Tilework::Graph.lower_high_level = previous
|
|
2038
|
+
end
|
|
2039
|
+
|
|
2040
|
+
# On the fast path all four stay :nn nodes (not decomposed). Otherwise both
|
|
2041
|
+
# sides would be decompositions and the comparison would pass trivially.
|
|
2042
|
+
Conformance.assert(fast_plan.nodes.count { |n| n.op == :nn } >= 4,
|
|
2043
|
+
"the fast path should keep the hand-written kernels as :nn nodes")
|
|
2044
|
+
Conformance.assert(lowered_plan.nodes.none? { |n| n.op == :nn },
|
|
2045
|
+
"the lowered path should have no :nn node left")
|
|
2046
|
+
|
|
2047
|
+
names = %w[softmax layernorm rmsnorm cross_entropy_sum cross_entropy_mean]
|
|
2048
|
+
fast.each_index do |i|
|
|
2049
|
+
what = "#{names[i / 4]} #{%w[loss d/dx d/dgamma d/dbeta][i % 4]}"
|
|
2050
|
+
begin
|
|
2051
|
+
Conformance.assert_close(lowered[i], fast[i], 1e-5)
|
|
2052
|
+
rescue RuntimeError => e
|
|
2053
|
+
raise "#{what}: the hand-written vjp disagrees with the decomposed one (#{e.message})"
|
|
2054
|
+
end
|
|
2055
|
+
end
|
|
2056
|
+
end
|
|
2057
|
+
|
|
2058
|
+
# Row fusion works even when a row reduction's result is read in two places.
|
|
2059
|
+
#
|
|
2060
|
+
# If two clusters pull in the same broadcast, only the first holds the
|
|
2061
|
+
# subtree, and the later one references an absorbed node as input and fails
|
|
2062
|
+
# with a nil buffer. Row fusion only kicks in above MIN_ROWS, so small-matrix
|
|
2063
|
+
# checks never show it. Norm vjps produce this shape.
|
|
2064
|
+
register(:graph_row_fusion_handles_a_reduction_read_twice) do
|
|
2065
|
+
rows = Tilework::Graph::Fusion::MIN_ROWS
|
|
2066
|
+
cols = 4
|
|
2067
|
+
x = Conformance.nd([rows, cols], Array.new(rows * cols) { |i| Math.sin((i + 0.7) * 0.13) })
|
|
2068
|
+
|
|
2069
|
+
build = lambda do
|
|
2070
|
+
Tilework::Graph.build do |g|
|
|
2071
|
+
t = g.input(:x, [rows, cols])
|
|
2072
|
+
# **An expression before the reduction** is needed; it has one consumer,
|
|
2073
|
+
# so the first cluster absorbs it.
|
|
2074
|
+
mean = (t * t).mean(axis: -1, keepdims: true)
|
|
2075
|
+
# Then two expressions read the folded value (= two readers of the
|
|
2076
|
+
# broadcast). The later cluster used to reference the absorbed `t * t`
|
|
2077
|
+
# as input.
|
|
2078
|
+
g.output((t - mean) * 2.0)
|
|
2079
|
+
g.output((t + mean) * 0.5)
|
|
2080
|
+
end.compile
|
|
2081
|
+
end
|
|
2082
|
+
|
|
2083
|
+
previous = Tilework::Graph.row_fusion
|
|
2084
|
+
begin
|
|
2085
|
+
Tilework::Graph.row_fusion = true
|
|
2086
|
+
fused = build.call.run(x: x).map(&:to_a)
|
|
2087
|
+
Tilework::Graph.row_fusion = false
|
|
2088
|
+
plain = build.call.run(x: x).map(&:to_a)
|
|
2089
|
+
ensure
|
|
2090
|
+
Tilework::Graph.row_fusion = previous
|
|
2091
|
+
end
|
|
2092
|
+
|
|
2093
|
+
plain.each_with_index { |values, i| Conformance.assert_close_relative(values, fused[i], 1e-5) }
|
|
2094
|
+
end
|
|
2095
|
+
|
|
2096
|
+
# Gradients point "the right way". Global mistakes that per-op checks miss
|
|
2097
|
+
# (the reduction's divisor, the update's sign, writing back to a stale
|
|
2098
|
+
# version) only show as training not progressing. Run a small MLP for a
|
|
2099
|
+
# few steps and check only that the loss decreases monotonically.
|
|
2100
|
+
register(:graph_a_training_step_reduces_the_loss) do
|
|
2101
|
+
n, d, hidden, classes = 8, 3, 6, 2
|
|
2102
|
+
xs = Conformance.nd([n, d], Array.new(n * d) { |i| Math.sin(i * 1.1) })
|
|
2103
|
+
labels = Buffer.new(:int32, n, shape: [n])
|
|
2104
|
+
labels.write(Array.new(n) { |i| i % classes })
|
|
2105
|
+
w1 = Conformance.nd([d, hidden], Array.new(d * hidden) { |i| Math.cos(i * 0.7) * 0.5 })
|
|
2106
|
+
w2 = Conformance.nd([hidden, classes], Array.new(hidden * classes) { |i| Math.sin(i * 0.5) * 0.5 })
|
|
2107
|
+
|
|
2108
|
+
plan = Tilework::Graph.build(differentiable: true) do |g|
|
|
2109
|
+
x = g.input(:x, [n, d])
|
|
2110
|
+
s1 = g.state(w1, name: :w1)
|
|
2111
|
+
s2 = g.state(w2, name: :w2)
|
|
2112
|
+
p1 = g.read(s1)
|
|
2113
|
+
p2 = g.read(s2)
|
|
2114
|
+
loss = NN.cross_entropy(NN.gelu(x.matmul(p1)).matmul(p2), g.param(labels))
|
|
2115
|
+
d1, d2 = g.grad(loss, wrt: [p1, p2])
|
|
2116
|
+
# Gradients and updates live in the same plan (the very reason for a
|
|
2117
|
+
# graph rather than a tape).
|
|
2118
|
+
g.write_into(s1, value: p1 - (0.5 * d1))
|
|
2119
|
+
g.write_into(s2, value: p2 - (0.5 * d2))
|
|
2120
|
+
g.output(loss)
|
|
2121
|
+
end.compile
|
|
2122
|
+
|
|
2123
|
+
losses = Array.new(12) { plan.run(x: xs).to_a[0] }
|
|
2124
|
+
Conformance.assert(losses.each_cons(2).all? { |a, b| b <= a + 1e-6 },
|
|
2125
|
+
"the loss did not fall monotonically: #{losses.inspect}")
|
|
2126
|
+
Conformance.assert(losses.last < losses.first * 0.9,
|
|
2127
|
+
"the loss barely moved: #{losses.first} -> #{losses.last}")
|
|
2128
|
+
end
|
|
2129
|
+
|
|
2130
|
+
# Whether the budget holds. peak_bytes is measured right after everything
|
|
2131
|
+
# needed is loaded (the point of peak const occupancy).
|
|
2132
|
+
register(:graph_offload_respects_the_budget) do
|
|
2133
|
+
weights = Array.new(5) { Conformance.square(4) }
|
|
2134
|
+
x = Conformance.square(4)
|
|
2135
|
+
build = lambda do |vram|
|
|
2136
|
+
Tilework::Graph.build do |g|
|
|
2137
|
+
value = g.input(:x, [4, 4])
|
|
2138
|
+
weights.each { |w| value = value.matmul(g.param(w)) }
|
|
2139
|
+
g.output(value)
|
|
2140
|
+
end.compile(vram: vram)
|
|
2141
|
+
end
|
|
2142
|
+
|
|
2143
|
+
probe = build.call(1 << 30).residency
|
|
2144
|
+
slot = probe.weights / weights.length
|
|
2145
|
+
budget = probe.reserved + (2 * slot)
|
|
2146
|
+
plan = build.call(budget)
|
|
2147
|
+
begin
|
|
2148
|
+
2.times { plan.run(x: x) }
|
|
2149
|
+
peak = plan.offload_stats[:peak_bytes]
|
|
2150
|
+
Conformance.assert(peak <= budget - probe.reserved,
|
|
2151
|
+
"paged weights peaked at #{peak} bytes, over the #{budget - probe.reserved} allowed")
|
|
2152
|
+
ensure
|
|
2153
|
+
plan.free_offload!
|
|
2154
|
+
end
|
|
2155
|
+
end
|
|
2156
|
+
|
|
2157
|
+
# Tilework.submit (#71) changes only how often the host waits. Device-side
|
|
2158
|
+
# order stays on the same queue, so it should be **bit-identical** to
|
|
2159
|
+
# synchronous eager. A mismatch means an op that must not go async did, or
|
|
2160
|
+
# a call leaving the queue was not waited on.
|
|
2161
|
+
#
|
|
2162
|
+
# A reduction (freeing intermediates as it goes), an NN fused kernel and a
|
|
2163
|
+
# free inside submit are mixed into one chain because that is exactly how
|
|
2164
|
+
# they combine in eager code.
|
|
2165
|
+
register(:submit_matches_the_synchronous_eager_path) do
|
|
2166
|
+
x = Tilework[Conformance.f32(Array.new(24) { |i| ((i % 7) - 3) * 0.5 })].reshape(4, 6)
|
|
2167
|
+
gamma = Tilework[Conformance.f32(Array.new(6) { |i| 1.0 + (i * 0.1) })]
|
|
2168
|
+
|
|
2169
|
+
expression = lambda do
|
|
2170
|
+
h = Tilework::NN.rmsnorm(x, gamma: gamma)
|
|
2171
|
+
scaled = h * 2.0
|
|
2172
|
+
probs = Tilework::NN.softmax(Tilework::NN.gelu(scaled) + 1.0)
|
|
2173
|
+
scaled.free # deferred until completion inside submit
|
|
2174
|
+
[probs.to_a, Tilework::Ops.reduce(:sum, probs)]
|
|
2175
|
+
end
|
|
2176
|
+
|
|
2177
|
+
expected_probs, expected_sum = expression.call
|
|
2178
|
+
actual_probs, actual_sum = Tilework.submit(&expression).wait
|
|
2179
|
+
|
|
2180
|
+
Conformance.assert_equal_arrays(expected_probs, actual_probs)
|
|
2181
|
+
Conformance.assert(expected_sum == actual_sum,
|
|
2182
|
+
"submit changed a reduction: #{expected_sum} vs #{actual_sum}")
|
|
2183
|
+
end
|
|
2184
|
+
|
|
2185
|
+
# A matmul between two queued kernels. Where the GEMM is delegated it is
|
|
2186
|
+
# submitted separately from our kernels, so it has to land **behind** the
|
|
2187
|
+
# work that produces its inputs and ahead of the work that reads its
|
|
2188
|
+
# output. A bridge that batches asynchronous launches gets this wrong by
|
|
2189
|
+
# submitting the GEMM while the producing kernel is still unsubmitted —
|
|
2190
|
+
# the GEMM then reads the operand before it is written.
|
|
2191
|
+
register(:submit_orders_a_matmul_between_queued_kernels) do
|
|
2192
|
+
n = 8
|
|
2193
|
+
x = Tilework[Conformance.f32(Array.new(n * n) { |i| ((i % 7) - 3) * 0.25 })].reshape(n, n)
|
|
2194
|
+
w = Tilework[Conformance.f32(Array.new(n * n) { |i| ((i % 5) - 2) * 0.5 })].reshape(n, n)
|
|
2195
|
+
|
|
2196
|
+
expression = -> { ((x * 3.0) + 1.0).matmul(w) * 0.5 }
|
|
2197
|
+
|
|
2198
|
+
expected = expression.call.to_a
|
|
2199
|
+
actual = Tilework.submit(&expression).wait.to_a
|
|
2200
|
+
Conformance.assert_equal_arrays(expected, actual)
|
|
2201
|
+
end
|
|
2202
|
+
|
|
2203
|
+
# Buffers made inside submit must not be freed before completion. Releasing
|
|
2204
|
+
# memory a running kernel reads is undefined behavior on CUDA/HIP, so this
|
|
2205
|
+
# pins not only that values match but that the buffer lives until the boundary.
|
|
2206
|
+
register(:submit_defers_a_free_to_the_boundary) do
|
|
2207
|
+
a = Tilework[Conformance.f32([1.0, 2.0, 3.0, 4.0])]
|
|
2208
|
+
temp = nil
|
|
2209
|
+
|
|
2210
|
+
job = Tilework.submit do
|
|
2211
|
+
temp = a * 2.0
|
|
2212
|
+
temp.free
|
|
2213
|
+
Conformance.assert(!temp.freed?, "a buffer freed inside submit must stay alive until the boundary")
|
|
2214
|
+
temp + 1.0
|
|
2215
|
+
end
|
|
2216
|
+
|
|
2217
|
+
Conformance.assert(!temp.freed?, "the deferred free must not run before wait")
|
|
2218
|
+
out = job.wait
|
|
2219
|
+
Conformance.assert(temp.freed?, "the deferred free must run at the boundary")
|
|
2220
|
+
Conformance.assert_equal_arrays([3.0, 5.0, 7.0, 9.0], out.to_a)
|
|
2221
|
+
end
|
|
2222
|
+
|
|
2223
|
+
# #86: ops that free their own temporaries mid-call (:nn / whole-buffer
|
|
2224
|
+
# reductions / batched matmul without a delegate) no longer run
|
|
2225
|
+
# synchronously, since the execution context takes over those frees.
|
|
2226
|
+
# **Values must not change** — if they do, a freed slot is being lent to
|
|
2227
|
+
# the next node while still in use.
|
|
2228
|
+
#
|
|
2229
|
+
# Three consecutive runs because double-lending a slot does not show on the first.
|
|
2230
|
+
register(:graph_deferred_free_matches_synchronous_execution) do
|
|
2231
|
+
state_buffer = -> { Tilework[Conformance.f32([0.5, -1.5, 2.0, 0.25, 1.0, -0.5])].reshape(3, 2) }
|
|
2232
|
+
|
|
2233
|
+
build = lambda do |cache|
|
|
2234
|
+
Tilework::Graph.build do |g|
|
|
2235
|
+
x = g.input(:x, [4, 6])
|
|
2236
|
+
h = Tilework::NN.silu(Tilework::NN.rmsnorm(x))
|
|
2237
|
+
total = h.sum # whole-buffer reduction (tree fold, freeing partials midway)
|
|
2238
|
+
s0 = g.state(cache, name: :acc)
|
|
2239
|
+
s1 = g.write_into(s0, at: 0..1, value: (g.read(s0, 0..1) * 1.5))
|
|
2240
|
+
g.output(h + total)
|
|
2241
|
+
g.output(g.read(s1, 0..2).reshape(6))
|
|
2242
|
+
end.compile
|
|
2243
|
+
end
|
|
2244
|
+
|
|
2245
|
+
x = Tilework[Conformance.f32(Array.new(24) { |i| ((i % 5) - 2) * 0.75 })].reshape(4, 6)
|
|
2246
|
+
previous = Tilework::Graph.async
|
|
2247
|
+
begin
|
|
2248
|
+
Tilework::Graph.async = false
|
|
2249
|
+
serial_cache = state_buffer.call
|
|
2250
|
+
serial = build.call(serial_cache)
|
|
2251
|
+
expected = 3.times.map { serial.run(x: x).map(&:to_a) }
|
|
2252
|
+
|
|
2253
|
+
Tilework::Graph.async = true
|
|
2254
|
+
cache = state_buffer.call
|
|
2255
|
+
plan = build.call(cache)
|
|
2256
|
+
actual = 3.times.map { plan.run(x: x).map(&:to_a) }
|
|
2257
|
+
ensure
|
|
2258
|
+
Tilework::Graph.async = previous
|
|
2259
|
+
end
|
|
2260
|
+
|
|
2261
|
+
expected.zip(actual).each do |want, got|
|
|
2262
|
+
want.zip(got).each { |a, b| Conformance.assert_equal_arrays(a, b) }
|
|
2263
|
+
end
|
|
2264
|
+
Conformance.assert(plan.sync_stats[:synchronous].zero?,
|
|
2265
|
+
"no node here should still wait, got #{plan.sync_stats.inspect}")
|
|
2266
|
+
end
|
|
2267
|
+
|
|
2268
|
+
# #86 stages 2-3: several plans run inside one submit give the same values as
|
|
2269
|
+
# running them standalone. A plan installs its own execution context, so
|
|
2270
|
+
# without ownership linked to the outer Job, frees of buffers made in the
|
|
2271
|
+
# submit block would land outside the boundary.
|
|
2272
|
+
register(:graph_inside_submit_matches_the_standalone_plans) do
|
|
2273
|
+
first = Tilework::Graph.build do |g|
|
|
2274
|
+
g.output(Tilework::NN.silu(g.input(:x, [4, 6])) * 2.0)
|
|
2275
|
+
end.compile
|
|
2276
|
+
second = Tilework::Graph.build do |g|
|
|
2277
|
+
g.output(Tilework::NN.softmax(g.input(:y, [4, 6])) + 0.5)
|
|
2278
|
+
end.compile
|
|
2279
|
+
|
|
2280
|
+
source = Tilework[Conformance.f32(Array.new(24) { |i| ((i % 6) - 2) * 0.5 })].reshape(4, 6)
|
|
2281
|
+
|
|
2282
|
+
chain = lambda do
|
|
2283
|
+
scaled = source * 1.25
|
|
2284
|
+
out = second.run(y: first.run(x: scaled))
|
|
2285
|
+
scaled.free # deferred to the boundary inside submit
|
|
2286
|
+
out
|
|
2287
|
+
end
|
|
2288
|
+
|
|
2289
|
+
expected = chain.call.to_a
|
|
2290
|
+
job = Tilework.submit { chain.call }
|
|
2291
|
+
actual = job.wait.to_a
|
|
2292
|
+
|
|
2293
|
+
Conformance.assert_equal_arrays(expected, actual)
|
|
2294
|
+
end
|
|
2295
|
+
end
|
|
2296
|
+
end
|