tilework 0.18.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE +21 -0
- data/lib/tilework/backend/cpu/dsl_codegen.rb +509 -0
- data/lib/tilework/backend/cpu/ops_codegen.rb +451 -0
- data/lib/tilework/backend/cpu.rb +156 -0
- data/lib/tilework/buffer.rb +753 -0
- data/lib/tilework/capabilities.rb +183 -0
- data/lib/tilework/compat/cuda.h +66 -0
- data/lib/tilework/compat/hip.h +35 -0
- data/lib/tilework/compat/metal.h +45 -0
- data/lib/tilework/conformance.rb +2296 -0
- data/lib/tilework/core_ext.rb +21 -0
- data/lib/tilework/dialect.rb +67 -0
- data/lib/tilework/dsl.rb +722 -0
- data/lib/tilework/dtype/bfloat.rb +31 -0
- data/lib/tilework/dtype/cast.rb +88 -0
- data/lib/tilework/dtype/half.rb +98 -0
- data/lib/tilework/graph/builder.rb +1073 -0
- data/lib/tilework/graph/fusion.rb +520 -0
- data/lib/tilework/graph/grad.rb +443 -0
- data/lib/tilework/graph/high_level.rb +266 -0
- data/lib/tilework/graph/kernels.rb +403 -0
- data/lib/tilework/graph/node.rb +27 -0
- data/lib/tilework/graph/op.rb +290 -0
- data/lib/tilework/graph/optim.rb +183 -0
- data/lib/tilework/graph/plan.rb +1086 -0
- data/lib/tilework/graph/pool.rb +178 -0
- data/lib/tilework/graph/region.rb +210 -0
- data/lib/tilework/graph/residency.rb +449 -0
- data/lib/tilework/graph/state.rb +64 -0
- data/lib/tilework/graph/tensor.rb +124 -0
- data/lib/tilework/graph.rb +213 -0
- data/lib/tilework/kernel.rb +163 -0
- data/lib/tilework/math/conv.rb +270 -0
- data/lib/tilework/math/fft.rb +172 -0
- data/lib/tilework/math/nn.rb +1103 -0
- data/lib/tilework/math/philox.rb +194 -0
- data/lib/tilework/math/random.rb +232 -0
- data/lib/tilework/math/tensor_core.rb +269 -0
- data/lib/tilework/ops.rb +682 -0
- data/lib/tilework/runtime/offload.rb +498 -0
- data/lib/tilework/runtime/submission.rb +163 -0
- data/lib/tilework/runtime/telemetry.rb +112 -0
- data/lib/tilework/scope.rb +33 -0
- data/lib/tilework/version.rb +5 -0
- data/lib/tilework.rb +345 -0
- metadata +92 -0
|
@@ -0,0 +1,449 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Tilework
|
|
4
|
+
module Graph
|
|
5
|
+
# Pages weights (:const node buffers) in and out to fit a VRAM budget
|
|
6
|
+
# (#17 Phase 4 / docs/graph-offload.md).
|
|
7
|
+
#
|
|
8
|
+
# Manual Offload (buffer.rb / runtime/offload.rb) is mechanism without
|
|
9
|
+
# policy — which buffer, when, and how far ahead are all the caller's call,
|
|
10
|
+
# and Offload.pipeline's depth: is chosen by hand. A plan knows the
|
|
11
|
+
# execution order and every shape before touching the device, so it takes
|
|
12
|
+
# over those decisions here.
|
|
13
|
+
#
|
|
14
|
+
# The budget is split statically at compile time:
|
|
15
|
+
#
|
|
16
|
+
# pageable budget = vram - (inputs/outputs + state + peak intermediates + pinned consts)
|
|
17
|
+
#
|
|
18
|
+
# Eviction is **cyclic Belady** — drop whatever is used farthest in the
|
|
19
|
+
# future. The execution order is static, so the optimal policy is
|
|
20
|
+
# available; making it cyclic is the key, otherwise a decode loop drops
|
|
21
|
+
# everything at the end of each run and reloads it at the start (a graph
|
|
22
|
+
# that should fit thrashes).
|
|
23
|
+
#
|
|
24
|
+
# **The victim is chosen at run time.** The plan holds statically only
|
|
25
|
+
# "what is needed at which index" and decides what to drop as it runs. The
|
|
26
|
+
# 4-weights-in-3-slots example (docs/graph-offload.md) shows why: in steady
|
|
27
|
+
# state the victim rotates from run to run, which a static index -> eviction
|
|
28
|
+
# table cannot express. The next-use table is static, so the run-time
|
|
29
|
+
# decision is a single O(pageable entries) scan.
|
|
30
|
+
class Residency
|
|
31
|
+
# The unit of paging: one :const node and its buffer.
|
|
32
|
+
#
|
|
33
|
+
# uses execution positions reading this weight (ascending); next_use wraps around
|
|
34
|
+
# has_file whether an offload file exists already. If not, the first
|
|
35
|
+
# eviction writes it (offload!); otherwise VRAM is just dropped (evict!)
|
|
36
|
+
Entry = Struct.new(:node_id, :buffer, :bytes, :uses, :has_file, :spilled) do
|
|
37
|
+
def resident? = !buffer.freed? && buffer.resident?
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
# What is decided statically: the interface for saying, before running,
|
|
41
|
+
# whether it fits and how much moves per run (#17's value 5).
|
|
42
|
+
# bytes_per_run is **an upper-side estimate**; offload_stats[:bytes_in] is
|
|
43
|
+
# the measurement. A sequential scan lands exactly on floor, but a tight
|
|
44
|
+
# budget keeps the eviction victim from rotating fully and exceeds floor
|
|
45
|
+
# (4 weights in 2 slots is an example).
|
|
46
|
+
Report = Struct.new(:vram, :reserved, :budget, :weights, :paged, :resident_always,
|
|
47
|
+
:resident_always_bytes, :bytes_per_run, :floor, :steps,
|
|
48
|
+
:prefetch_headroom, keyword_init: true) do
|
|
49
|
+
# How many prefetches can overlap is **headroom / one weight's size**.
|
|
50
|
+
# "Greater than 0" is not enough — Jetson's tight budget had exactly one
|
|
51
|
+
# weight of headroom, but a single transfer in flight gave no bandwidth
|
|
52
|
+
# (0.66 GB/s), over 3x below the half budget's two (2.16 GB/s). 4b's gain
|
|
53
|
+
# needs **at least two weights** of headroom.
|
|
54
|
+
def prefetch_slots(weight_bytes)
|
|
55
|
+
return 0 unless weight_bytes.to_i.positive?
|
|
56
|
+
|
|
57
|
+
prefetch_headroom.to_i / weight_bytes
|
|
58
|
+
end
|
|
59
|
+
# The lower bound on bytes moved per run. A run touches every weight, so
|
|
60
|
+
# at most budget carries over; the difference moves under any policy.
|
|
61
|
+
def optimal? = bytes_per_run <= floor + 1
|
|
62
|
+
|
|
63
|
+
def to_s
|
|
64
|
+
pinned = resident_always.positive? ? format(", %d pinned (%s)", resident_always, Residency.human(resident_always_bytes)) : ""
|
|
65
|
+
format("vram %s, reserved %s, weights %s of %s, paged %d of %d%s, bytes/run <= %s (floor %s), prefetch headroom %s",
|
|
66
|
+
Residency.human(vram), Residency.human(reserved), Residency.human(budget),
|
|
67
|
+
Residency.human(weights), paged, paged + resident_always, pinned,
|
|
68
|
+
Residency.human(bytes_per_run), Residency.human(floor),
|
|
69
|
+
Residency.human(prefetch_headroom.to_i))
|
|
70
|
+
end
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
def self.human(bytes)
|
|
74
|
+
units = %w[B KB MB GB TB]
|
|
75
|
+
value = bytes.to_f
|
|
76
|
+
unit = 0
|
|
77
|
+
while value >= 1024.0 && unit < units.length - 1
|
|
78
|
+
value /= 1024.0
|
|
79
|
+
unit += 1
|
|
80
|
+
end
|
|
81
|
+
format("%.2f %s", value, units[unit])
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
def self.bytes_of(dtype, length) = length * Buffer::DTYPES.fetch(dtype)[:size]
|
|
85
|
+
|
|
86
|
+
# The VRAM prefetches may hold: **the budget minus the largest single
|
|
87
|
+
# step's need**, which directly decides how many can overlap.
|
|
88
|
+
#
|
|
89
|
+
# The subtraction is for correctness: a prefetching buffer cannot be
|
|
90
|
+
# evicted (it is not even resident yet), so prefetching too much would
|
|
91
|
+
# leave a cold fetch that must succeed unable to make room. compile only
|
|
92
|
+
# guarantees that every single step's need fits in budget, so one step's
|
|
93
|
+
# worth is always kept free.
|
|
94
|
+
#
|
|
95
|
+
# As a side effect it **also caps performance**: Jetson's tight budget
|
|
96
|
+
# (2 slots, one step needing 1) leaves 1 slot of headroom, so one transfer
|
|
97
|
+
# in flight = no concurrency, and 4b's bandwidth does not show (0.66 GB/s
|
|
98
|
+
# measured vs 2.16 GB/s at the half budget). A budget buys **the ability
|
|
99
|
+
# to overlap**, not only bytes.
|
|
100
|
+
def self.prefetch_ceiling(needs, budget)
|
|
101
|
+
budget - needs.each_value.filter_map { |list| list&.sum(&:bytes) }.max.to_i
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
# How many transfers prefetch keeps in flight. On Jetson
|
|
105
|
+
# (docs/graph-offload.md, "Corrected by measurement") depth=2 gave
|
|
106
|
+
# 1.85 GB/s and depth=4 2.24 GB/s — about 40% more, but most of it already
|
|
107
|
+
# at 2, so 2 is the default as the balance against extra resident VRAM.
|
|
108
|
+
PREFETCH_DEPTH = 2
|
|
109
|
+
|
|
110
|
+
# How many steps ahead to look for the next demand. The scan only runs
|
|
111
|
+
# until it finds an unfilled demand and returns at once on a miss, so a
|
|
112
|
+
# wide window is cheap — up to one full cycle.
|
|
113
|
+
LOOKAHEAD_HOPS = 64
|
|
114
|
+
|
|
115
|
+
attr_reader :entries, :report
|
|
116
|
+
|
|
117
|
+
# entries: the pageable Entry list
|
|
118
|
+
# needs: execution position -> Entries that must be resident there
|
|
119
|
+
# steps: executed nodes per run (next_use's wraparound width)
|
|
120
|
+
# budget: bytes available for paging
|
|
121
|
+
def initialize(entries:, needs:, steps:, budget:, report:, spill_dir:, backing: :file)
|
|
122
|
+
@entries = entries
|
|
123
|
+
@by_id = entries.to_h { |entry| [entry.node_id, entry] }
|
|
124
|
+
@needs = needs
|
|
125
|
+
# Only positions with demand. Prefetch just walks this forward, which
|
|
126
|
+
# misses less than stepping over positions without demand one by one.
|
|
127
|
+
# needs is a Hash with a default (Plan#residency_needs), so build from
|
|
128
|
+
# keys to avoid [] inserting empty arrays.
|
|
129
|
+
@demand_positions = needs.keys.select { |i| !needs[i].to_a.empty? }.sort
|
|
130
|
+
@steps = steps
|
|
131
|
+
@budget = budget
|
|
132
|
+
# The ceiling prefetches may occupy (derivation in Residency.prefetch_ceiling).
|
|
133
|
+
@prefetch_ceiling = Residency.prefetch_ceiling(needs, budget)
|
|
134
|
+
@report = report
|
|
135
|
+
@spill_dir = spill_dir
|
|
136
|
+
@backing = backing
|
|
137
|
+
@spilled_paths = []
|
|
138
|
+
# node_id -> Offload::Transfer: prefetches issued but not yet resident.
|
|
139
|
+
# Their VRAM is allocated at issue time (see buffer.rb's restore_async!),
|
|
140
|
+
# so they must count toward occupancy.
|
|
141
|
+
@pending = {}
|
|
142
|
+
# The plan does not synchronize at its end, so the queue is assumed dirty on entry.
|
|
143
|
+
@queue_dirty = true
|
|
144
|
+
reset_stats!
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
def reset_stats!
|
|
148
|
+
@fetches = 0
|
|
149
|
+
@evictions = 0
|
|
150
|
+
@bytes_in = 0
|
|
151
|
+
@spills = 0
|
|
152
|
+
@spill_bytes = 0
|
|
153
|
+
@waited = 0.0
|
|
154
|
+
@added_fences = 0
|
|
155
|
+
@free_fences = 0
|
|
156
|
+
@peak = 0
|
|
157
|
+
end
|
|
158
|
+
|
|
159
|
+
def stats
|
|
160
|
+
{ fetches: @fetches, bytes_in: @bytes_in, waited_seconds: @waited.round(6),
|
|
161
|
+
evictions: @evictions, spills: @spills, spill_bytes: @spill_bytes,
|
|
162
|
+
added_fences: @added_fences, free_fences: @free_fences, peak_bytes: @peak,
|
|
163
|
+
prefetched: @pending.length }
|
|
164
|
+
end
|
|
165
|
+
|
|
166
|
+
# Called before running the node at execution position index: loads the
|
|
167
|
+
# weights it needs, making room with Belady when short.
|
|
168
|
+
#
|
|
169
|
+
# 4a fetched synchronously here (as did the early docs/graph-offload.md).
|
|
170
|
+
# On Jetson the same transfers measured 0.56 GB/s one at a time
|
|
171
|
+
# synchronously vs 1.85–2.24 GB/s asynchronously at depth 2–4, a 3–4x gap
|
|
172
|
+
# (Offload.pipeline prefetched asynchronously while 4a called `restore!`
|
|
173
|
+
# one by one). So "fetch synchronously here if it has not arrived" is
|
|
174
|
+
# demoted to **the fallback on a miss**, and the main path harvests
|
|
175
|
+
# transfers `prefetch!` set up asynchronously beforehand (#17 Phase 4b).
|
|
176
|
+
def ensure!(index)
|
|
177
|
+
needed = @needs[index] || []
|
|
178
|
+
cold = []
|
|
179
|
+
needed.each do |entry|
|
|
180
|
+
next if entry.resident?
|
|
181
|
+
|
|
182
|
+
if (transfer = @pending.delete(entry.node_id))
|
|
183
|
+
wait_transfer(entry, transfer)
|
|
184
|
+
else
|
|
185
|
+
cold << entry
|
|
186
|
+
end
|
|
187
|
+
end
|
|
188
|
+
# Besides loading what is missing, shrink when over budget (a plan
|
|
189
|
+
# binding resident Buffers starts over budget).
|
|
190
|
+
#
|
|
191
|
+
# make_room! is called even when cold is empty: with want=0 it still
|
|
192
|
+
# shrinks if occupancy exceeds budget. Calling it only when cold is
|
|
193
|
+
# non-empty would never detect being over budget whenever the demand
|
|
194
|
+
# happens to be satisfied already.
|
|
195
|
+
make_room!(cold.sum(&:bytes), keep: needed, from: index)
|
|
196
|
+
cold.each { |entry| fetch_sync(entry) }
|
|
197
|
+
prefetch!(index, needed) if Graph.offload_prefetch && Offload.async?
|
|
198
|
+
# Staying within budget can only be shown by measurement, so record the
|
|
199
|
+
# moment right after everything needed is loaded (= peak const
|
|
200
|
+
# occupancy). VRAM held by prefetches is charged here too, hence
|
|
201
|
+
# occupied_bytes.
|
|
202
|
+
@peak = [@peak, occupied_bytes].max
|
|
203
|
+
nil
|
|
204
|
+
end
|
|
205
|
+
|
|
206
|
+
# Sets up near-future demand asynchronously before it is due. **Issues
|
|
207
|
+
# only, never waits** — waiting happens in ensure! (wait_transfer) when
|
|
208
|
+
# the index that needs the weight arrives. Several in flight overlap, so
|
|
209
|
+
# the depth effect measured on Jetson carries over.
|
|
210
|
+
#
|
|
211
|
+
# Failure never breaks the run: if no room can be made it quietly gives
|
|
212
|
+
# up and that weight is fetched synchronously at its demand point as
|
|
213
|
+
# before (the cold fallback always works, so with zero prefetches the
|
|
214
|
+
# result matches 4a).
|
|
215
|
+
def prefetch!(index, needed)
|
|
216
|
+
return if @pending.length >= PREFETCH_DEPTH
|
|
217
|
+
return if @prefetch_ceiling <= 0 # the largest single step's need uses the whole budget
|
|
218
|
+
|
|
219
|
+
candidates = upcoming_targets(index, needed)
|
|
220
|
+
candidates.each do |entry, target_index, keep|
|
|
221
|
+
break if @pending.length >= PREFETCH_DEPTH
|
|
222
|
+
|
|
223
|
+
next if entry.resident? || @pending.key?(entry.node_id)
|
|
224
|
+
next unless pending_bytes + entry.bytes <= @prefetch_ceiling
|
|
225
|
+
|
|
226
|
+
begin
|
|
227
|
+
make_room!(entry.bytes, keep: keep, from: target_index) unless occupied_bytes + entry.bytes <= @budget
|
|
228
|
+
rescue Error
|
|
229
|
+
next # no room; this weight waits for its demand point
|
|
230
|
+
end
|
|
231
|
+
fetch_async(entry)
|
|
232
|
+
end
|
|
233
|
+
nil
|
|
234
|
+
end
|
|
235
|
+
|
|
236
|
+
# Searches from after index up to LOOKAHEAD_HOPS ahead for weights not
|
|
237
|
+
# yet resident and not already prefetching. Returns each as (Entry,
|
|
238
|
+
# demand position, set of Entries to protect until then).
|
|
239
|
+
#
|
|
240
|
+
# **keep starts with the current step's (index) demand.** ensure! has
|
|
241
|
+
# made all of index's demand resident before getting here, but the node
|
|
242
|
+
# has not run yet — evicting one in this call would make the very next
|
|
243
|
+
# node read freed VRAM. Belady could misjudge it as distant (it looks
|
|
244
|
+
# unused until the next cycle), so it must be protected explicitly rather
|
|
245
|
+
# than by proximity.
|
|
246
|
+
def upcoming_targets(index, needed)
|
|
247
|
+
return [] if @demand_positions.empty?
|
|
248
|
+
|
|
249
|
+
start = @demand_positions.bsearch_index { |pos| pos > index } || 0
|
|
250
|
+
seen = {}
|
|
251
|
+
keep = needed.dup
|
|
252
|
+
found = []
|
|
253
|
+
LOOKAHEAD_HOPS.times do |hop|
|
|
254
|
+
pos = @demand_positions[(start + hop) % @demand_positions.length]
|
|
255
|
+
break if hop.positive? && pos == @demand_positions[start] # went all the way around
|
|
256
|
+
|
|
257
|
+
(@needs[pos] || []).each do |entry|
|
|
258
|
+
keep << entry
|
|
259
|
+
next if seen[entry.node_id]
|
|
260
|
+
next if entry.resident? || @pending.key?(entry.node_id)
|
|
261
|
+
|
|
262
|
+
seen[entry.node_id] = true
|
|
263
|
+
found << [entry, pos, keep.dup]
|
|
264
|
+
end
|
|
265
|
+
break if found.length >= PREFETCH_DEPTH
|
|
266
|
+
end
|
|
267
|
+
found
|
|
268
|
+
end
|
|
269
|
+
|
|
270
|
+
# The Plan reports the queue state. **Eviction needs a fence**, but if the
|
|
271
|
+
# previous node ran synchronously (:nn / a reduction / a delegated GEMM)
|
|
272
|
+
# its wait already left the queue empty — riding on it adds no
|
|
273
|
+
# Tilework.synchronize at all (#17 Phase 4c).
|
|
274
|
+
#
|
|
275
|
+
# Riding along **only removes waits**; it changes neither the victim nor
|
|
276
|
+
# the timing. Dropping early (freeing future room at the fence) was
|
|
277
|
+
# measured and rejected: dropping early has fewer candidates (weights not
|
|
278
|
+
# yet done are ineligible), so it picks a worse victim than waiting would.
|
|
279
|
+
# Measured, waits went 10 -> 4 but transfers grew 2304 -> 2816 bytes; on
|
|
280
|
+
# a disk-bound workload trading bytes for waits does not pay.
|
|
281
|
+
def note_queue(dirty:)
|
|
282
|
+
@queue_dirty = dirty
|
|
283
|
+
nil
|
|
284
|
+
end
|
|
285
|
+
|
|
286
|
+
# Deletes only the offload files the plan wrote. **Files of weights the
|
|
287
|
+
# user offloaded themselves are left alone** — they are not ours.
|
|
288
|
+
#
|
|
289
|
+
# Before deleting, **return what was borrowed**: if a weight we wrote out
|
|
290
|
+
# is still offloaded, bring it back to VRAM first. Deleting without
|
|
291
|
+
# returning would leave the user's Buffer with neither file nor VRAM,
|
|
292
|
+
# unusable forever.
|
|
293
|
+
#
|
|
294
|
+
# Transfers set up by prefetch that nobody waited on are collected here
|
|
295
|
+
# too; otherwise they keep holding VRAM and the bridge's worker lingers
|
|
296
|
+
# (the same reason as Offload.pipeline's ensure block — see offload.rb).
|
|
297
|
+
def free_spills!
|
|
298
|
+
require "fileutils"
|
|
299
|
+
@pending.each_value(&:wait)
|
|
300
|
+
@pending.clear
|
|
301
|
+
@entries.each do |entry|
|
|
302
|
+
next unless entry.spilled
|
|
303
|
+
next if entry.buffer.freed?
|
|
304
|
+
|
|
305
|
+
entry.buffer.restore! unless entry.resident?
|
|
306
|
+
entry.buffer.discard_host_backing! if @backing == :host
|
|
307
|
+
entry.has_file = false
|
|
308
|
+
end
|
|
309
|
+
@spilled_paths.each { |path| FileUtils.rm_f(path) }
|
|
310
|
+
@spilled_paths.clear
|
|
311
|
+
@entries.each { |entry| entry.spilled = false }
|
|
312
|
+
nil
|
|
313
|
+
end
|
|
314
|
+
|
|
315
|
+
# What is known without running.
|
|
316
|
+
def needs_at(index) = @needs[index] || []
|
|
317
|
+
|
|
318
|
+
private
|
|
319
|
+
|
|
320
|
+
# Evictions are batched under one fence. **evict! is buffer_free**, and a
|
|
321
|
+
# kernel not yet executed may still read that VRAM (the plan does not
|
|
322
|
+
# synchronize between nodes) — the same hazard Pool avoids by returning
|
|
323
|
+
# rather than freeing — so always wait before dropping. Batching means a
|
|
324
|
+
# single wait instead of one per eviction.
|
|
325
|
+
def make_room!(want, keep:, from:)
|
|
326
|
+
free = @budget - occupied_bytes
|
|
327
|
+
return if free >= want
|
|
328
|
+
|
|
329
|
+
victims = choose_victims(want - free, keep: keep, from: from)
|
|
330
|
+
return if victims.empty?
|
|
331
|
+
|
|
332
|
+
# Wait here only if the previous node did not. Rides are counted in
|
|
333
|
+
# free_fences so they can be told apart from added waits.
|
|
334
|
+
if @queue_dirty
|
|
335
|
+
Tilework.synchronize
|
|
336
|
+
@added_fences += 1
|
|
337
|
+
@queue_dirty = false
|
|
338
|
+
else
|
|
339
|
+
@free_fences += 1
|
|
340
|
+
end
|
|
341
|
+
victims.each { |entry| evict(entry) }
|
|
342
|
+
end
|
|
343
|
+
|
|
344
|
+
# Drop whatever is used farthest in the future (Belady). Being cyclic,
|
|
345
|
+
# something unused for the rest of this run is judged by its position in
|
|
346
|
+
# the next run.
|
|
347
|
+
def choose_victims(want, keep:, from:)
|
|
348
|
+
keep_ids = keep.to_h { |entry| [entry.node_id, true] }
|
|
349
|
+
candidates = @entries.select { |entry| entry.resident? && !keep_ids[entry.node_id] }
|
|
350
|
+
.sort_by { |entry| -next_use(entry, from) }
|
|
351
|
+
victims = []
|
|
352
|
+
freed = 0
|
|
353
|
+
candidates.each do |entry|
|
|
354
|
+
break if freed >= want
|
|
355
|
+
|
|
356
|
+
victims << entry
|
|
357
|
+
freed += entry.bytes
|
|
358
|
+
end
|
|
359
|
+
return victims if freed >= want
|
|
360
|
+
|
|
361
|
+
# Unreachable if compile-time checks passed (they verify that the
|
|
362
|
+
# weights one node needs at once fit in the budget).
|
|
363
|
+
raise Error, "cannot free #{Residency.human(want)} of VRAM at step #{from}: " \
|
|
364
|
+
"#{Residency.human(freed)} is all the paged weights hold. " \
|
|
365
|
+
"Raise the vram: budget passed to compile."
|
|
366
|
+
end
|
|
367
|
+
|
|
368
|
+
def next_use(entry, from)
|
|
369
|
+
found = entry.uses.bsearch { |use| use >= from }
|
|
370
|
+
found || (entry.uses.first + @steps)
|
|
371
|
+
end
|
|
372
|
+
|
|
373
|
+
def resident_bytes = @entries.sum { |entry| entry.resident? ? entry.bytes : 0 }
|
|
374
|
+
|
|
375
|
+
# Prefetching weights occupy VRAM too — restore_async! has called
|
|
376
|
+
# buffer_alloc at issue time and resident? stays false until the transfer
|
|
377
|
+
# ends (see buffer.rb's __finish_restore__). Missing them here would let
|
|
378
|
+
# actual occupancy exceed budget without make_room! noticing.
|
|
379
|
+
def occupied_bytes = resident_bytes + pending_bytes
|
|
380
|
+
|
|
381
|
+
def pending_bytes = @pending.each_key.sum { |id| @by_id.fetch(id).bytes }
|
|
382
|
+
|
|
383
|
+
# The fallback when something has not arrived at its demand point. One
|
|
384
|
+
# synchronous transfer at a time is slow (0.56 GB/s measured on Jetson),
|
|
385
|
+
# but it alone guarantees correctness — with zero effective prefetches the
|
|
386
|
+
# run gives the same result as 4a.
|
|
387
|
+
def fetch_sync(entry)
|
|
388
|
+
started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
389
|
+
entry.buffer.restore!
|
|
390
|
+
@waited += Process.clock_gettime(Process::CLOCK_MONOTONIC) - started
|
|
391
|
+
entry.has_file = true # restore! succeeding means the file exists
|
|
392
|
+
@fetches += 1
|
|
393
|
+
@bytes_in += entry.bytes
|
|
394
|
+
entry
|
|
395
|
+
end
|
|
396
|
+
|
|
397
|
+
# Issue only. Stats (fetches / bytes_in / waited) are counted in
|
|
398
|
+
# wait_transfer — nothing has finished at issue time, so counting here
|
|
399
|
+
# would break bytes_in's meaning of "bytes transferred".
|
|
400
|
+
def fetch_async(entry)
|
|
401
|
+
@pending[entry.node_id] = entry.buffer.restore_async!
|
|
402
|
+
entry.has_file = true
|
|
403
|
+
entry
|
|
404
|
+
end
|
|
405
|
+
|
|
406
|
+
def wait_transfer(entry, transfer)
|
|
407
|
+
started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
408
|
+
transfer.wait
|
|
409
|
+
@waited += Process.clock_gettime(Process::CLOCK_MONOTONIC) - started
|
|
410
|
+
@fetches += 1
|
|
411
|
+
@bytes_in += entry.bytes
|
|
412
|
+
entry
|
|
413
|
+
end
|
|
414
|
+
|
|
415
|
+
# If an offload file already exists, just drop the VRAM. Weights are
|
|
416
|
+
# read-only, so writing back would rewrite the same contents with
|
|
417
|
+
# synchronous I/O (the same reason Offload.pipeline's write_back: defaults
|
|
418
|
+
# to false). A missing file means the user bound a resident Buffer; only
|
|
419
|
+
# then is it written out, once.
|
|
420
|
+
def evict(entry)
|
|
421
|
+
if entry.has_file
|
|
422
|
+
entry.buffer.evict!
|
|
423
|
+
elsif @backing == :host
|
|
424
|
+
entry.buffer.offload_to_host!
|
|
425
|
+
entry.has_file = true
|
|
426
|
+
entry.spilled = true
|
|
427
|
+
@spills += 1
|
|
428
|
+
@spill_bytes += entry.bytes
|
|
429
|
+
else
|
|
430
|
+
path = spill_path(entry)
|
|
431
|
+
entry.buffer.offload!(path)
|
|
432
|
+
entry.has_file = true
|
|
433
|
+
entry.spilled = true
|
|
434
|
+
@spilled_paths << path
|
|
435
|
+
@spills += 1
|
|
436
|
+
@spill_bytes += entry.bytes
|
|
437
|
+
end
|
|
438
|
+
@evictions += 1
|
|
439
|
+
entry
|
|
440
|
+
end
|
|
441
|
+
|
|
442
|
+
def spill_path(entry)
|
|
443
|
+
require "fileutils"
|
|
444
|
+
FileUtils.mkdir_p(@spill_dir)
|
|
445
|
+
File.join(@spill_dir, format("const-%08d.rmgpu", entry.node_id))
|
|
446
|
+
end
|
|
447
|
+
end
|
|
448
|
+
end
|
|
449
|
+
end
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Tilework
|
|
4
|
+
module Graph
|
|
5
|
+
# Mutable state brought into the graph: buffers that **outlive a plan
|
|
6
|
+
# run**, such as KV caches, optimizer moments and gradient accumulators.
|
|
7
|
+
#
|
|
8
|
+
# It binds an existing Buffer as is (no reallocation). Only two things are
|
|
9
|
+
# special from the graph's point of view:
|
|
10
|
+
#
|
|
11
|
+
# - it is live-in and live-out of the plan, so it is never reused as an
|
|
12
|
+
# intermediate
|
|
13
|
+
# - a write advances its version, and a read depends on a version
|
|
14
|
+
class State
|
|
15
|
+
attr_reader :id, :name, :buffer, :dtype, :shape
|
|
16
|
+
attr_accessor :leaves, :version_count
|
|
17
|
+
|
|
18
|
+
def initialize(id, buffer, name)
|
|
19
|
+
@id = id
|
|
20
|
+
@name = name || :"state_#{id}"
|
|
21
|
+
@buffer = buffer
|
|
22
|
+
@dtype = buffer.dtype
|
|
23
|
+
@shape = buffer.shape
|
|
24
|
+
@version_count = 0
|
|
25
|
+
@leaves = []
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def next_version = @version_count += 1
|
|
29
|
+
def to_s = name.to_s
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
# A reference to **one version** of a state. `write_into` does not mutate
|
|
33
|
+
# in place; it returns a new StateRef (the next version). Reads depend on
|
|
34
|
+
# the version, so a real data edge connects a write and its readers and the
|
|
35
|
+
# scheduler cannot reorder them.
|
|
36
|
+
#
|
|
37
|
+
# Unlike SSA, a version points at the same memory rather than a new value,
|
|
38
|
+
# which is why WAR ordering is needed as well as RAW (added by Plan#compile).
|
|
39
|
+
class StateRef
|
|
40
|
+
attr_reader :graph, :state, :version, :node
|
|
41
|
+
|
|
42
|
+
def initialize(graph, state, version, node)
|
|
43
|
+
@graph = graph
|
|
44
|
+
@state = state
|
|
45
|
+
@version = version
|
|
46
|
+
@node = node
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
def dtype = state.dtype
|
|
50
|
+
def shape = state.shape
|
|
51
|
+
|
|
52
|
+
# Region constructors. They are relative to the state's shape, so they live here.
|
|
53
|
+
def rows(range) = Region.rows(state.shape, range)
|
|
54
|
+
def axis(axis, range) = Region.axis(state.shape, axis, range)
|
|
55
|
+
def whole = Region.whole(state.shape)
|
|
56
|
+
|
|
57
|
+
# Shorthand delegations (the same as g.write_into / g.read).
|
|
58
|
+
def write(value, at: nil) = graph.write_into(self, value: value, at: at)
|
|
59
|
+
def read(at = nil) = graph.read(self, at)
|
|
60
|
+
|
|
61
|
+
def inspect = "#<Tilework::Graph::StateRef #{state.name} v#{version}>"
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
end
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Tilework
|
|
4
|
+
module Graph
|
|
5
|
+
# A symbolic graph value.
|
|
6
|
+
#
|
|
7
|
+
# Arithmetic methods record nodes and return another tensor; they never execute a kernel.
|
|
8
|
+
#
|
|
9
|
+
# The operator surface matches Ops — the same expression runs immediately
|
|
10
|
+
# on a Buffer and records into the graph on a Tensor. Type and shape rules
|
|
11
|
+
# follow Ops too.
|
|
12
|
+
class Tensor
|
|
13
|
+
# @return [Builder] graph that owns this tensor
|
|
14
|
+
# @return [Node] underlying graph node
|
|
15
|
+
attr_reader :graph, :node
|
|
16
|
+
|
|
17
|
+
def initialize(graph, node)
|
|
18
|
+
@graph = graph
|
|
19
|
+
@node = node
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
# @return [Symbol] tensor storage dtype
|
|
23
|
+
def dtype = node.dtype
|
|
24
|
+
# @return [Array<Integer>] logical shape
|
|
25
|
+
def shape = node.shape
|
|
26
|
+
# @return [Integer] element count
|
|
27
|
+
def length = node.length
|
|
28
|
+
# @return [Boolean] whether this tensor is two-dimensional
|
|
29
|
+
def matrix? = shape.length == 2
|
|
30
|
+
# @return [Integer] first dimension
|
|
31
|
+
def rows = shape[0]
|
|
32
|
+
# @return [Integer] second dimension
|
|
33
|
+
def cols = shape[1]
|
|
34
|
+
def symbolic? = true
|
|
35
|
+
|
|
36
|
+
Ops::OPERATORS.each do |op|
|
|
37
|
+
define_method(op) { |other| graph.arithmetic(op, self, other) }
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
Ops::COMPARISONS.each do |op|
|
|
41
|
+
define_method(op) { |other| graph.compare(op, self, other) }
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
Ops::UNARY_MATH.each do |fn|
|
|
45
|
+
define_method(fn) { graph.unary_math(fn, self) }
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
def eq(other) = graph.compare(:==, self, other)
|
|
49
|
+
def ne(other) = graph.compare(:!=, self, other)
|
|
50
|
+
def abs = graph.unary_abs(self)
|
|
51
|
+
def matmul(other, out_dtype: nil) = graph.matmul(self, other, out_dtype: out_dtype)
|
|
52
|
+
|
|
53
|
+
# Reductions. Unlike eager, they return a **Tensor, not a host number**
|
|
54
|
+
# (dropping to the host mid-graph would end the plan). Without axis: the
|
|
55
|
+
# whole tensor folds to shape [1] and runs on Ops' device: true path.
|
|
56
|
+
def sum(axis: nil, keepdims: false) = graph.reduce(:sum, self, axis: axis, keepdims: keepdims)
|
|
57
|
+
def min(axis: nil, keepdims: false) = graph.reduce(:min, self, axis: axis, keepdims: keepdims)
|
|
58
|
+
def max(axis: nil, keepdims: false) = graph.reduce(:max, self, axis: axis, keepdims: keepdims)
|
|
59
|
+
def mean(axis: nil, keepdims: false) = graph.reduce(:mean, self, axis: axis, keepdims: keepdims)
|
|
60
|
+
def argmax(axis: nil, keepdims: false) = graph.arg_reduce(:max, self, axis: axis, keepdims: keepdims)
|
|
61
|
+
def argmin(axis: nil, keepdims: false) = graph.arg_reduce(:min, self, axis: axis, keepdims: keepdims)
|
|
62
|
+
|
|
63
|
+
# Broadcasts the tensor to a compatible shape.
|
|
64
|
+
#
|
|
65
|
+
# @param shape [Array<Integer>]
|
|
66
|
+
# @return [Tensor]
|
|
67
|
+
def broadcast_to(shape) = graph.broadcast(self, shape)
|
|
68
|
+
|
|
69
|
+
# Changes only the logical shape, with at most one inferred `-1` dimension.
|
|
70
|
+
#
|
|
71
|
+
# @param dims [Array<Integer>]
|
|
72
|
+
# @return [Tensor]
|
|
73
|
+
def reshape(*dims) = graph.reshape(self, *dims)
|
|
74
|
+
|
|
75
|
+
# Casts to a different storage dtype.
|
|
76
|
+
#
|
|
77
|
+
# @param dtype [:float32, :float16, :bfloat16, :int32]
|
|
78
|
+
# @return [Tensor]
|
|
79
|
+
def cast(dtype) = graph.cast(self, dtype)
|
|
80
|
+
|
|
81
|
+
# Permutes tensor axes.
|
|
82
|
+
#
|
|
83
|
+
# @param axes [Array<Integer>] permutation of every axis
|
|
84
|
+
# @return [Tensor]
|
|
85
|
+
def permute(*axes) = graph.permute(self, *axes)
|
|
86
|
+
|
|
87
|
+
# Slices the outermost dimension without a kernel launch.
|
|
88
|
+
#
|
|
89
|
+
# @param range [Integer, Range]
|
|
90
|
+
# @return [Tensor]
|
|
91
|
+
def slice(range) = graph.slice(self, range)
|
|
92
|
+
|
|
93
|
+
# Gathers entries from this tensor using int32 indices.
|
|
94
|
+
#
|
|
95
|
+
# @param indices [Tensor]
|
|
96
|
+
# @param axis [Integer]
|
|
97
|
+
# @return [Tensor]
|
|
98
|
+
def gather(indices, axis: 0) = graph.gather(self, indices, axis: axis)
|
|
99
|
+
|
|
100
|
+
# Makes scalar-on-the-left expressions like 2.0 * tensor work (as Ops::Scalar does).
|
|
101
|
+
def coerce(num) = [Scalar.new(self, num), self]
|
|
102
|
+
|
|
103
|
+
def inspect = "#<Tilework::Graph::Tensor #{shape.inspect} #{dtype} (#{node.label})>"
|
|
104
|
+
|
|
105
|
+
# The scalar-side proxy returned by coerce. Non-commutative ops record a
|
|
106
|
+
# node with swapped operands; comparisons flip the operator.
|
|
107
|
+
class Scalar
|
|
108
|
+
def initialize(tensor, value)
|
|
109
|
+
@tensor = tensor
|
|
110
|
+
@value = value
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
def +(other) = other.graph.arithmetic(:+, other, @value)
|
|
114
|
+
def *(other) = other.graph.arithmetic(:*, other, @value)
|
|
115
|
+
def -(other) = other.graph.arithmetic(:-, other, @value, reversed: true)
|
|
116
|
+
def /(other) = other.graph.arithmetic(:/, other, @value, reversed: true)
|
|
117
|
+
def >(other) = other.graph.compare(:<, other, @value)
|
|
118
|
+
def <(other) = other.graph.compare(:>, other, @value)
|
|
119
|
+
def >=(other) = other.graph.compare(:<=, other, @value)
|
|
120
|
+
def <=(other) = other.graph.compare(:>=, other, @value)
|
|
121
|
+
end
|
|
122
|
+
end
|
|
123
|
+
end
|
|
124
|
+
end
|