tilework 0.18.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE +21 -0
- data/lib/tilework/backend/cpu/dsl_codegen.rb +509 -0
- data/lib/tilework/backend/cpu/ops_codegen.rb +451 -0
- data/lib/tilework/backend/cpu.rb +156 -0
- data/lib/tilework/buffer.rb +753 -0
- data/lib/tilework/capabilities.rb +183 -0
- data/lib/tilework/compat/cuda.h +66 -0
- data/lib/tilework/compat/hip.h +35 -0
- data/lib/tilework/compat/metal.h +45 -0
- data/lib/tilework/conformance.rb +2296 -0
- data/lib/tilework/core_ext.rb +21 -0
- data/lib/tilework/dialect.rb +67 -0
- data/lib/tilework/dsl.rb +722 -0
- data/lib/tilework/dtype/bfloat.rb +31 -0
- data/lib/tilework/dtype/cast.rb +88 -0
- data/lib/tilework/dtype/half.rb +98 -0
- data/lib/tilework/graph/builder.rb +1073 -0
- data/lib/tilework/graph/fusion.rb +520 -0
- data/lib/tilework/graph/grad.rb +443 -0
- data/lib/tilework/graph/high_level.rb +266 -0
- data/lib/tilework/graph/kernels.rb +403 -0
- data/lib/tilework/graph/node.rb +27 -0
- data/lib/tilework/graph/op.rb +290 -0
- data/lib/tilework/graph/optim.rb +183 -0
- data/lib/tilework/graph/plan.rb +1086 -0
- data/lib/tilework/graph/pool.rb +178 -0
- data/lib/tilework/graph/region.rb +210 -0
- data/lib/tilework/graph/residency.rb +449 -0
- data/lib/tilework/graph/state.rb +64 -0
- data/lib/tilework/graph/tensor.rb +124 -0
- data/lib/tilework/graph.rb +213 -0
- data/lib/tilework/kernel.rb +163 -0
- data/lib/tilework/math/conv.rb +270 -0
- data/lib/tilework/math/fft.rb +172 -0
- data/lib/tilework/math/nn.rb +1103 -0
- data/lib/tilework/math/philox.rb +194 -0
- data/lib/tilework/math/random.rb +232 -0
- data/lib/tilework/math/tensor_core.rb +269 -0
- data/lib/tilework/ops.rb +682 -0
- data/lib/tilework/runtime/offload.rb +498 -0
- data/lib/tilework/runtime/submission.rb +163 -0
- data/lib/tilework/runtime/telemetry.rb +112 -0
- data/lib/tilework/scope.rb +33 -0
- data/lib/tilework/version.rb +5 -0
- data/lib/tilework.rb +345 -0
- metadata +92 -0
|
@@ -0,0 +1,498 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "fileutils"
|
|
4
|
+
|
|
5
|
+
module Tilework
|
|
6
|
+
# Explicit persistence and restoration of device buffers.
|
|
7
|
+
#
|
|
8
|
+
# The purpose is moving what does not fit in VRAM out to SSD, not backup, so
|
|
9
|
+
# offload! always releases VRAM.
|
|
10
|
+
#
|
|
11
|
+
# Computing on an offloaded Buffer raises instead of restoring implicitly:
|
|
12
|
+
# swap-like implicit restores would make performance unpredictable — the
|
|
13
|
+
# same thinking that made to_a an explicit transfer. Buffer#with_resident is
|
|
14
|
+
# the explicit sugar.
|
|
15
|
+
module Offload
|
|
16
|
+
MAGIC = "RMGPUOFF"
|
|
17
|
+
VERSION = 1
|
|
18
|
+
# Header and body both sit on 4096-byte boundaries to suit O_DIRECT. The
|
|
19
|
+
# Ruby fallback writes the same format, so swapping in the C ABI
|
|
20
|
+
# implementation later keeps files compatible.
|
|
21
|
+
HEADER_BYTES = 4096
|
|
22
|
+
ALIGN = 4096
|
|
23
|
+
|
|
24
|
+
DTYPE_CODES = { float32: 0, int32: 1, float16: 2, bfloat16: 3 }.freeze
|
|
25
|
+
CODE_DTYPES = DTYPE_CODES.invert.freeze
|
|
26
|
+
|
|
27
|
+
MAX_DIMS = 8
|
|
28
|
+
|
|
29
|
+
module_function
|
|
30
|
+
|
|
31
|
+
# Non-resident Buffer payloads. The public `Buffer#offload!(path)` API
|
|
32
|
+
# remains path-shaped, but Buffer itself owns a backing object so native
|
|
33
|
+
# host memory can use the identical residency transitions without being
|
|
34
|
+
# disguised as a filesystem path.
|
|
35
|
+
class Backing
|
|
36
|
+
def kind = :backing
|
|
37
|
+
|
|
38
|
+
def diagnostic = kind.to_s
|
|
39
|
+
|
|
40
|
+
def delete; end
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
class FileBacking < Backing
|
|
44
|
+
attr_reader :path, :checksum
|
|
45
|
+
|
|
46
|
+
def initialize(path, checksum: false)
|
|
47
|
+
@path = String(path)
|
|
48
|
+
@checksum = checksum
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
def kind = :file
|
|
52
|
+
|
|
53
|
+
def diagnostic = "file backing at #{@path.inspect}"
|
|
54
|
+
|
|
55
|
+
def write(buffer)
|
|
56
|
+
Offload.write_file(@path, buffer, checksum: @checksum)
|
|
57
|
+
self
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
def metadata
|
|
61
|
+
Offload.read_header(@path)
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
def validate!(buffer)
|
|
65
|
+
meta = metadata
|
|
66
|
+
unless meta[:dtype] == buffer.dtype && meta[:length] == buffer.length
|
|
67
|
+
raise Error, "offload file #{@path} does not match this buffer " \
|
|
68
|
+
"(#{meta[:dtype]} x#{meta[:length]} vs #{buffer.dtype} x#{buffer.length})"
|
|
69
|
+
end
|
|
70
|
+
meta
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
def restore_into(buffer, meta = validate!(buffer))
|
|
74
|
+
Offload.read_body_into(@path, buffer, meta)
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
# Returns a bridge transfer token or nil when the durable file requires
|
|
78
|
+
# the synchronous/checksummed path.
|
|
79
|
+
def begin_restore(handle, bytes, meta = validate_metadata)
|
|
80
|
+
return nil unless Offload.async? && meta[:checksum].nil?
|
|
81
|
+
|
|
82
|
+
Tilework.bridge.transfer_begin(handle, @path, bytes, 0, 1)
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
def delete
|
|
86
|
+
FileUtils.rm_f(@path)
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
private
|
|
90
|
+
|
|
91
|
+
def validate_metadata = metadata
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
# Volatile pageable host storage. Its opaque native handle is intentionally
|
|
95
|
+
# never represented as a path or exposed in diagnostics. Only backends
|
|
96
|
+
# implementing the optional host-backing bridge ABI can construct it.
|
|
97
|
+
class HostBacking < Backing
|
|
98
|
+
Ownership = Struct.new(:handle, :bridge, :freed)
|
|
99
|
+
|
|
100
|
+
attr_reader :dtype, :length, :shape
|
|
101
|
+
|
|
102
|
+
def self.available?
|
|
103
|
+
Tilework.bridge.respond_to?(:host_backing_available?) && Tilework.bridge.host_backing_available?
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
def initialize(dtype:, length:, shape:, bytesize:)
|
|
107
|
+
@dtype, @length, @shape = dtype, length, shape.dup
|
|
108
|
+
bridge = Tilework.bridge
|
|
109
|
+
@handle = bridge.host_backing_alloc(bytesize)
|
|
110
|
+
@ownership = Ownership.new(@handle, bridge, false)
|
|
111
|
+
ObjectSpace.define_finalizer(self, self.class.finalizer(@ownership))
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
def self.finalizer(ownership)
|
|
115
|
+
proc do
|
|
116
|
+
next if ownership.freed
|
|
117
|
+
|
|
118
|
+
ownership.bridge.host_backing_free(ownership.handle)
|
|
119
|
+
ownership.freed = true
|
|
120
|
+
rescue StandardError
|
|
121
|
+
nil # finalizers must never turn GC into an application exception
|
|
122
|
+
end
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
def kind = :host
|
|
126
|
+
|
|
127
|
+
def diagnostic = "pageable host backing"
|
|
128
|
+
|
|
129
|
+
def validate!(buffer)
|
|
130
|
+
return self if buffer.dtype == @dtype && buffer.length == @length && buffer.shape == @shape
|
|
131
|
+
|
|
132
|
+
raise Error, "host backing does not match this buffer"
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
def write(buffer)
|
|
136
|
+
validate!(buffer)
|
|
137
|
+
Tilework.bridge.host_backing_store(@handle, buffer.__handle__, buffer.bytesize, 0)
|
|
138
|
+
self
|
|
139
|
+
end
|
|
140
|
+
|
|
141
|
+
def restore_into(buffer, _meta = self)
|
|
142
|
+
validate!(buffer)
|
|
143
|
+
Tilework.bridge.host_backing_restore(@handle, buffer.__handle__, buffer.bytesize, 0)
|
|
144
|
+
buffer
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
def async_available?
|
|
148
|
+
Tilework.bridge.respond_to?(:host_backing_async_available?) && Tilework.bridge.host_backing_async_available?
|
|
149
|
+
end
|
|
150
|
+
|
|
151
|
+
def begin_restore(handle, bytes)
|
|
152
|
+
return nil unless async_available?
|
|
153
|
+
|
|
154
|
+
[:host, Tilework.bridge.host_backing_transfer_begin(@handle, handle, bytes, 0)]
|
|
155
|
+
end
|
|
156
|
+
|
|
157
|
+
def delete
|
|
158
|
+
return if @ownership.freed
|
|
159
|
+
|
|
160
|
+
@ownership.bridge.host_backing_free(@handle)
|
|
161
|
+
@ownership.freed = true
|
|
162
|
+
end
|
|
163
|
+
end
|
|
164
|
+
|
|
165
|
+
# @return [:staged, :posix] active file-transfer implementation
|
|
166
|
+
#
|
|
167
|
+
# :staged — the bridge's pinned staging (O_DIRECT where possible)
|
|
168
|
+
# :posix — the plain D2H -> Ruby String -> File.write path
|
|
169
|
+
#
|
|
170
|
+
# GPUDirect Storage (:gpudirect) is not implemented; on the test machine
|
|
171
|
+
# cuFile only reaches compatibility mode and gains nothing.
|
|
172
|
+
def transport
|
|
173
|
+
return :posix if ENV["GPU_OFFLOAD_FORCE_POSIX"] == "1"
|
|
174
|
+
return :posix unless Tilework.backend
|
|
175
|
+
return :posix unless Tilework.bridge.respond_to?(:offload_available?) && Tilework.bridge.offload_available?
|
|
176
|
+
|
|
177
|
+
:staged
|
|
178
|
+
end
|
|
179
|
+
|
|
180
|
+
# @return [Boolean] whether the backend supplies a native staged path
|
|
181
|
+
def native? = transport == :staged
|
|
182
|
+
|
|
183
|
+
# @return [Boolean] whether asynchronous restore is supported
|
|
184
|
+
#
|
|
185
|
+
# Whether the async transfer calls (transfer_begin/poll/end) exist. Without
|
|
186
|
+
# them it falls back to the synchronous version and prefetch can no longer
|
|
187
|
+
# hide transfers.
|
|
188
|
+
def async?
|
|
189
|
+
return false if ENV["GPU_OFFLOAD_FORCE_POSIX"] == "1"
|
|
190
|
+
return false unless Tilework.backend
|
|
191
|
+
|
|
192
|
+
Tilework.bridge.respond_to?(:transfer_available?) && Tilework.bridge.transfer_available?
|
|
193
|
+
end
|
|
194
|
+
|
|
195
|
+
# A pending asynchronous buffer restore.
|
|
196
|
+
class Transfer
|
|
197
|
+
def initialize(token, buffer, poll: :transfer_poll, finish: :transfer_end)
|
|
198
|
+
@token = token
|
|
199
|
+
@buffer = buffer
|
|
200
|
+
@poll = poll
|
|
201
|
+
@finish = finish
|
|
202
|
+
@finished = token.nil?
|
|
203
|
+
end
|
|
204
|
+
|
|
205
|
+
# Polls without waiting.
|
|
206
|
+
# @return [Boolean]
|
|
207
|
+
def done?
|
|
208
|
+
return true if @finished
|
|
209
|
+
|
|
210
|
+
Tilework.bridge.public_send(@poll, @token) != 0
|
|
211
|
+
end
|
|
212
|
+
|
|
213
|
+
# Waits for the transfer and makes its buffer resident.
|
|
214
|
+
# @return [Buffer]
|
|
215
|
+
def wait
|
|
216
|
+
return @buffer if @finished
|
|
217
|
+
|
|
218
|
+
Tilework.bridge.public_send(@finish, @token)
|
|
219
|
+
@finished = true
|
|
220
|
+
@buffer.__finish_restore__
|
|
221
|
+
end
|
|
222
|
+
end
|
|
223
|
+
|
|
224
|
+
# Builds the 4096-byte header. No endianness conversion — only x86_64 /
|
|
225
|
+
# aarch64 are targeted, so Buffer's internal representation is written as is.
|
|
226
|
+
def pack_header(dtype:, length:, shape:, checksum: nil)
|
|
227
|
+
code = DTYPE_CODES[dtype] or raise ArgumentError, "cannot offload dtype #{dtype.inspect}"
|
|
228
|
+
raise ArgumentError, "shape has too many dimensions: #{shape.length}" if shape.length > MAX_DIMS
|
|
229
|
+
|
|
230
|
+
dims = shape + [0] * (MAX_DIMS - shape.length)
|
|
231
|
+
flags = checksum ? 1 : 0
|
|
232
|
+
head = [MAGIC, VERSION, code, length, shape.length, *dims, flags, checksum || 0]
|
|
233
|
+
.pack("a8 L L Q L L#{MAX_DIMS} L Q")
|
|
234
|
+
head + ("\0" * (HEADER_BYTES - head.bytesize))
|
|
235
|
+
end
|
|
236
|
+
|
|
237
|
+
def unpack_header(bytes)
|
|
238
|
+
raise Error, "offload file is truncated (#{bytes.bytesize} bytes)" if bytes.bytesize < HEADER_BYTES
|
|
239
|
+
|
|
240
|
+
magic, version, code, length, ndim, *rest = bytes.unpack("a8 L L Q L L#{MAX_DIMS} L Q")
|
|
241
|
+
raise Error, "not a tilework offload file (bad magic)" unless magic == MAGIC
|
|
242
|
+
raise Error, "unsupported offload file version #{version}" unless version == VERSION
|
|
243
|
+
|
|
244
|
+
dtype = CODE_DTYPES[code] or raise Error, "unknown dtype code #{code} in offload file"
|
|
245
|
+
dims = rest.first(MAX_DIMS)
|
|
246
|
+
flags = rest[MAX_DIMS]
|
|
247
|
+
checksum = rest[MAX_DIMS + 1]
|
|
248
|
+
{ dtype: dtype, length: length, shape: dims.first(ndim),
|
|
249
|
+
checksum: flags.anybits?(1) ? checksum : nil }
|
|
250
|
+
end
|
|
251
|
+
|
|
252
|
+
# Buffer -> file. Uses the bridge's optional save ABI if implemented,
|
|
253
|
+
# otherwise the plain D2H -> Ruby String -> File.write path. The file
|
|
254
|
+
# contents are the same either way.
|
|
255
|
+
def write_file(path, buffer, checksum: false)
|
|
256
|
+
FileUtils.mkdir_p(File.dirname(path))
|
|
257
|
+
|
|
258
|
+
# A checksum requires looking at the body in Ruby, which cancels the
|
|
259
|
+
# native path's advantage (not routing D2H through a Ruby String), so
|
|
260
|
+
# use the posix path in that case.
|
|
261
|
+
if native? && !checksum
|
|
262
|
+
write_file_native(path, buffer)
|
|
263
|
+
else
|
|
264
|
+
write_file_posix(path, buffer, checksum: checksum)
|
|
265
|
+
end
|
|
266
|
+
path
|
|
267
|
+
end
|
|
268
|
+
|
|
269
|
+
def write_file_posix(path, buffer, checksum: false)
|
|
270
|
+
body = buffer.read_bytes
|
|
271
|
+
sum = checksum ? checksum_of(body) : nil
|
|
272
|
+
padding = (-body.bytesize) % ALIGN
|
|
273
|
+
|
|
274
|
+
File.open(path, "wb") do |f|
|
|
275
|
+
f.write(pack_header(dtype: buffer.dtype, length: buffer.length,
|
|
276
|
+
shape: buffer.shape, checksum: sum))
|
|
277
|
+
f.write(body)
|
|
278
|
+
f.write("\0" * padding) if padding.positive?
|
|
279
|
+
# A checkpoint lost to a power failure is useless, so make it durable.
|
|
280
|
+
# The native path fsyncs too, so both paths are equally durable.
|
|
281
|
+
f.fsync
|
|
282
|
+
end
|
|
283
|
+
end
|
|
284
|
+
|
|
285
|
+
# The header is written first with ordinary I/O and only the body goes
|
|
286
|
+
# through the bridge's O_DIRECT. The file is bit-identical to the posix path's.
|
|
287
|
+
def write_file_native(path, buffer)
|
|
288
|
+
File.open(path, "wb") do |f|
|
|
289
|
+
f.write(pack_header(dtype: buffer.dtype, length: buffer.length, shape: buffer.shape))
|
|
290
|
+
# Reserve the O_DIRECT target (pwrite could extend the file, but
|
|
291
|
+
# truncate explicitly so the size matches the posix path)
|
|
292
|
+
f.truncate(HEADER_BYTES + ((buffer.bytesize + ALIGN - 1) / ALIGN * ALIGN))
|
|
293
|
+
end
|
|
294
|
+
Tilework.bridge.buffer_save(buffer.__handle__, path, buffer.bytesize, 0)
|
|
295
|
+
end
|
|
296
|
+
|
|
297
|
+
# File -> new Buffer. dtype/shape are fully restored from the header, so
|
|
298
|
+
# callers need not remember the shape.
|
|
299
|
+
def read_file(path)
|
|
300
|
+
meta = read_header(path)
|
|
301
|
+
buffer = Buffer.new(meta[:dtype], meta[:length], shape: meta[:shape])
|
|
302
|
+
read_body_into(path, buffer, meta)
|
|
303
|
+
buffer
|
|
304
|
+
end
|
|
305
|
+
|
|
306
|
+
def read_header(path)
|
|
307
|
+
unpack_header(File.binread(path, HEADER_BYTES).to_s)
|
|
308
|
+
end
|
|
309
|
+
|
|
310
|
+
# Reads the body into an existing Buffer. Checksum verification needs the
|
|
311
|
+
# body in Ruby, so that case uses the posix path.
|
|
312
|
+
def read_body_into(path, buffer, meta)
|
|
313
|
+
if native? && meta[:checksum].nil?
|
|
314
|
+
Tilework.bridge.buffer_load(buffer.__handle__, path, buffer.bytesize, 0)
|
|
315
|
+
return buffer
|
|
316
|
+
end
|
|
317
|
+
|
|
318
|
+
body = File.open(path, "rb") do |f|
|
|
319
|
+
f.seek(HEADER_BYTES)
|
|
320
|
+
f.read(buffer.bytesize).to_s
|
|
321
|
+
end
|
|
322
|
+
raise Error, "offload file body is truncated (#{body.bytesize} of #{buffer.bytesize} bytes)" if body.bytesize < buffer.bytesize
|
|
323
|
+
|
|
324
|
+
if meta[:checksum] && checksum_of(body) != meta[:checksum]
|
|
325
|
+
raise Error, "offload file checksum mismatch for #{path} (corrupted?)"
|
|
326
|
+
end
|
|
327
|
+
buffer.write_bytes(body)
|
|
328
|
+
buffer
|
|
329
|
+
end
|
|
330
|
+
|
|
331
|
+
# The checksum is optional: off by default since it is wasted computation
|
|
332
|
+
# for training spills, on for checkpoints. xxh64 is not in the standard
|
|
333
|
+
# library, so FNV-1a folds to 64 bits without adding a dependency.
|
|
334
|
+
def checksum_of(bytes)
|
|
335
|
+
hash = 0xcbf29ce484222325
|
|
336
|
+
bytes.each_byte do |b|
|
|
337
|
+
hash = ((hash ^ b) * 0x100000001b3) & 0xFFFFFFFFFFFFFFFF
|
|
338
|
+
end
|
|
339
|
+
hash
|
|
340
|
+
end
|
|
341
|
+
|
|
342
|
+
def store(dir)
|
|
343
|
+
Store.new(dir)
|
|
344
|
+
end
|
|
345
|
+
|
|
346
|
+
# An alias of pipeline, for when the name should say the block runs once
|
|
347
|
+
# per element (in the family of each_slice / each_with_index). To load one
|
|
348
|
+
# or N buffers at once, use Tilework::Buffer.with_resident instead.
|
|
349
|
+
def each_resident(buffers, depth: 2, write_back: false, &block)
|
|
350
|
+
pipeline(buffers, depth: depth, write_back: write_back, &block)
|
|
351
|
+
end
|
|
352
|
+
|
|
353
|
+
# Makes an offloaded series resident in turn while prefetching ahead.
|
|
354
|
+
#
|
|
355
|
+
# Tilework::Offload.pipeline(layers, depth: 2) do |buf|
|
|
356
|
+
# ... # buf is resident; the next depth-1 are transferring meanwhile
|
|
357
|
+
# end
|
|
358
|
+
#
|
|
359
|
+
# depth: 2 means one element of prefetch. VRAM use scales with one
|
|
360
|
+
# element's size x depth, so choose depth from the free VRAM. A Buffer
|
|
361
|
+
# leaving the block goes back to its offload target, freeing VRAM.
|
|
362
|
+
#
|
|
363
|
+
# Whether transfers are hidden depends on the ratio of compute time to load
|
|
364
|
+
# time: if per-element compute is shorter than its transfer, raising depth
|
|
365
|
+
# adds no bandwidth and the waits remain.
|
|
366
|
+
#
|
|
367
|
+
# write_back: defaults to false. When streaming read-only weights, the
|
|
368
|
+
# offload file does not change, so writing back only rewrites the same
|
|
369
|
+
# contents, and as synchronous I/O it cannot overlap compute and defeats
|
|
370
|
+
# hiding (measured: raising depth stayed at 1.0x). **Set it true only when
|
|
371
|
+
# changes made in the block must be kept.**
|
|
372
|
+
def pipeline(items, depth: 2, write_back: false)
|
|
373
|
+
raise ArgumentError, "depth must be >= 1" unless depth.is_a?(Integer) && depth >= 1
|
|
374
|
+
|
|
375
|
+
# An element may be a single Buffer or an array of Buffers. A real
|
|
376
|
+
# model's "layer" is a group of tensors rather than one weight matrix (in a
|
|
377
|
+
# transformer, heads x {wq,wk,wv,wo} + w1 + w2), and layer-wise streaming
|
|
378
|
+
# needs groups to flow as a unit.
|
|
379
|
+
#
|
|
380
|
+
# Do not use Array() — it would call Buffer#to_a (a read from the device).
|
|
381
|
+
groups = items.map { |item| item.is_a?(Array) ? item : [item] }
|
|
382
|
+
|
|
383
|
+
pending = {}
|
|
384
|
+
results = []
|
|
385
|
+
prefetch = lambda do |i|
|
|
386
|
+
next if i >= groups.length || pending.key?(i)
|
|
387
|
+
|
|
388
|
+
pending[i] = groups[i].reject { |b| b.freed? || b.resident? }.map(&:restore_async!)
|
|
389
|
+
end
|
|
390
|
+
|
|
391
|
+
# At most depth elements occupy VRAM at once (one being computed plus
|
|
392
|
+
# depth-1 prefetching), so depth: 1 means no prefetch and depth: 2 one
|
|
393
|
+
# element ahead. With groups, VRAM is "group size x depth".
|
|
394
|
+
depth.times { |i| prefetch.call(i) }
|
|
395
|
+
|
|
396
|
+
begin
|
|
397
|
+
items.each_with_index do |item, i|
|
|
398
|
+
Offload.wait_all(pending.delete(i) || [])
|
|
399
|
+
groups[i].each { |b| b.restore! unless b.resident? } # backstop for environments without async
|
|
400
|
+
begin
|
|
401
|
+
results << yield(item)
|
|
402
|
+
ensure
|
|
403
|
+
groups[i].each do |b|
|
|
404
|
+
next if b.freed?
|
|
405
|
+
|
|
406
|
+
write_back ? b.offload_back! : b.evict!
|
|
407
|
+
end
|
|
408
|
+
# Issue the next one only after one element's VRAM is free; issuing
|
|
409
|
+
# earlier would exceed the depth limit.
|
|
410
|
+
prefetch.call(i + depth)
|
|
411
|
+
end
|
|
412
|
+
end
|
|
413
|
+
ensure
|
|
414
|
+
# If the block raises (or breaks), prefetched but unconsumed elements
|
|
415
|
+
# would keep holding VRAM. restore_async! allocates device memory before
|
|
416
|
+
# the wait, so they hold it even while resident? is false. Always
|
|
417
|
+
# collect them here.
|
|
418
|
+
pending.each do |i, transfers|
|
|
419
|
+
begin
|
|
420
|
+
Offload.wait_all(transfers)
|
|
421
|
+
rescue StandardError
|
|
422
|
+
nil # a secondary failure during cleanup must not hide the original exception
|
|
423
|
+
end
|
|
424
|
+
groups[i].each { |b| b.evict! unless b.freed? }
|
|
425
|
+
end
|
|
426
|
+
pending.clear
|
|
427
|
+
end
|
|
428
|
+
results
|
|
429
|
+
end
|
|
430
|
+
|
|
431
|
+
# Collects every launched transfer before raising the first exception;
|
|
432
|
+
# leaving early would strand worker threads, so wait on all of them even on failure.
|
|
433
|
+
def wait_all(transfers)
|
|
434
|
+
first_error = nil
|
|
435
|
+
transfers.each do |t|
|
|
436
|
+
t.wait
|
|
437
|
+
rescue StandardError => e
|
|
438
|
+
first_error ||= e
|
|
439
|
+
end
|
|
440
|
+
raise first_error if first_error
|
|
441
|
+
end
|
|
442
|
+
|
|
443
|
+
# A container for when file names should not be the caller's concern. A
|
|
444
|
+
# Store is explicit persistence: it survives the end of a Scope and stays
|
|
445
|
+
# until clear is called.
|
|
446
|
+
class Store
|
|
447
|
+
attr_reader :dir
|
|
448
|
+
|
|
449
|
+
def initialize(dir)
|
|
450
|
+
@dir = dir
|
|
451
|
+
@seq = 0
|
|
452
|
+
@handles = {}
|
|
453
|
+
FileUtils.mkdir_p(dir)
|
|
454
|
+
end
|
|
455
|
+
|
|
456
|
+
# Stores a buffer and returns its generated handle. The buffer's VRAM is
|
|
457
|
+
# released; get it back with the returned handle.
|
|
458
|
+
# @param buffer [Buffer]
|
|
459
|
+
# @param checksum [Boolean]
|
|
460
|
+
# @return [String]
|
|
461
|
+
def put(buffer, checksum: false)
|
|
462
|
+
handle = (@seq += 1)
|
|
463
|
+
path = File.join(@dir, format("%08d.rmgpu", handle))
|
|
464
|
+
buffer.offload!(path, checksum: checksum)
|
|
465
|
+
@handles[handle] = path
|
|
466
|
+
handle
|
|
467
|
+
end
|
|
468
|
+
|
|
469
|
+
# Restores a stored buffer by handle.
|
|
470
|
+
# @param handle [String]
|
|
471
|
+
# @return [Buffer]
|
|
472
|
+
def get(handle)
|
|
473
|
+
path = @handles[handle] or raise Error, "unknown offload handle #{handle.inspect}"
|
|
474
|
+
Offload.read_file(path)
|
|
475
|
+
end
|
|
476
|
+
|
|
477
|
+
# Deletes a stored buffer.
|
|
478
|
+
# @param handle [String]
|
|
479
|
+
# @return [nil]
|
|
480
|
+
def delete(handle)
|
|
481
|
+
path = @handles.delete(handle) or return false
|
|
482
|
+
|
|
483
|
+
FileUtils.rm_f(path)
|
|
484
|
+
true
|
|
485
|
+
end
|
|
486
|
+
|
|
487
|
+
# Deletes every buffer stored by this instance.
|
|
488
|
+
# @return [nil]
|
|
489
|
+
def clear
|
|
490
|
+
@handles.each_value { |path| FileUtils.rm_f(path) }
|
|
491
|
+
@handles.clear
|
|
492
|
+
nil
|
|
493
|
+
end
|
|
494
|
+
|
|
495
|
+
def size = @handles.size
|
|
496
|
+
end
|
|
497
|
+
end
|
|
498
|
+
end
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Tilework
|
|
4
|
+
# An explicit asynchronous submit boundary for the eager path (#71).
|
|
5
|
+
#
|
|
6
|
+
# `Kernel#launch` always uses `wait: true`, so the host waits on every
|
|
7
|
+
# launch. On hardware that wait is an order of magnitude larger than a small
|
|
8
|
+
# kernel (a no-op kernel on HIP: 162 µs/op synchronous vs 2.4 µs/op queued on
|
|
9
|
+
# the same stream with one final synchronization). Chains of small launches,
|
|
10
|
+
# like elementwise chains and decode loops, are dominated by synchronization
|
|
11
|
+
# rather than arithmetic.
|
|
12
|
+
#
|
|
13
|
+
# job = Tilework.submit { (a * 2 + b).relu }
|
|
14
|
+
# out = job.wait
|
|
15
|
+
#
|
|
16
|
+
# Only inside the block are launches enqueue-only on the same queue;
|
|
17
|
+
# `Job#wait` synchronizes once and returns the block's value. **Semantics
|
|
18
|
+
# outside the block do not change at all** — the point is an explicit
|
|
19
|
+
# boundary, not implicitly making a synchronous API asynchronous.
|
|
20
|
+
#
|
|
21
|
+
# It rides on the same `Ops` execution-context contract as the graph's
|
|
22
|
+
# `Plan#run` (see the `Ops::CONTEXT_KEY` comment), so going asynchronous
|
|
23
|
+
# needs no new machinery. What differs is ownership, which is most of this
|
|
24
|
+
# file:
|
|
25
|
+
#
|
|
26
|
+
# - **The Job holds every output allocated inside.** `Buffer`'s finalizer
|
|
27
|
+
# calls `buffer_free`, so an intermediate created during submit that lost
|
|
28
|
+
# its last reference could vanish under a kernel reading it.
|
|
29
|
+
# - **`free` is deferred until completion**, rather than rejected, so eager
|
|
30
|
+
# code written with `ensure buf.free` still works wrapped in submit
|
|
31
|
+
# (releasing memory a running kernel uses is undefined behavior on CUDA/HIP).
|
|
32
|
+
# - **No pool.** The graph's `Pool` is safe asynchronously because it knows
|
|
33
|
+
# the execution order and can say "the next writer of a slot is queued
|
|
34
|
+
# after its last reader". Eager has no such information.
|
|
35
|
+
module Submission
|
|
36
|
+
# The context plugged into Ops while running. Satisfies the same shape as
|
|
37
|
+
# Graph::Context (`#allocate` / `#async?` / `#synchronous`).
|
|
38
|
+
class Context
|
|
39
|
+
def initialize(job)
|
|
40
|
+
@job = job
|
|
41
|
+
@async = true
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
attr_reader :job
|
|
45
|
+
|
|
46
|
+
def async? = @async
|
|
47
|
+
|
|
48
|
+
def allocate(dtype, length, shape)
|
|
49
|
+
Buffer.new(dtype, length, shape: shape).tap { |buffer| @job.retain(buffer) }
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
# `Buffer#free` inside the block is deferred to the boundary (the Ops.defer_free contract).
|
|
53
|
+
def defer_free(buffer) = @job.defer_free(buffer)
|
|
54
|
+
|
|
55
|
+
# Delegated libraries run synchronously. Sharing the name with
|
|
56
|
+
# Graph::Context is deliberate, so callers that wrap either context the
|
|
57
|
+
# same way (`Plan#dispatch_node`) need no branch.
|
|
58
|
+
def synchronous
|
|
59
|
+
previous = @async
|
|
60
|
+
@async = false
|
|
61
|
+
yield
|
|
62
|
+
ensure
|
|
63
|
+
@async = previous
|
|
64
|
+
end
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
# The return value of `Tilework.submit`.
|
|
68
|
+
class Job
|
|
69
|
+
def initialize
|
|
70
|
+
@value = nil
|
|
71
|
+
@done = false
|
|
72
|
+
@retained = []
|
|
73
|
+
@deferred_free = []
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
# The block's value, waiting first if `wait` has not been called.
|
|
77
|
+
#
|
|
78
|
+
# @return [Object]
|
|
79
|
+
def wait
|
|
80
|
+
return @value if @done
|
|
81
|
+
|
|
82
|
+
Tilework.synchronize
|
|
83
|
+
complete!
|
|
84
|
+
@value
|
|
85
|
+
end
|
|
86
|
+
alias value wait
|
|
87
|
+
|
|
88
|
+
def done? = @done
|
|
89
|
+
|
|
90
|
+
# An output allocated during submit. While the Job holds it the finalizer
|
|
91
|
+
# cannot run, so memory never vanishes under a running kernel.
|
|
92
|
+
def retain(buffer)
|
|
93
|
+
@retained << buffer
|
|
94
|
+
buffer
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
# Defers a `Buffer#free` made during submit until completion.
|
|
98
|
+
def defer_free(buffer)
|
|
99
|
+
@deferred_free << buffer
|
|
100
|
+
buffer
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
# How many frees are pending (for tests and debugging).
|
|
104
|
+
def deferred_free_count = @deferred_free.length
|
|
105
|
+
|
|
106
|
+
# @api private
|
|
107
|
+
def fulfill(value)
|
|
108
|
+
@value = value
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
# @api private -- an exception unwind also drains the queue before getting here.
|
|
112
|
+
def complete!
|
|
113
|
+
return if @done
|
|
114
|
+
|
|
115
|
+
@done = true
|
|
116
|
+
@retained.clear
|
|
117
|
+
pending = @deferred_free
|
|
118
|
+
@deferred_free = []
|
|
119
|
+
pending.each(&:free)
|
|
120
|
+
nil
|
|
121
|
+
end
|
|
122
|
+
end
|
|
123
|
+
|
|
124
|
+
class << self
|
|
125
|
+
# Whether this thread is inside a submit.
|
|
126
|
+
def context
|
|
127
|
+
context = Ops.context
|
|
128
|
+
context if context.is_a?(Context)
|
|
129
|
+
end
|
|
130
|
+
|
|
131
|
+
def active? = !context.nil?
|
|
132
|
+
|
|
133
|
+
# @api private -- the public API is Tilework.submit
|
|
134
|
+
def run(&block)
|
|
135
|
+
# A nested submit opens no new boundary. Synchronizing inside would also
|
|
136
|
+
# wait for the outer submit's queued work, defeating the explicit
|
|
137
|
+
# boundary. The inner Job is returned completed (its value is already final).
|
|
138
|
+
return nested(&block) if active?
|
|
139
|
+
|
|
140
|
+
job = Job.new
|
|
141
|
+
begin
|
|
142
|
+
job.fulfill(Ops.with_context(Context.new(job), &block))
|
|
143
|
+
rescue StandardError, ScriptError
|
|
144
|
+
# Queued kernels may still be running. Drain before freeing and
|
|
145
|
+
# re-raising, so the unwind does not release memory under them.
|
|
146
|
+
Tilework.synchronize
|
|
147
|
+
job.complete!
|
|
148
|
+
raise
|
|
149
|
+
end
|
|
150
|
+
job
|
|
151
|
+
end
|
|
152
|
+
|
|
153
|
+
private
|
|
154
|
+
|
|
155
|
+
def nested
|
|
156
|
+
job = Job.new
|
|
157
|
+
job.fulfill(yield)
|
|
158
|
+
job.complete!
|
|
159
|
+
job
|
|
160
|
+
end
|
|
161
|
+
end
|
|
162
|
+
end
|
|
163
|
+
end
|