tilework 0.18.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE +21 -0
  3. data/lib/tilework/backend/cpu/dsl_codegen.rb +509 -0
  4. data/lib/tilework/backend/cpu/ops_codegen.rb +451 -0
  5. data/lib/tilework/backend/cpu.rb +156 -0
  6. data/lib/tilework/buffer.rb +753 -0
  7. data/lib/tilework/capabilities.rb +183 -0
  8. data/lib/tilework/compat/cuda.h +66 -0
  9. data/lib/tilework/compat/hip.h +35 -0
  10. data/lib/tilework/compat/metal.h +45 -0
  11. data/lib/tilework/conformance.rb +2296 -0
  12. data/lib/tilework/core_ext.rb +21 -0
  13. data/lib/tilework/dialect.rb +67 -0
  14. data/lib/tilework/dsl.rb +722 -0
  15. data/lib/tilework/dtype/bfloat.rb +31 -0
  16. data/lib/tilework/dtype/cast.rb +88 -0
  17. data/lib/tilework/dtype/half.rb +98 -0
  18. data/lib/tilework/graph/builder.rb +1073 -0
  19. data/lib/tilework/graph/fusion.rb +520 -0
  20. data/lib/tilework/graph/grad.rb +443 -0
  21. data/lib/tilework/graph/high_level.rb +266 -0
  22. data/lib/tilework/graph/kernels.rb +403 -0
  23. data/lib/tilework/graph/node.rb +27 -0
  24. data/lib/tilework/graph/op.rb +290 -0
  25. data/lib/tilework/graph/optim.rb +183 -0
  26. data/lib/tilework/graph/plan.rb +1086 -0
  27. data/lib/tilework/graph/pool.rb +178 -0
  28. data/lib/tilework/graph/region.rb +210 -0
  29. data/lib/tilework/graph/residency.rb +449 -0
  30. data/lib/tilework/graph/state.rb +64 -0
  31. data/lib/tilework/graph/tensor.rb +124 -0
  32. data/lib/tilework/graph.rb +213 -0
  33. data/lib/tilework/kernel.rb +163 -0
  34. data/lib/tilework/math/conv.rb +270 -0
  35. data/lib/tilework/math/fft.rb +172 -0
  36. data/lib/tilework/math/nn.rb +1103 -0
  37. data/lib/tilework/math/philox.rb +194 -0
  38. data/lib/tilework/math/random.rb +232 -0
  39. data/lib/tilework/math/tensor_core.rb +269 -0
  40. data/lib/tilework/ops.rb +682 -0
  41. data/lib/tilework/runtime/offload.rb +498 -0
  42. data/lib/tilework/runtime/submission.rb +163 -0
  43. data/lib/tilework/runtime/telemetry.rb +112 -0
  44. data/lib/tilework/scope.rb +33 -0
  45. data/lib/tilework/version.rb +5 -0
  46. data/lib/tilework.rb +345 -0
  47. metadata +92 -0
@@ -0,0 +1,753 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Tilework
4
+ # A typed, device-resident linear memory allocation.
5
+ #
6
+ # Buffers can have a logical shape without changing their row-major storage layout.
7
+ # Use {Buffer.from} to transfer Ruby Arrays and {#to_a} to read values back.
8
+ class Buffer
9
+ # Element size and host<->bytes conversion per dtype.
10
+ #
11
+ # Not a single pack directive because of float16 — Ruby's Array#pack has
12
+ # no half-precision directive (lib/tilework/dtype/half.rb). Holding
13
+ # encode/decode as procs leaves each dtype free to convert however it needs.
14
+ DTYPES = {
15
+ float32: { size: 4, encode: ->(a) { a.pack("f*") }, decode: ->(s) { s.unpack("f*") } },
16
+ int32: { size: 4, encode: ->(a) { a.pack("l*") }, decode: ->(s) { s.unpack("l*") } },
17
+ # float16 can be allocated only on backends listing it in
18
+ # capabilities[:dtypes]. Accumulation and outputs stay FP32, so it is
19
+ # used only for storage, matmul and offload.
20
+ float16: { size: 2, encode: ->(a) { Half.encode(a) }, decode: ->(s) { Half.decode(s) } },
21
+ bfloat16: { size: 2, encode: ->(a) { BFloat.encode(a) }, decode: ->(s) { BFloat.decode(s) } },
22
+ }.freeze
23
+
24
+ # float16 conversions above this many elements go to the GPU. The pure-Ruby
25
+ # Tilework::Half does per-element bit manipulation and takes 1.3 s to encode
26
+ # a million elements (0.5 ms on the GPU path). Rounding is the same, so the
27
+ # results agree bit for bit. Small arrays are not worth a temporary buffer,
28
+ # so they stay in Ruby.
29
+ GPU_CONVERSION_THRESHOLD = 4096
30
+
31
+ attr_reader :dtype, :length, :shape
32
+
33
+ # Allocates a zero-initialized buffer.
34
+ #
35
+ # @param dtype [:float32, :float16, :bfloat16, :int32] storage dtype supported by the active backend
36
+ # @param length [Integer] positive element count
37
+ # @param shape [Array<Integer>, nil] logical shape; defaults to `[length]`
38
+ # @return [Buffer]
39
+ # @raise [ArgumentError] if the dtype, length, or shape is invalid
40
+ # @raise [DeviceError] if the backend cannot allocate the dtype
41
+ def initialize(dtype, length, shape: nil)
42
+ info = DTYPES[dtype] or
43
+ raise ArgumentError, "unknown dtype: #{dtype.inspect} (available: #{DTYPES.keys.join(', ')})"
44
+ raise ArgumentError, "length must be a positive Integer" unless length.is_a?(Integer) && length.positive?
45
+
46
+ check_dtype_supported!(dtype)
47
+ @dtype = dtype
48
+ @length = length
49
+ @shape = validate_shape(shape || [length])
50
+ @encode = info[:encode]
51
+ @decode = info[:decode]
52
+ @elem_size = info[:size]
53
+ @handle = Tilework.bridge.buffer_alloc(bytesize)
54
+ @freed = false
55
+ Scope.register(self)
56
+ Telemetry::Resources.track_buffer(@handle, bytesize)
57
+ # A safety net for a forgotten explicit free (on some backends @handle is
58
+ # not TypedData). A proc capturing self would never be collected, so a
59
+ # class method builds it.
60
+ ObjectSpace.define_finalizer(self, self.class.finalizer(@handle, Tilework.bridge))
61
+ end
62
+
63
+ # --- Views (the same memory seen with another shape / range) ---
64
+ #
65
+ # A view **does not allocate**. It is another Buffer pointing at the
66
+ # parent's device memory, and writes reach the parent. Splitting attention
67
+ # heads, KV caches, slicing batches and channel operations would all be
68
+ # copies without it.
69
+ #
70
+ # x = Tilework.random([4, 256])
71
+ # x.rows(0...2) # the first two rows (no copy)
72
+ # x.reshape(2, 512) # the same memory with another shape
73
+ # x.view(256, 256) # 256 elements starting at element 256
74
+ #
75
+ # **Freeing the parent invalidates its views**; do not let a view outlive
76
+ # its parent. A view's own #free releases no memory; it only makes that
77
+ # view unusable.
78
+
79
+ # Returns a zero-copy element-range view.
80
+ #
81
+ # @param offset [Integer] zero-based element offset
82
+ # @param length [Integer] number of elements
83
+ # @param shape [Array<Integer>, nil] logical shape for the view
84
+ # @return [Buffer] a view that aliases this buffer's storage
85
+ # @raise [ArgumentError] if the requested range is outside this buffer
86
+ def view(offset, length, shape: nil)
87
+ require_resident!("view")
88
+ raise ArgumentError, "offset must be a non-negative Integer" unless offset.is_a?(Integer) && !offset.negative?
89
+ raise ArgumentError, "length must be a positive Integer" unless length.is_a?(Integer) && length.positive?
90
+
91
+ unless offset + length <= @length
92
+ raise ArgumentError,
93
+ "view #{offset}...#{offset + length} is out of bounds for a buffer of #{@length} elements"
94
+ end
95
+
96
+ build_view(offset, length, shape || [length])
97
+ end
98
+
99
+ # Returns a zero-copy view with a different logical shape.
100
+ #
101
+ # @param dims [Array<Integer>] dimensions with at most one inferred `-1`
102
+ # @return [Buffer]
103
+ # @raise [ArgumentError] unless the dimensions contain exactly this buffer's element count
104
+ def reshape(*dims)
105
+ dims = dims.first if dims.length == 1 && dims.first.is_a?(Array)
106
+ build_view(0, @length, resolve_shape(dims))
107
+ end
108
+
109
+ # Returns a zero-copy slice along the outermost dimension.
110
+ #
111
+ # @param range [Integer, Range] row or outer-dimension range
112
+ # @return [Buffer]
113
+ # @raise [Error] unless this buffer has at least two dimensions
114
+ #
115
+ # A row range for 2D, a batch slice for NCHW.
116
+ #
117
+ # x.slice(1..2) # [4, 256] -> [2, 256]
118
+ # images.slice(0) # [8, 3, 64, 64] -> [1, 3, 64, 64]
119
+ #
120
+ # (Named differently because `#rows` already returns the row count.)
121
+ def slice(range)
122
+ raise Error, "slice requires a shaped buffer (shape: #{@shape.inspect})" if @shape.length < 2
123
+
124
+ first, count = normalize_range(range, @shape[0])
125
+ stride = @shape.drop(1).inject(1, :*)
126
+ view(first * stride, count * stride, shape: [count, *@shape.drop(1)])
127
+ end
128
+
129
+ # Whether this buffer is a view; if so #base holds the parent.
130
+ def view? = !@base.nil?
131
+
132
+ attr_reader :base
133
+
134
+ # --- Internal (views) ---
135
+
136
+ def build_view(offset, length, shape)
137
+ v = self.class.allocate
138
+ v.send(:init_view, self, offset, length, shape)
139
+ v
140
+ end
141
+ private :build_view
142
+
143
+ # Bypasses initialize (which would allocate).
144
+ def init_view(parent, offset, length, shape)
145
+ @dtype = parent.dtype
146
+ @length = length
147
+ @shape = validate_shape(shape)
148
+ info = DTYPES[@dtype]
149
+ @encode = info[:encode]
150
+ @decode = info[:decode]
151
+ @elem_size = info[:size]
152
+ # In a chain of views the parent's handle already carries its offset, so
153
+ # only this view's share is added here (the C side does the addition).
154
+ @handle = Tilework.bridge.buffer_view(parent.__handle__, offset * @elem_size)
155
+ @base = parent.view? ? parent.base : parent # always points at the owning buffer
156
+ @view_offset = offset
157
+ @freed = false
158
+ # Not registered with Telemetry (nothing was allocated, so it would be
159
+ # double-counted), and no finalizer (nothing to free).
160
+ Scope.register(self)
161
+ end
162
+ private :init_view
163
+
164
+ def normalize_range(range, limit)
165
+ case range
166
+ when Integer
167
+ idx = range.negative? ? limit + range : range
168
+ raise ArgumentError, "row #{range} is out of bounds (#{limit} rows)" unless (0...limit).cover?(idx)
169
+
170
+ [idx, 1]
171
+ when Range
172
+ first = range.begin || 0
173
+ first += limit if first.negative?
174
+ last = range.end || limit
175
+ last += limit if last.negative?
176
+ last -= 1 if range.exclude_end? && range.end
177
+ count = last - first + 1
178
+ unless first >= 0 && count.positive? && first + count <= limit
179
+ raise ArgumentError, "row range #{range} is out of bounds (#{limit} rows)"
180
+ end
181
+
182
+ [first, count]
183
+ else
184
+ raise ArgumentError, "expected an Integer or a Range, got #{range.class}"
185
+ end
186
+ end
187
+ private :normalize_range
188
+
189
+ def resolve_shape(dims)
190
+ unless dims.all? { |d| d.is_a?(Integer) }
191
+ raise ArgumentError, "shape dimensions must be Integers, got #{dims.inspect}"
192
+ end
193
+
194
+ wild = dims.count(-1)
195
+ raise ArgumentError, "at most one dimension may be -1 (got #{dims.inspect})" if wild > 1
196
+
197
+ if wild == 1
198
+ known = dims.reject { |d| d == -1 }.inject(1, :*)
199
+ unless known.positive? && (@length % known).zero?
200
+ raise ArgumentError, "cannot infer -1 in #{dims.inspect} for #{@length} elements"
201
+ end
202
+
203
+ dims = dims.map { |d| d == -1 ? @length / known : d }
204
+ end
205
+
206
+ unless dims.inject(1, :*) == @length
207
+ raise ArgumentError, "shape #{dims.inspect} has #{dims.inject(1, :*)} elements, but the buffer has #{@length}"
208
+ end
209
+
210
+ dims
211
+ end
212
+ private :resolve_shape
213
+
214
+ # Offloading swaps the handle, which would break a view's link to its parent.
215
+ def reject_view!(action)
216
+ return unless view?
217
+
218
+ raise Error, "cannot #{action} a view; call it on the base buffer instead"
219
+ end
220
+ private :reject_view!
221
+
222
+ def require_resident!(action)
223
+ return if resident?
224
+
225
+ raise Error, "cannot #{action} a buffer that is not resident (offloaded or freed)"
226
+ end
227
+ private :require_resident!
228
+
229
+ # bridge.buffer_free is safe to call twice (idempotent) on every backend,
230
+ # so a GC call after an explicit #free is harmless.
231
+ def self.finalizer(handle, bridge)
232
+ proc do
233
+ bridge.buffer_free(handle)
234
+ Telemetry::Resources.untrack_buffer(handle)
235
+ end
236
+ end
237
+
238
+ # Transfers a one- or two-dimensional Ruby Array into a buffer.
239
+ #
240
+ # @param array [Array<Numeric>, Array<Array<Numeric>>] values to transfer
241
+ # @param dtype [:float32, :float16, :int32, nil] storage dtype; inferred when omitted
242
+ # @return [Buffer]
243
+ def self.from(array, dtype: nil)
244
+ raise ArgumentError, "array must not be empty" if array.empty?
245
+
246
+ if array.first.is_a?(Array)
247
+ unless array.all? { |e| e.is_a?(Array) }
248
+ raise ArgumentError, "array must not mix nested Arrays with scalar elements"
249
+ end
250
+
251
+ return from_matrix(array, dtype: dtype)
252
+ end
253
+
254
+ dtype ||= array.all?(Integer) ? :int32 : :float32
255
+ buffer = new(dtype, array.length)
256
+ buffer.write(array)
257
+ buffer
258
+ end
259
+
260
+ # Transfers a rectangular nested Array into a row-major two-dimensional buffer.
261
+ #
262
+ # @param rows [Array<Array<Numeric>>]
263
+ # @param dtype [:float32, :float16, :int32, nil]
264
+ # @return [Buffer]
265
+ def self.from_matrix(rows, dtype: nil)
266
+ raise ArgumentError, "rows must not be empty" if rows.empty?
267
+ unless rows.all? { |r| r.is_a?(Array) && r.length == rows.first.length }
268
+ raise ArgumentError, "all rows must be non-empty Arrays of the same length"
269
+ end
270
+
271
+ flat = rows.flatten(1)
272
+ dtype ||= flat.all?(Integer) ? :int32 : :float32
273
+ buffer = new(dtype, flat.length, shape: [rows.length, rows.first.length])
274
+ buffer.write(flat)
275
+ buffer
276
+ end
277
+
278
+ # @return [Boolean] whether this buffer has exactly two dimensions
279
+ def matrix?
280
+ @shape.length == 2
281
+ end
282
+
283
+ # @return [Integer] row count for a two-dimensional buffer
284
+ # @raise [Error] unless this buffer is two-dimensional
285
+ def rows
286
+ raise Error, "buffer is not 2D (shape: #{@shape.inspect})" unless matrix?
287
+
288
+ @shape[0]
289
+ end
290
+
291
+ # @return [Integer] column count for a two-dimensional buffer
292
+ # @raise [Error] unless this buffer is two-dimensional
293
+ def cols
294
+ raise Error, "buffer is not 2D (shape: #{@shape.inspect})" unless matrix?
295
+
296
+ @shape[1]
297
+ end
298
+
299
+ # Reads a one- or two-dimensional buffer into a Ruby Array.
300
+ #
301
+ # @return [Array<Numeric>, Array<Array<Numeric>>]
302
+ def to_nested_a
303
+ return to_a unless matrix?
304
+
305
+ to_a.each_slice(cols).to_a
306
+ end
307
+
308
+ # @return [Integer] allocation size in bytes
309
+ def bytesize
310
+ @length * @elem_size
311
+ end
312
+
313
+ # Replaces every element with values from a Ruby Array.
314
+ #
315
+ # This synchronizes the active backend before writing.
316
+ #
317
+ # @param array [Array<Numeric>] exactly {#length} values
318
+ # @return [Buffer] this buffer
319
+ def write(array)
320
+ check_freed!
321
+ check_resident!
322
+ unless array.is_a?(Array) && array.length == @length
323
+ raise ArgumentError, "expected an Array of #{@length} elements"
324
+ end
325
+ if gpu_conversion?
326
+ # Send as float32, then convert on the GPU
327
+ staged = Buffer.new(:float32, @length)
328
+ begin
329
+ staged.write(array)
330
+ Cast.convert_into(staged, self)
331
+ ensure
332
+ staged.free
333
+ end
334
+ else
335
+ # Wait for pending async kernels. On unified-memory backends
336
+ # buffer_write is a raw memcpy from the host, so writing while an
337
+ # in-flight kernel reads is a data race.
338
+ Tilework.synchronize
339
+ Telemetry.instrument(:h2d, bytes: bytesize) { Tilework.bridge.buffer_write(@handle, @encode.call(array), 0) }
340
+ end
341
+ self
342
+ end
343
+
344
+ # Reads every element into a Ruby Array.
345
+ #
346
+ # This synchronizes the active backend before reading.
347
+ #
348
+ # @return [Array<Numeric>]
349
+ def to_a
350
+ check_freed!
351
+ check_resident!
352
+ Tilework.synchronize
353
+ if gpu_conversion?
354
+ staged = cast(:float32)
355
+ begin
356
+ return staged.to_a
357
+ ensure
358
+ staged.free
359
+ end
360
+ end
361
+ @decode.call(Telemetry.instrument(:d2h, bytes: bytesize) { Tilework.bridge.buffer_read(@handle, bytesize, 0) })
362
+ end
363
+
364
+ # Converts this buffer to a different storage dtype.
365
+ #
366
+ # @param target [:float32, :float16, :bfloat16, :int32]
367
+ # @return [Buffer] a converted buffer, or this buffer when the dtype is unchanged
368
+ #
369
+ # The GPU also rounds to nearest even, so the result matches Tilework::Half
370
+ # bit for bit.
371
+ def cast(target)
372
+ check_freed!
373
+ check_resident!
374
+ return self if target == @dtype
375
+
376
+ Tilework::Cast.convert(self, target)
377
+ end
378
+
379
+ # Reads raw bytes without packing or unpacking values.
380
+ #
381
+ # @param length [Integer, nil] byte count; defaults to the remaining allocation
382
+ # @param offset [Integer] byte offset
383
+ # @return [String]
384
+ #
385
+ # Unlike `to_a` there is no pack/unpack, so **this works directly for
386
+ # reading and writing weight files**. "Header + raw tensors" formats like
387
+ # safetensors can be implemented in the application with this alone (the
388
+ # library need not know the format).
389
+ #
390
+ # w = Tilework::Buffer.new(:float32, 1024)
391
+ # File.open(path) { |f| f.seek(data_offset); w.write_bytes(f.read(w.bytesize)) }
392
+ #
393
+ # offset and length are **bytes, not elements**, for streaming a large file
394
+ # in chunks.
395
+ #
396
+ # The bytes are the backend's native representation (float32 is
397
+ # little-endian IEEE-754 single, float16 is binary16). x86 and ARM are both
398
+ # little-endian, so there is no mismatch with file formats to worry about.
399
+ def read_bytes(length = nil, offset: 0)
400
+ check_freed!
401
+ check_resident!
402
+ length ||= bytesize - offset
403
+ check_byte_range!(offset, length)
404
+ Tilework.synchronize
405
+ Telemetry.instrument(:d2h, bytes: length) { Tilework.bridge.buffer_read(@handle, length, offset) }
406
+ end
407
+
408
+ # Writes raw bytes without converting values.
409
+ #
410
+ # @param bytes [String] byte sequence
411
+ # @param offset [Integer] byte offset
412
+ # @return [Buffer] this buffer
413
+ def write_bytes(bytes, offset: 0)
414
+ check_freed!
415
+ raise ArgumentError, "expected a String of bytes, got #{bytes.class}" unless bytes.is_a?(String)
416
+
417
+ check_byte_range!(offset, bytes.bytesize)
418
+ # Same as #write: host writes race in-flight kernels.
419
+ Tilework.synchronize
420
+ Telemetry.instrument(:h2d, bytes: bytes.bytesize) { Tilework.bridge.buffer_write(@handle, bytes, offset) }
421
+ self
422
+ end
423
+
424
+ # Bytes per element, for read_bytes / write_bytes offset arithmetic.
425
+ attr_reader :elem_size
426
+
427
+ def check_byte_range!(offset, length)
428
+ unless offset.is_a?(Integer) && !offset.negative? && length.is_a?(Integer) && !length.negative?
429
+ raise ArgumentError, "offset and length must be non-negative Integers"
430
+ end
431
+ return if offset + length <= bytesize
432
+
433
+ raise ArgumentError,
434
+ "byte range #{offset}...#{offset + length} is out of bounds for #{bytesize} bytes"
435
+ end
436
+ private :check_byte_range!
437
+
438
+ # --- VRAM <-> SSD offload ---
439
+
440
+ # Saves this buffer to disk and releases its device allocation.
441
+ #
442
+ # @param path [String] destination file
443
+ # @param checksum [Boolean] include an integrity checksum
444
+ # @return [Buffer] this non-resident buffer
445
+ def offload!(path, checksum: false)
446
+ reject_view!("offload!")
447
+ check_freed!
448
+ return self unless resident?
449
+
450
+ backing = Offload::FileBacking.new(path, checksum: checksum).write(self)
451
+ Tilework.bridge.buffer_free(@handle)
452
+ Telemetry::Resources.untrack_buffer(@handle)
453
+ @handle = nil
454
+ @backing = backing
455
+ @last_backing = backing
456
+ self
457
+ end
458
+
459
+ # Moves this buffer into an explicitly allocated volatile CPU-RAM backing.
460
+ # Unlike #offload!, no filesystem path is interpreted or created.
461
+ def offload_to_host!
462
+ reject_view!("offload_to_host!")
463
+ check_freed!
464
+ return self unless resident?
465
+ unless Offload::HostBacking.available?
466
+ raise Error, "pageable host backing is not available on the #{Tilework.backend.inspect} backend"
467
+ end
468
+
469
+ backing = Offload::HostBacking.new(dtype: @dtype, length: @length, shape: @shape, bytesize: bytesize)
470
+ begin
471
+ backing.write(self)
472
+ rescue StandardError
473
+ backing.delete
474
+ raise
475
+ end
476
+ Tilework.bridge.buffer_free(@handle)
477
+ Telemetry::Resources.untrack_buffer(@handle)
478
+ @handle = nil
479
+ @backing = backing
480
+ @last_backing = backing
481
+ self
482
+ end
483
+
484
+ # Restores an offloaded buffer synchronously.
485
+ #
486
+ # @return [Buffer] this resident buffer
487
+ def restore!
488
+ reject_view!("restore!")
489
+ check_freed!
490
+ return self if resident?
491
+ reject_pending_restore!
492
+
493
+ backing = backing!
494
+ meta = backing.validate!(self)
495
+
496
+ @handle = Tilework.bridge.buffer_alloc(bytesize)
497
+ Telemetry::Resources.track_buffer(@handle, bytesize)
498
+ ObjectSpace.define_finalizer(self, self.class.finalizer(@handle, Tilework.bridge))
499
+ @backing = nil
500
+ backing.restore_into(self, meta)
501
+ self
502
+ end
503
+
504
+ # Starts restoring an offloaded buffer asynchronously.
505
+ #
506
+ # @return [Offload::Transfer] call `#wait` before using the buffer
507
+ #
508
+ # The contents are complete only after Transfer#wait. Until then resident?
509
+ # stays false, so using it without waiting raises NotResidentError.
510
+ def restore_async!(async: true)
511
+ reject_view!("restore_async!")
512
+ check_freed!
513
+ return Offload::Transfer.new(nil, self) if resident?
514
+ reject_pending_restore!
515
+ return Offload::Transfer.new(nil, restore!) unless async
516
+
517
+ backing = backing!
518
+ backing.validate!(self)
519
+
520
+ handle = Tilework.bridge.buffer_alloc(bytesize)
521
+ token = backing.begin_restore(handle, bytesize)
522
+ # Backings that cannot load asynchronously (such as a checksummed
523
+ # FileBacking) fall back to a synchronous restore. Reallocating without
524
+ # first releasing the handle already allocated would leak, so free it here.
525
+ if token.nil?
526
+ Tilework.bridge.buffer_free(handle)
527
+ return Offload::Transfer.new(nil, restore!)
528
+ end
529
+ kind, token = token if token.is_a?(Array)
530
+ @pending_handle = handle
531
+ @pending_backing = backing
532
+ Offload::Transfer.new(token, self,
533
+ poll: kind == :host ? :host_backing_transfer_poll : :transfer_poll,
534
+ finish: kind == :host ? :host_backing_transfer_end : :transfer_end)
535
+ end
536
+
537
+ # Called from Transfer#wait; this is where it becomes resident.
538
+ def __finish_restore__
539
+ return self unless @pending_handle
540
+
541
+ @handle = @pending_handle
542
+ Telemetry::Resources.track_buffer(@handle, bytesize)
543
+ ObjectSpace.define_finalizer(self, self.class.finalizer(@handle, Tilework.bridge))
544
+ @pending_handle = nil
545
+ @backing = nil
546
+ @pending_backing = nil
547
+ self
548
+ end
549
+
550
+ # Called after pipeline leaves the block; writes back to the latest offload target.
551
+ def offload_back!
552
+ reject_view!("offload_back!")
553
+ return self if @freed || !resident?
554
+
555
+ backing = backing_for_reoffload
556
+ backing.write(self)
557
+ Tilework.bridge.buffer_free(@handle)
558
+ Telemetry::Resources.untrack_buffer(@handle)
559
+ @handle = nil
560
+ @backing = backing
561
+ self
562
+ end
563
+
564
+ # Releases device memory without rewriting the existing offload file.
565
+ #
566
+ # @return [Buffer] this non-resident buffer
567
+ # @note Changes since restoration are discarded.
568
+ #
569
+ # Assumes the offload file exists and is unchanged — when streaming
570
+ # read-only weights, writing back only rewrites the same contents, and as
571
+ # synchronous I/O it cannot overlap compute and defeats hiding.
572
+ #
573
+ # **Changes made in the block are lost.** To keep them use offload_back!
574
+ # (write_back: true for pipeline).
575
+ def evict!
576
+ reject_view!("evict!")
577
+ return self if @freed || !resident?
578
+
579
+ backing = backing_for_reoffload
580
+ Tilework.bridge.buffer_free(@handle)
581
+ Telemetry::Resources.untrack_buffer(@handle)
582
+ @handle = nil
583
+ @backing = backing
584
+ self
585
+ end
586
+
587
+ # @return [Boolean] whether this buffer can be used by device operations
588
+ def resident?
589
+ !@freed && !@handle.nil?
590
+ end
591
+
592
+ # Temporarily restores offloaded buffers for a block.
593
+ #
594
+ # @param buffers [Array<Buffer>]
595
+ # @param write_back [Boolean] re-offload modified buffers when the block exits
596
+ # @yieldparam buffers [Buffer]
597
+ # @return [Object] the block result
598
+ #
599
+ # Tilework::Buffer.with_resident(w1, w2, w3) { |a, b, c| a.matmul(b).matmul(c) }
600
+ #
601
+ # Restores are **issued in parallel and waited on once**. Nesting
602
+ # with_resident one buffer at a time serializes the transfers and cannot
603
+ # fill NVMe's queue depth (measured: 1.48x at 4, 1.69x at 8).
604
+ #
605
+ # write_back: defaults to true so changes made in the block are not lost.
606
+ # When the data is known to be read-only, false skips rewriting the same
607
+ # bytes entirely.
608
+ #
609
+ # Buffers that were already resident are left alone (neither loaded nor
610
+ # offloaded).
611
+ def self.with_resident(*buffers, write_back: true)
612
+ targets = buffers.reject { |b| b.freed? || b.resident? }
613
+ Offload.wait_all(targets.map(&:restore_async!))
614
+
615
+ yield(*buffers)
616
+ ensure
617
+ targets&.each do |b|
618
+ next if b.freed?
619
+
620
+ write_back ? b.offload_back! : b.evict!
621
+ end
622
+ end
623
+
624
+ # Temporarily restores this buffer for a block.
625
+ #
626
+ # @param write_back [Boolean] re-offload this buffer when the block exits
627
+ # @yieldparam buffer [Buffer]
628
+ # @return [Object] the block result
629
+ #
630
+ # restore! on entry, offload on exit: the explicit sugar in place of
631
+ # restoring offloaded buffers implicitly.
632
+ def with_resident(write_back: true, &block)
633
+ self.class.with_resident(self, write_back: write_back, &block)
634
+ end
635
+
636
+ # Releases this buffer's device allocation and any owned offload file.
637
+ #
638
+ # Inside a {Tilework.submit} block the release is **deferred** to the end of
639
+ # the block: launched kernels may still be reading this memory, and handing
640
+ # it back before they finish is undefined behavior. The buffer therefore
641
+ # stays usable — and `#freed?` stays false — until the job completes.
642
+ #
643
+ # @return [nil]
644
+ #
645
+ # An offloaded buffer's file is deleted too (so a Scope ending releases VRAM
646
+ # and deletes offload files alike). Persistence through Offload::Store is
647
+ # untouched, since the Store owns those paths.
648
+ def free
649
+ return if @freed
650
+ return if Ops.defer_free(self)
651
+
652
+ reject_pending_restore!
653
+
654
+ if @handle
655
+ Tilework.bridge.buffer_free(@handle) # a view releases no real memory (decided on the C side)
656
+ Telemetry::Resources.untrack_buffer(@handle) unless view?
657
+ elsif @backing
658
+ @backing.delete
659
+ end
660
+ # A HostBacking is volatile and the Buffer is its only owner. After
661
+ # restore! @backing is nil but @last_backing remains as the re-offload
662
+ # target, so it must be reclaimed here. A FileBacking keeps the existing
663
+ # Store/path ownership.
664
+ if @last_backing.is_a?(Offload::HostBacking)
665
+ @last_backing.delete
666
+ @last_backing = nil
667
+ end
668
+ @handle = nil
669
+ @freed = true
670
+ nil
671
+ end
672
+
673
+ # Graph residency uses this after restoring a plan-owned volatile backing.
674
+ # Durable FileBacking remains owned by its path/store and is intentionally
675
+ # untouched.
676
+ def discard_host_backing!
677
+ return self unless @last_backing.is_a?(Offload::HostBacking)
678
+
679
+ @last_backing.delete
680
+ @last_backing = nil
681
+ self
682
+ end
683
+
684
+ def freed?
685
+ @freed
686
+ end
687
+
688
+ # The native handle passed to the bridge layer; not public API.
689
+ def __handle__
690
+ check_freed!
691
+ check_resident!
692
+ @handle
693
+ end
694
+
695
+ private
696
+
697
+ # Large float16 conversions are orders of magnitude faster on the GPU (see the constant's comment).
698
+ def gpu_conversion?
699
+ %i[float16 bfloat16].include?(@dtype) && @length >= GPU_CONVERSION_THRESHOLD &&
700
+ Cast.supported?(:float32, @dtype)
701
+ end
702
+
703
+ def check_dtype_supported!(dtype)
704
+ supported = Tilework.capabilities[:dtypes]
705
+ return if supported.include?(dtype)
706
+
707
+ raise DeviceError,
708
+ "dtype #{dtype.inspect} is not supported on the #{Tilework.backend.inspect} backend " \
709
+ "(supported: #{supported.map(&:inspect).join(' / ')})"
710
+ end
711
+
712
+ # Where with_resident's exit writes back. restore! sets @backing to nil, so
713
+ # the latest backing is remembered separately.
714
+ def backing_for_reoffload
715
+ @last_backing or
716
+ raise Error, "this buffer has never been offloaded, so there is nowhere to put it back; " \
717
+ "call #offload!(path) first (Tilework::Offload.pipeline expects an already-offloaded series)"
718
+ end
719
+
720
+ def backing!
721
+ @backing or raise Error, "buffer has no backing to restore"
722
+ end
723
+
724
+ # The pending GPU allocation and HostBacking are owned by Transfer until
725
+ # #wait joins it. Starting another restore or freeing them beforehand
726
+ # would race the same backing store.
727
+ def reject_pending_restore!
728
+ return unless @pending_handle
729
+
730
+ raise Error, "buffer restore is still in progress; call Transfer#wait first"
731
+ end
732
+
733
+ def check_resident!
734
+ return if resident?
735
+
736
+ raise NotResidentError,
737
+ "buffer is offloaded to #{backing_for_reoffload.diagnostic}; call #restore! " \
738
+ "(or wrap the use in #with_resident) before operating on it"
739
+ end
740
+
741
+ def validate_shape(shape)
742
+ unless shape.is_a?(Array) && shape.all? { |d| d.is_a?(Integer) && d.positive? } && shape.reduce(:*) == @length
743
+ raise ArgumentError, "shape #{shape.inspect} does not match length #{@length}"
744
+ end
745
+
746
+ shape
747
+ end
748
+
749
+ def check_freed!
750
+ raise Error, "buffer has been freed" if @freed
751
+ end
752
+ end
753
+ end