tilework 0.18.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE +21 -0
- data/lib/tilework/backend/cpu/dsl_codegen.rb +509 -0
- data/lib/tilework/backend/cpu/ops_codegen.rb +451 -0
- data/lib/tilework/backend/cpu.rb +156 -0
- data/lib/tilework/buffer.rb +753 -0
- data/lib/tilework/capabilities.rb +183 -0
- data/lib/tilework/compat/cuda.h +66 -0
- data/lib/tilework/compat/hip.h +35 -0
- data/lib/tilework/compat/metal.h +45 -0
- data/lib/tilework/conformance.rb +2296 -0
- data/lib/tilework/core_ext.rb +21 -0
- data/lib/tilework/dialect.rb +67 -0
- data/lib/tilework/dsl.rb +722 -0
- data/lib/tilework/dtype/bfloat.rb +31 -0
- data/lib/tilework/dtype/cast.rb +88 -0
- data/lib/tilework/dtype/half.rb +98 -0
- data/lib/tilework/graph/builder.rb +1073 -0
- data/lib/tilework/graph/fusion.rb +520 -0
- data/lib/tilework/graph/grad.rb +443 -0
- data/lib/tilework/graph/high_level.rb +266 -0
- data/lib/tilework/graph/kernels.rb +403 -0
- data/lib/tilework/graph/node.rb +27 -0
- data/lib/tilework/graph/op.rb +290 -0
- data/lib/tilework/graph/optim.rb +183 -0
- data/lib/tilework/graph/plan.rb +1086 -0
- data/lib/tilework/graph/pool.rb +178 -0
- data/lib/tilework/graph/region.rb +210 -0
- data/lib/tilework/graph/residency.rb +449 -0
- data/lib/tilework/graph/state.rb +64 -0
- data/lib/tilework/graph/tensor.rb +124 -0
- data/lib/tilework/graph.rb +213 -0
- data/lib/tilework/kernel.rb +163 -0
- data/lib/tilework/math/conv.rb +270 -0
- data/lib/tilework/math/fft.rb +172 -0
- data/lib/tilework/math/nn.rb +1103 -0
- data/lib/tilework/math/philox.rb +194 -0
- data/lib/tilework/math/random.rb +232 -0
- data/lib/tilework/math/tensor_core.rb +269 -0
- data/lib/tilework/ops.rb +682 -0
- data/lib/tilework/runtime/offload.rb +498 -0
- data/lib/tilework/runtime/submission.rb +163 -0
- data/lib/tilework/runtime/telemetry.rb +112 -0
- data/lib/tilework/scope.rb +33 -0
- data/lib/tilework/version.rb +5 -0
- data/lib/tilework.rb +345 -0
- metadata +92 -0
|
@@ -0,0 +1,753 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Tilework
|
|
4
|
+
# A typed, device-resident linear memory allocation.
|
|
5
|
+
#
|
|
6
|
+
# Buffers can have a logical shape without changing their row-major storage layout.
|
|
7
|
+
# Use {Buffer.from} to transfer Ruby Arrays and {#to_a} to read values back.
|
|
8
|
+
class Buffer
|
|
9
|
+
# Element size and host<->bytes conversion per dtype.
|
|
10
|
+
#
|
|
11
|
+
# Not a single pack directive because of float16 — Ruby's Array#pack has
|
|
12
|
+
# no half-precision directive (lib/tilework/dtype/half.rb). Holding
|
|
13
|
+
# encode/decode as procs leaves each dtype free to convert however it needs.
|
|
14
|
+
DTYPES = {
|
|
15
|
+
float32: { size: 4, encode: ->(a) { a.pack("f*") }, decode: ->(s) { s.unpack("f*") } },
|
|
16
|
+
int32: { size: 4, encode: ->(a) { a.pack("l*") }, decode: ->(s) { s.unpack("l*") } },
|
|
17
|
+
# float16 can be allocated only on backends listing it in
|
|
18
|
+
# capabilities[:dtypes]. Accumulation and outputs stay FP32, so it is
|
|
19
|
+
# used only for storage, matmul and offload.
|
|
20
|
+
float16: { size: 2, encode: ->(a) { Half.encode(a) }, decode: ->(s) { Half.decode(s) } },
|
|
21
|
+
bfloat16: { size: 2, encode: ->(a) { BFloat.encode(a) }, decode: ->(s) { BFloat.decode(s) } },
|
|
22
|
+
}.freeze
|
|
23
|
+
|
|
24
|
+
# float16 conversions above this many elements go to the GPU. The pure-Ruby
|
|
25
|
+
# Tilework::Half does per-element bit manipulation and takes 1.3 s to encode
|
|
26
|
+
# a million elements (0.5 ms on the GPU path). Rounding is the same, so the
|
|
27
|
+
# results agree bit for bit. Small arrays are not worth a temporary buffer,
|
|
28
|
+
# so they stay in Ruby.
|
|
29
|
+
GPU_CONVERSION_THRESHOLD = 4096
|
|
30
|
+
|
|
31
|
+
attr_reader :dtype, :length, :shape
|
|
32
|
+
|
|
33
|
+
# Allocates a zero-initialized buffer.
|
|
34
|
+
#
|
|
35
|
+
# @param dtype [:float32, :float16, :bfloat16, :int32] storage dtype supported by the active backend
|
|
36
|
+
# @param length [Integer] positive element count
|
|
37
|
+
# @param shape [Array<Integer>, nil] logical shape; defaults to `[length]`
|
|
38
|
+
# @return [Buffer]
|
|
39
|
+
# @raise [ArgumentError] if the dtype, length, or shape is invalid
|
|
40
|
+
# @raise [DeviceError] if the backend cannot allocate the dtype
|
|
41
|
+
def initialize(dtype, length, shape: nil)
|
|
42
|
+
info = DTYPES[dtype] or
|
|
43
|
+
raise ArgumentError, "unknown dtype: #{dtype.inspect} (available: #{DTYPES.keys.join(', ')})"
|
|
44
|
+
raise ArgumentError, "length must be a positive Integer" unless length.is_a?(Integer) && length.positive?
|
|
45
|
+
|
|
46
|
+
check_dtype_supported!(dtype)
|
|
47
|
+
@dtype = dtype
|
|
48
|
+
@length = length
|
|
49
|
+
@shape = validate_shape(shape || [length])
|
|
50
|
+
@encode = info[:encode]
|
|
51
|
+
@decode = info[:decode]
|
|
52
|
+
@elem_size = info[:size]
|
|
53
|
+
@handle = Tilework.bridge.buffer_alloc(bytesize)
|
|
54
|
+
@freed = false
|
|
55
|
+
Scope.register(self)
|
|
56
|
+
Telemetry::Resources.track_buffer(@handle, bytesize)
|
|
57
|
+
# A safety net for a forgotten explicit free (on some backends @handle is
|
|
58
|
+
# not TypedData). A proc capturing self would never be collected, so a
|
|
59
|
+
# class method builds it.
|
|
60
|
+
ObjectSpace.define_finalizer(self, self.class.finalizer(@handle, Tilework.bridge))
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
# --- Views (the same memory seen with another shape / range) ---
|
|
64
|
+
#
|
|
65
|
+
# A view **does not allocate**. It is another Buffer pointing at the
|
|
66
|
+
# parent's device memory, and writes reach the parent. Splitting attention
|
|
67
|
+
# heads, KV caches, slicing batches and channel operations would all be
|
|
68
|
+
# copies without it.
|
|
69
|
+
#
|
|
70
|
+
# x = Tilework.random([4, 256])
|
|
71
|
+
# x.rows(0...2) # the first two rows (no copy)
|
|
72
|
+
# x.reshape(2, 512) # the same memory with another shape
|
|
73
|
+
# x.view(256, 256) # 256 elements starting at element 256
|
|
74
|
+
#
|
|
75
|
+
# **Freeing the parent invalidates its views**; do not let a view outlive
|
|
76
|
+
# its parent. A view's own #free releases no memory; it only makes that
|
|
77
|
+
# view unusable.
|
|
78
|
+
|
|
79
|
+
# Returns a zero-copy element-range view.
|
|
80
|
+
#
|
|
81
|
+
# @param offset [Integer] zero-based element offset
|
|
82
|
+
# @param length [Integer] number of elements
|
|
83
|
+
# @param shape [Array<Integer>, nil] logical shape for the view
|
|
84
|
+
# @return [Buffer] a view that aliases this buffer's storage
|
|
85
|
+
# @raise [ArgumentError] if the requested range is outside this buffer
|
|
86
|
+
def view(offset, length, shape: nil)
|
|
87
|
+
require_resident!("view")
|
|
88
|
+
raise ArgumentError, "offset must be a non-negative Integer" unless offset.is_a?(Integer) && !offset.negative?
|
|
89
|
+
raise ArgumentError, "length must be a positive Integer" unless length.is_a?(Integer) && length.positive?
|
|
90
|
+
|
|
91
|
+
unless offset + length <= @length
|
|
92
|
+
raise ArgumentError,
|
|
93
|
+
"view #{offset}...#{offset + length} is out of bounds for a buffer of #{@length} elements"
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
build_view(offset, length, shape || [length])
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
# Returns a zero-copy view with a different logical shape.
|
|
100
|
+
#
|
|
101
|
+
# @param dims [Array<Integer>] dimensions with at most one inferred `-1`
|
|
102
|
+
# @return [Buffer]
|
|
103
|
+
# @raise [ArgumentError] unless the dimensions contain exactly this buffer's element count
|
|
104
|
+
def reshape(*dims)
|
|
105
|
+
dims = dims.first if dims.length == 1 && dims.first.is_a?(Array)
|
|
106
|
+
build_view(0, @length, resolve_shape(dims))
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
# Returns a zero-copy slice along the outermost dimension.
|
|
110
|
+
#
|
|
111
|
+
# @param range [Integer, Range] row or outer-dimension range
|
|
112
|
+
# @return [Buffer]
|
|
113
|
+
# @raise [Error] unless this buffer has at least two dimensions
|
|
114
|
+
#
|
|
115
|
+
# A row range for 2D, a batch slice for NCHW.
|
|
116
|
+
#
|
|
117
|
+
# x.slice(1..2) # [4, 256] -> [2, 256]
|
|
118
|
+
# images.slice(0) # [8, 3, 64, 64] -> [1, 3, 64, 64]
|
|
119
|
+
#
|
|
120
|
+
# (Named differently because `#rows` already returns the row count.)
|
|
121
|
+
def slice(range)
|
|
122
|
+
raise Error, "slice requires a shaped buffer (shape: #{@shape.inspect})" if @shape.length < 2
|
|
123
|
+
|
|
124
|
+
first, count = normalize_range(range, @shape[0])
|
|
125
|
+
stride = @shape.drop(1).inject(1, :*)
|
|
126
|
+
view(first * stride, count * stride, shape: [count, *@shape.drop(1)])
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
# Whether this buffer is a view; if so #base holds the parent.
|
|
130
|
+
def view? = !@base.nil?
|
|
131
|
+
|
|
132
|
+
attr_reader :base
|
|
133
|
+
|
|
134
|
+
# --- Internal (views) ---
|
|
135
|
+
|
|
136
|
+
def build_view(offset, length, shape)
|
|
137
|
+
v = self.class.allocate
|
|
138
|
+
v.send(:init_view, self, offset, length, shape)
|
|
139
|
+
v
|
|
140
|
+
end
|
|
141
|
+
private :build_view
|
|
142
|
+
|
|
143
|
+
# Bypasses initialize (which would allocate).
|
|
144
|
+
def init_view(parent, offset, length, shape)
|
|
145
|
+
@dtype = parent.dtype
|
|
146
|
+
@length = length
|
|
147
|
+
@shape = validate_shape(shape)
|
|
148
|
+
info = DTYPES[@dtype]
|
|
149
|
+
@encode = info[:encode]
|
|
150
|
+
@decode = info[:decode]
|
|
151
|
+
@elem_size = info[:size]
|
|
152
|
+
# In a chain of views the parent's handle already carries its offset, so
|
|
153
|
+
# only this view's share is added here (the C side does the addition).
|
|
154
|
+
@handle = Tilework.bridge.buffer_view(parent.__handle__, offset * @elem_size)
|
|
155
|
+
@base = parent.view? ? parent.base : parent # always points at the owning buffer
|
|
156
|
+
@view_offset = offset
|
|
157
|
+
@freed = false
|
|
158
|
+
# Not registered with Telemetry (nothing was allocated, so it would be
|
|
159
|
+
# double-counted), and no finalizer (nothing to free).
|
|
160
|
+
Scope.register(self)
|
|
161
|
+
end
|
|
162
|
+
private :init_view
|
|
163
|
+
|
|
164
|
+
def normalize_range(range, limit)
|
|
165
|
+
case range
|
|
166
|
+
when Integer
|
|
167
|
+
idx = range.negative? ? limit + range : range
|
|
168
|
+
raise ArgumentError, "row #{range} is out of bounds (#{limit} rows)" unless (0...limit).cover?(idx)
|
|
169
|
+
|
|
170
|
+
[idx, 1]
|
|
171
|
+
when Range
|
|
172
|
+
first = range.begin || 0
|
|
173
|
+
first += limit if first.negative?
|
|
174
|
+
last = range.end || limit
|
|
175
|
+
last += limit if last.negative?
|
|
176
|
+
last -= 1 if range.exclude_end? && range.end
|
|
177
|
+
count = last - first + 1
|
|
178
|
+
unless first >= 0 && count.positive? && first + count <= limit
|
|
179
|
+
raise ArgumentError, "row range #{range} is out of bounds (#{limit} rows)"
|
|
180
|
+
end
|
|
181
|
+
|
|
182
|
+
[first, count]
|
|
183
|
+
else
|
|
184
|
+
raise ArgumentError, "expected an Integer or a Range, got #{range.class}"
|
|
185
|
+
end
|
|
186
|
+
end
|
|
187
|
+
private :normalize_range
|
|
188
|
+
|
|
189
|
+
def resolve_shape(dims)
|
|
190
|
+
unless dims.all? { |d| d.is_a?(Integer) }
|
|
191
|
+
raise ArgumentError, "shape dimensions must be Integers, got #{dims.inspect}"
|
|
192
|
+
end
|
|
193
|
+
|
|
194
|
+
wild = dims.count(-1)
|
|
195
|
+
raise ArgumentError, "at most one dimension may be -1 (got #{dims.inspect})" if wild > 1
|
|
196
|
+
|
|
197
|
+
if wild == 1
|
|
198
|
+
known = dims.reject { |d| d == -1 }.inject(1, :*)
|
|
199
|
+
unless known.positive? && (@length % known).zero?
|
|
200
|
+
raise ArgumentError, "cannot infer -1 in #{dims.inspect} for #{@length} elements"
|
|
201
|
+
end
|
|
202
|
+
|
|
203
|
+
dims = dims.map { |d| d == -1 ? @length / known : d }
|
|
204
|
+
end
|
|
205
|
+
|
|
206
|
+
unless dims.inject(1, :*) == @length
|
|
207
|
+
raise ArgumentError, "shape #{dims.inspect} has #{dims.inject(1, :*)} elements, but the buffer has #{@length}"
|
|
208
|
+
end
|
|
209
|
+
|
|
210
|
+
dims
|
|
211
|
+
end
|
|
212
|
+
private :resolve_shape
|
|
213
|
+
|
|
214
|
+
# Offloading swaps the handle, which would break a view's link to its parent.
|
|
215
|
+
def reject_view!(action)
|
|
216
|
+
return unless view?
|
|
217
|
+
|
|
218
|
+
raise Error, "cannot #{action} a view; call it on the base buffer instead"
|
|
219
|
+
end
|
|
220
|
+
private :reject_view!
|
|
221
|
+
|
|
222
|
+
def require_resident!(action)
|
|
223
|
+
return if resident?
|
|
224
|
+
|
|
225
|
+
raise Error, "cannot #{action} a buffer that is not resident (offloaded or freed)"
|
|
226
|
+
end
|
|
227
|
+
private :require_resident!
|
|
228
|
+
|
|
229
|
+
# bridge.buffer_free is safe to call twice (idempotent) on every backend,
|
|
230
|
+
# so a GC call after an explicit #free is harmless.
|
|
231
|
+
def self.finalizer(handle, bridge)
|
|
232
|
+
proc do
|
|
233
|
+
bridge.buffer_free(handle)
|
|
234
|
+
Telemetry::Resources.untrack_buffer(handle)
|
|
235
|
+
end
|
|
236
|
+
end
|
|
237
|
+
|
|
238
|
+
# Transfers a one- or two-dimensional Ruby Array into a buffer.
|
|
239
|
+
#
|
|
240
|
+
# @param array [Array<Numeric>, Array<Array<Numeric>>] values to transfer
|
|
241
|
+
# @param dtype [:float32, :float16, :int32, nil] storage dtype; inferred when omitted
|
|
242
|
+
# @return [Buffer]
|
|
243
|
+
def self.from(array, dtype: nil)
|
|
244
|
+
raise ArgumentError, "array must not be empty" if array.empty?
|
|
245
|
+
|
|
246
|
+
if array.first.is_a?(Array)
|
|
247
|
+
unless array.all? { |e| e.is_a?(Array) }
|
|
248
|
+
raise ArgumentError, "array must not mix nested Arrays with scalar elements"
|
|
249
|
+
end
|
|
250
|
+
|
|
251
|
+
return from_matrix(array, dtype: dtype)
|
|
252
|
+
end
|
|
253
|
+
|
|
254
|
+
dtype ||= array.all?(Integer) ? :int32 : :float32
|
|
255
|
+
buffer = new(dtype, array.length)
|
|
256
|
+
buffer.write(array)
|
|
257
|
+
buffer
|
|
258
|
+
end
|
|
259
|
+
|
|
260
|
+
# Transfers a rectangular nested Array into a row-major two-dimensional buffer.
|
|
261
|
+
#
|
|
262
|
+
# @param rows [Array<Array<Numeric>>]
|
|
263
|
+
# @param dtype [:float32, :float16, :int32, nil]
|
|
264
|
+
# @return [Buffer]
|
|
265
|
+
def self.from_matrix(rows, dtype: nil)
|
|
266
|
+
raise ArgumentError, "rows must not be empty" if rows.empty?
|
|
267
|
+
unless rows.all? { |r| r.is_a?(Array) && r.length == rows.first.length }
|
|
268
|
+
raise ArgumentError, "all rows must be non-empty Arrays of the same length"
|
|
269
|
+
end
|
|
270
|
+
|
|
271
|
+
flat = rows.flatten(1)
|
|
272
|
+
dtype ||= flat.all?(Integer) ? :int32 : :float32
|
|
273
|
+
buffer = new(dtype, flat.length, shape: [rows.length, rows.first.length])
|
|
274
|
+
buffer.write(flat)
|
|
275
|
+
buffer
|
|
276
|
+
end
|
|
277
|
+
|
|
278
|
+
# @return [Boolean] whether this buffer has exactly two dimensions
|
|
279
|
+
def matrix?
|
|
280
|
+
@shape.length == 2
|
|
281
|
+
end
|
|
282
|
+
|
|
283
|
+
# @return [Integer] row count for a two-dimensional buffer
|
|
284
|
+
# @raise [Error] unless this buffer is two-dimensional
|
|
285
|
+
def rows
|
|
286
|
+
raise Error, "buffer is not 2D (shape: #{@shape.inspect})" unless matrix?
|
|
287
|
+
|
|
288
|
+
@shape[0]
|
|
289
|
+
end
|
|
290
|
+
|
|
291
|
+
# @return [Integer] column count for a two-dimensional buffer
|
|
292
|
+
# @raise [Error] unless this buffer is two-dimensional
|
|
293
|
+
def cols
|
|
294
|
+
raise Error, "buffer is not 2D (shape: #{@shape.inspect})" unless matrix?
|
|
295
|
+
|
|
296
|
+
@shape[1]
|
|
297
|
+
end
|
|
298
|
+
|
|
299
|
+
# Reads a one- or two-dimensional buffer into a Ruby Array.
|
|
300
|
+
#
|
|
301
|
+
# @return [Array<Numeric>, Array<Array<Numeric>>]
|
|
302
|
+
def to_nested_a
|
|
303
|
+
return to_a unless matrix?
|
|
304
|
+
|
|
305
|
+
to_a.each_slice(cols).to_a
|
|
306
|
+
end
|
|
307
|
+
|
|
308
|
+
# @return [Integer] allocation size in bytes
|
|
309
|
+
def bytesize
|
|
310
|
+
@length * @elem_size
|
|
311
|
+
end
|
|
312
|
+
|
|
313
|
+
# Replaces every element with values from a Ruby Array.
|
|
314
|
+
#
|
|
315
|
+
# This synchronizes the active backend before writing.
|
|
316
|
+
#
|
|
317
|
+
# @param array [Array<Numeric>] exactly {#length} values
|
|
318
|
+
# @return [Buffer] this buffer
|
|
319
|
+
def write(array)
|
|
320
|
+
check_freed!
|
|
321
|
+
check_resident!
|
|
322
|
+
unless array.is_a?(Array) && array.length == @length
|
|
323
|
+
raise ArgumentError, "expected an Array of #{@length} elements"
|
|
324
|
+
end
|
|
325
|
+
if gpu_conversion?
|
|
326
|
+
# Send as float32, then convert on the GPU
|
|
327
|
+
staged = Buffer.new(:float32, @length)
|
|
328
|
+
begin
|
|
329
|
+
staged.write(array)
|
|
330
|
+
Cast.convert_into(staged, self)
|
|
331
|
+
ensure
|
|
332
|
+
staged.free
|
|
333
|
+
end
|
|
334
|
+
else
|
|
335
|
+
# Wait for pending async kernels. On unified-memory backends
|
|
336
|
+
# buffer_write is a raw memcpy from the host, so writing while an
|
|
337
|
+
# in-flight kernel reads is a data race.
|
|
338
|
+
Tilework.synchronize
|
|
339
|
+
Telemetry.instrument(:h2d, bytes: bytesize) { Tilework.bridge.buffer_write(@handle, @encode.call(array), 0) }
|
|
340
|
+
end
|
|
341
|
+
self
|
|
342
|
+
end
|
|
343
|
+
|
|
344
|
+
# Reads every element into a Ruby Array.
|
|
345
|
+
#
|
|
346
|
+
# This synchronizes the active backend before reading.
|
|
347
|
+
#
|
|
348
|
+
# @return [Array<Numeric>]
|
|
349
|
+
def to_a
|
|
350
|
+
check_freed!
|
|
351
|
+
check_resident!
|
|
352
|
+
Tilework.synchronize
|
|
353
|
+
if gpu_conversion?
|
|
354
|
+
staged = cast(:float32)
|
|
355
|
+
begin
|
|
356
|
+
return staged.to_a
|
|
357
|
+
ensure
|
|
358
|
+
staged.free
|
|
359
|
+
end
|
|
360
|
+
end
|
|
361
|
+
@decode.call(Telemetry.instrument(:d2h, bytes: bytesize) { Tilework.bridge.buffer_read(@handle, bytesize, 0) })
|
|
362
|
+
end
|
|
363
|
+
|
|
364
|
+
# Converts this buffer to a different storage dtype.
|
|
365
|
+
#
|
|
366
|
+
# @param target [:float32, :float16, :bfloat16, :int32]
|
|
367
|
+
# @return [Buffer] a converted buffer, or this buffer when the dtype is unchanged
|
|
368
|
+
#
|
|
369
|
+
# The GPU also rounds to nearest even, so the result matches Tilework::Half
|
|
370
|
+
# bit for bit.
|
|
371
|
+
def cast(target)
|
|
372
|
+
check_freed!
|
|
373
|
+
check_resident!
|
|
374
|
+
return self if target == @dtype
|
|
375
|
+
|
|
376
|
+
Tilework::Cast.convert(self, target)
|
|
377
|
+
end
|
|
378
|
+
|
|
379
|
+
# Reads raw bytes without packing or unpacking values.
|
|
380
|
+
#
|
|
381
|
+
# @param length [Integer, nil] byte count; defaults to the remaining allocation
|
|
382
|
+
# @param offset [Integer] byte offset
|
|
383
|
+
# @return [String]
|
|
384
|
+
#
|
|
385
|
+
# Unlike `to_a` there is no pack/unpack, so **this works directly for
|
|
386
|
+
# reading and writing weight files**. "Header + raw tensors" formats like
|
|
387
|
+
# safetensors can be implemented in the application with this alone (the
|
|
388
|
+
# library need not know the format).
|
|
389
|
+
#
|
|
390
|
+
# w = Tilework::Buffer.new(:float32, 1024)
|
|
391
|
+
# File.open(path) { |f| f.seek(data_offset); w.write_bytes(f.read(w.bytesize)) }
|
|
392
|
+
#
|
|
393
|
+
# offset and length are **bytes, not elements**, for streaming a large file
|
|
394
|
+
# in chunks.
|
|
395
|
+
#
|
|
396
|
+
# The bytes are the backend's native representation (float32 is
|
|
397
|
+
# little-endian IEEE-754 single, float16 is binary16). x86 and ARM are both
|
|
398
|
+
# little-endian, so there is no mismatch with file formats to worry about.
|
|
399
|
+
def read_bytes(length = nil, offset: 0)
|
|
400
|
+
check_freed!
|
|
401
|
+
check_resident!
|
|
402
|
+
length ||= bytesize - offset
|
|
403
|
+
check_byte_range!(offset, length)
|
|
404
|
+
Tilework.synchronize
|
|
405
|
+
Telemetry.instrument(:d2h, bytes: length) { Tilework.bridge.buffer_read(@handle, length, offset) }
|
|
406
|
+
end
|
|
407
|
+
|
|
408
|
+
# Writes raw bytes without converting values.
|
|
409
|
+
#
|
|
410
|
+
# @param bytes [String] byte sequence
|
|
411
|
+
# @param offset [Integer] byte offset
|
|
412
|
+
# @return [Buffer] this buffer
|
|
413
|
+
def write_bytes(bytes, offset: 0)
|
|
414
|
+
check_freed!
|
|
415
|
+
raise ArgumentError, "expected a String of bytes, got #{bytes.class}" unless bytes.is_a?(String)
|
|
416
|
+
|
|
417
|
+
check_byte_range!(offset, bytes.bytesize)
|
|
418
|
+
# Same as #write: host writes race in-flight kernels.
|
|
419
|
+
Tilework.synchronize
|
|
420
|
+
Telemetry.instrument(:h2d, bytes: bytes.bytesize) { Tilework.bridge.buffer_write(@handle, bytes, offset) }
|
|
421
|
+
self
|
|
422
|
+
end
|
|
423
|
+
|
|
424
|
+
# Bytes per element, for read_bytes / write_bytes offset arithmetic.
|
|
425
|
+
attr_reader :elem_size
|
|
426
|
+
|
|
427
|
+
def check_byte_range!(offset, length)
|
|
428
|
+
unless offset.is_a?(Integer) && !offset.negative? && length.is_a?(Integer) && !length.negative?
|
|
429
|
+
raise ArgumentError, "offset and length must be non-negative Integers"
|
|
430
|
+
end
|
|
431
|
+
return if offset + length <= bytesize
|
|
432
|
+
|
|
433
|
+
raise ArgumentError,
|
|
434
|
+
"byte range #{offset}...#{offset + length} is out of bounds for #{bytesize} bytes"
|
|
435
|
+
end
|
|
436
|
+
private :check_byte_range!
|
|
437
|
+
|
|
438
|
+
# --- VRAM <-> SSD offload ---
|
|
439
|
+
|
|
440
|
+
# Saves this buffer to disk and releases its device allocation.
|
|
441
|
+
#
|
|
442
|
+
# @param path [String] destination file
|
|
443
|
+
# @param checksum [Boolean] include an integrity checksum
|
|
444
|
+
# @return [Buffer] this non-resident buffer
|
|
445
|
+
def offload!(path, checksum: false)
|
|
446
|
+
reject_view!("offload!")
|
|
447
|
+
check_freed!
|
|
448
|
+
return self unless resident?
|
|
449
|
+
|
|
450
|
+
backing = Offload::FileBacking.new(path, checksum: checksum).write(self)
|
|
451
|
+
Tilework.bridge.buffer_free(@handle)
|
|
452
|
+
Telemetry::Resources.untrack_buffer(@handle)
|
|
453
|
+
@handle = nil
|
|
454
|
+
@backing = backing
|
|
455
|
+
@last_backing = backing
|
|
456
|
+
self
|
|
457
|
+
end
|
|
458
|
+
|
|
459
|
+
# Moves this buffer into an explicitly allocated volatile CPU-RAM backing.
|
|
460
|
+
# Unlike #offload!, no filesystem path is interpreted or created.
|
|
461
|
+
def offload_to_host!
|
|
462
|
+
reject_view!("offload_to_host!")
|
|
463
|
+
check_freed!
|
|
464
|
+
return self unless resident?
|
|
465
|
+
unless Offload::HostBacking.available?
|
|
466
|
+
raise Error, "pageable host backing is not available on the #{Tilework.backend.inspect} backend"
|
|
467
|
+
end
|
|
468
|
+
|
|
469
|
+
backing = Offload::HostBacking.new(dtype: @dtype, length: @length, shape: @shape, bytesize: bytesize)
|
|
470
|
+
begin
|
|
471
|
+
backing.write(self)
|
|
472
|
+
rescue StandardError
|
|
473
|
+
backing.delete
|
|
474
|
+
raise
|
|
475
|
+
end
|
|
476
|
+
Tilework.bridge.buffer_free(@handle)
|
|
477
|
+
Telemetry::Resources.untrack_buffer(@handle)
|
|
478
|
+
@handle = nil
|
|
479
|
+
@backing = backing
|
|
480
|
+
@last_backing = backing
|
|
481
|
+
self
|
|
482
|
+
end
|
|
483
|
+
|
|
484
|
+
# Restores an offloaded buffer synchronously.
|
|
485
|
+
#
|
|
486
|
+
# @return [Buffer] this resident buffer
|
|
487
|
+
def restore!
|
|
488
|
+
reject_view!("restore!")
|
|
489
|
+
check_freed!
|
|
490
|
+
return self if resident?
|
|
491
|
+
reject_pending_restore!
|
|
492
|
+
|
|
493
|
+
backing = backing!
|
|
494
|
+
meta = backing.validate!(self)
|
|
495
|
+
|
|
496
|
+
@handle = Tilework.bridge.buffer_alloc(bytesize)
|
|
497
|
+
Telemetry::Resources.track_buffer(@handle, bytesize)
|
|
498
|
+
ObjectSpace.define_finalizer(self, self.class.finalizer(@handle, Tilework.bridge))
|
|
499
|
+
@backing = nil
|
|
500
|
+
backing.restore_into(self, meta)
|
|
501
|
+
self
|
|
502
|
+
end
|
|
503
|
+
|
|
504
|
+
# Starts restoring an offloaded buffer asynchronously.
|
|
505
|
+
#
|
|
506
|
+
# @return [Offload::Transfer] call `#wait` before using the buffer
|
|
507
|
+
#
|
|
508
|
+
# The contents are complete only after Transfer#wait. Until then resident?
|
|
509
|
+
# stays false, so using it without waiting raises NotResidentError.
|
|
510
|
+
def restore_async!(async: true)
|
|
511
|
+
reject_view!("restore_async!")
|
|
512
|
+
check_freed!
|
|
513
|
+
return Offload::Transfer.new(nil, self) if resident?
|
|
514
|
+
reject_pending_restore!
|
|
515
|
+
return Offload::Transfer.new(nil, restore!) unless async
|
|
516
|
+
|
|
517
|
+
backing = backing!
|
|
518
|
+
backing.validate!(self)
|
|
519
|
+
|
|
520
|
+
handle = Tilework.bridge.buffer_alloc(bytesize)
|
|
521
|
+
token = backing.begin_restore(handle, bytesize)
|
|
522
|
+
# Backings that cannot load asynchronously (such as a checksummed
|
|
523
|
+
# FileBacking) fall back to a synchronous restore. Reallocating without
|
|
524
|
+
# first releasing the handle already allocated would leak, so free it here.
|
|
525
|
+
if token.nil?
|
|
526
|
+
Tilework.bridge.buffer_free(handle)
|
|
527
|
+
return Offload::Transfer.new(nil, restore!)
|
|
528
|
+
end
|
|
529
|
+
kind, token = token if token.is_a?(Array)
|
|
530
|
+
@pending_handle = handle
|
|
531
|
+
@pending_backing = backing
|
|
532
|
+
Offload::Transfer.new(token, self,
|
|
533
|
+
poll: kind == :host ? :host_backing_transfer_poll : :transfer_poll,
|
|
534
|
+
finish: kind == :host ? :host_backing_transfer_end : :transfer_end)
|
|
535
|
+
end
|
|
536
|
+
|
|
537
|
+
# Called from Transfer#wait; this is where it becomes resident.
|
|
538
|
+
def __finish_restore__
|
|
539
|
+
return self unless @pending_handle
|
|
540
|
+
|
|
541
|
+
@handle = @pending_handle
|
|
542
|
+
Telemetry::Resources.track_buffer(@handle, bytesize)
|
|
543
|
+
ObjectSpace.define_finalizer(self, self.class.finalizer(@handle, Tilework.bridge))
|
|
544
|
+
@pending_handle = nil
|
|
545
|
+
@backing = nil
|
|
546
|
+
@pending_backing = nil
|
|
547
|
+
self
|
|
548
|
+
end
|
|
549
|
+
|
|
550
|
+
# Called after pipeline leaves the block; writes back to the latest offload target.
|
|
551
|
+
def offload_back!
|
|
552
|
+
reject_view!("offload_back!")
|
|
553
|
+
return self if @freed || !resident?
|
|
554
|
+
|
|
555
|
+
backing = backing_for_reoffload
|
|
556
|
+
backing.write(self)
|
|
557
|
+
Tilework.bridge.buffer_free(@handle)
|
|
558
|
+
Telemetry::Resources.untrack_buffer(@handle)
|
|
559
|
+
@handle = nil
|
|
560
|
+
@backing = backing
|
|
561
|
+
self
|
|
562
|
+
end
|
|
563
|
+
|
|
564
|
+
# Releases device memory without rewriting the existing offload file.
|
|
565
|
+
#
|
|
566
|
+
# @return [Buffer] this non-resident buffer
|
|
567
|
+
# @note Changes since restoration are discarded.
|
|
568
|
+
#
|
|
569
|
+
# Assumes the offload file exists and is unchanged — when streaming
|
|
570
|
+
# read-only weights, writing back only rewrites the same contents, and as
|
|
571
|
+
# synchronous I/O it cannot overlap compute and defeats hiding.
|
|
572
|
+
#
|
|
573
|
+
# **Changes made in the block are lost.** To keep them use offload_back!
|
|
574
|
+
# (write_back: true for pipeline).
|
|
575
|
+
def evict!
|
|
576
|
+
reject_view!("evict!")
|
|
577
|
+
return self if @freed || !resident?
|
|
578
|
+
|
|
579
|
+
backing = backing_for_reoffload
|
|
580
|
+
Tilework.bridge.buffer_free(@handle)
|
|
581
|
+
Telemetry::Resources.untrack_buffer(@handle)
|
|
582
|
+
@handle = nil
|
|
583
|
+
@backing = backing
|
|
584
|
+
self
|
|
585
|
+
end
|
|
586
|
+
|
|
587
|
+
# @return [Boolean] whether this buffer can be used by device operations
|
|
588
|
+
def resident?
|
|
589
|
+
!@freed && !@handle.nil?
|
|
590
|
+
end
|
|
591
|
+
|
|
592
|
+
# Temporarily restores offloaded buffers for a block.
|
|
593
|
+
#
|
|
594
|
+
# @param buffers [Array<Buffer>]
|
|
595
|
+
# @param write_back [Boolean] re-offload modified buffers when the block exits
|
|
596
|
+
# @yieldparam buffers [Buffer]
|
|
597
|
+
# @return [Object] the block result
|
|
598
|
+
#
|
|
599
|
+
# Tilework::Buffer.with_resident(w1, w2, w3) { |a, b, c| a.matmul(b).matmul(c) }
|
|
600
|
+
#
|
|
601
|
+
# Restores are **issued in parallel and waited on once**. Nesting
|
|
602
|
+
# with_resident one buffer at a time serializes the transfers and cannot
|
|
603
|
+
# fill NVMe's queue depth (measured: 1.48x at 4, 1.69x at 8).
|
|
604
|
+
#
|
|
605
|
+
# write_back: defaults to true so changes made in the block are not lost.
|
|
606
|
+
# When the data is known to be read-only, false skips rewriting the same
|
|
607
|
+
# bytes entirely.
|
|
608
|
+
#
|
|
609
|
+
# Buffers that were already resident are left alone (neither loaded nor
|
|
610
|
+
# offloaded).
|
|
611
|
+
def self.with_resident(*buffers, write_back: true)
|
|
612
|
+
targets = buffers.reject { |b| b.freed? || b.resident? }
|
|
613
|
+
Offload.wait_all(targets.map(&:restore_async!))
|
|
614
|
+
|
|
615
|
+
yield(*buffers)
|
|
616
|
+
ensure
|
|
617
|
+
targets&.each do |b|
|
|
618
|
+
next if b.freed?
|
|
619
|
+
|
|
620
|
+
write_back ? b.offload_back! : b.evict!
|
|
621
|
+
end
|
|
622
|
+
end
|
|
623
|
+
|
|
624
|
+
# Temporarily restores this buffer for a block.
|
|
625
|
+
#
|
|
626
|
+
# @param write_back [Boolean] re-offload this buffer when the block exits
|
|
627
|
+
# @yieldparam buffer [Buffer]
|
|
628
|
+
# @return [Object] the block result
|
|
629
|
+
#
|
|
630
|
+
# restore! on entry, offload on exit: the explicit sugar in place of
|
|
631
|
+
# restoring offloaded buffers implicitly.
|
|
632
|
+
def with_resident(write_back: true, &block)
|
|
633
|
+
self.class.with_resident(self, write_back: write_back, &block)
|
|
634
|
+
end
|
|
635
|
+
|
|
636
|
+
# Releases this buffer's device allocation and any owned offload file.
|
|
637
|
+
#
|
|
638
|
+
# Inside a {Tilework.submit} block the release is **deferred** to the end of
|
|
639
|
+
# the block: launched kernels may still be reading this memory, and handing
|
|
640
|
+
# it back before they finish is undefined behavior. The buffer therefore
|
|
641
|
+
# stays usable — and `#freed?` stays false — until the job completes.
|
|
642
|
+
#
|
|
643
|
+
# @return [nil]
|
|
644
|
+
#
|
|
645
|
+
# An offloaded buffer's file is deleted too (so a Scope ending releases VRAM
|
|
646
|
+
# and deletes offload files alike). Persistence through Offload::Store is
|
|
647
|
+
# untouched, since the Store owns those paths.
|
|
648
|
+
def free
|
|
649
|
+
return if @freed
|
|
650
|
+
return if Ops.defer_free(self)
|
|
651
|
+
|
|
652
|
+
reject_pending_restore!
|
|
653
|
+
|
|
654
|
+
if @handle
|
|
655
|
+
Tilework.bridge.buffer_free(@handle) # a view releases no real memory (decided on the C side)
|
|
656
|
+
Telemetry::Resources.untrack_buffer(@handle) unless view?
|
|
657
|
+
elsif @backing
|
|
658
|
+
@backing.delete
|
|
659
|
+
end
|
|
660
|
+
# A HostBacking is volatile and the Buffer is its only owner. After
|
|
661
|
+
# restore! @backing is nil but @last_backing remains as the re-offload
|
|
662
|
+
# target, so it must be reclaimed here. A FileBacking keeps the existing
|
|
663
|
+
# Store/path ownership.
|
|
664
|
+
if @last_backing.is_a?(Offload::HostBacking)
|
|
665
|
+
@last_backing.delete
|
|
666
|
+
@last_backing = nil
|
|
667
|
+
end
|
|
668
|
+
@handle = nil
|
|
669
|
+
@freed = true
|
|
670
|
+
nil
|
|
671
|
+
end
|
|
672
|
+
|
|
673
|
+
# Graph residency uses this after restoring a plan-owned volatile backing.
|
|
674
|
+
# Durable FileBacking remains owned by its path/store and is intentionally
|
|
675
|
+
# untouched.
|
|
676
|
+
def discard_host_backing!
|
|
677
|
+
return self unless @last_backing.is_a?(Offload::HostBacking)
|
|
678
|
+
|
|
679
|
+
@last_backing.delete
|
|
680
|
+
@last_backing = nil
|
|
681
|
+
self
|
|
682
|
+
end
|
|
683
|
+
|
|
684
|
+
def freed?
|
|
685
|
+
@freed
|
|
686
|
+
end
|
|
687
|
+
|
|
688
|
+
# The native handle passed to the bridge layer; not public API.
|
|
689
|
+
def __handle__
|
|
690
|
+
check_freed!
|
|
691
|
+
check_resident!
|
|
692
|
+
@handle
|
|
693
|
+
end
|
|
694
|
+
|
|
695
|
+
private
|
|
696
|
+
|
|
697
|
+
# Large float16 conversions are orders of magnitude faster on the GPU (see the constant's comment).
|
|
698
|
+
def gpu_conversion?
|
|
699
|
+
%i[float16 bfloat16].include?(@dtype) && @length >= GPU_CONVERSION_THRESHOLD &&
|
|
700
|
+
Cast.supported?(:float32, @dtype)
|
|
701
|
+
end
|
|
702
|
+
|
|
703
|
+
def check_dtype_supported!(dtype)
|
|
704
|
+
supported = Tilework.capabilities[:dtypes]
|
|
705
|
+
return if supported.include?(dtype)
|
|
706
|
+
|
|
707
|
+
raise DeviceError,
|
|
708
|
+
"dtype #{dtype.inspect} is not supported on the #{Tilework.backend.inspect} backend " \
|
|
709
|
+
"(supported: #{supported.map(&:inspect).join(' / ')})"
|
|
710
|
+
end
|
|
711
|
+
|
|
712
|
+
# Where with_resident's exit writes back. restore! sets @backing to nil, so
|
|
713
|
+
# the latest backing is remembered separately.
|
|
714
|
+
def backing_for_reoffload
|
|
715
|
+
@last_backing or
|
|
716
|
+
raise Error, "this buffer has never been offloaded, so there is nowhere to put it back; " \
|
|
717
|
+
"call #offload!(path) first (Tilework::Offload.pipeline expects an already-offloaded series)"
|
|
718
|
+
end
|
|
719
|
+
|
|
720
|
+
def backing!
|
|
721
|
+
@backing or raise Error, "buffer has no backing to restore"
|
|
722
|
+
end
|
|
723
|
+
|
|
724
|
+
# The pending GPU allocation and HostBacking are owned by Transfer until
|
|
725
|
+
# #wait joins it. Starting another restore or freeing them beforehand
|
|
726
|
+
# would race the same backing store.
|
|
727
|
+
def reject_pending_restore!
|
|
728
|
+
return unless @pending_handle
|
|
729
|
+
|
|
730
|
+
raise Error, "buffer restore is still in progress; call Transfer#wait first"
|
|
731
|
+
end
|
|
732
|
+
|
|
733
|
+
def check_resident!
|
|
734
|
+
return if resident?
|
|
735
|
+
|
|
736
|
+
raise NotResidentError,
|
|
737
|
+
"buffer is offloaded to #{backing_for_reoffload.diagnostic}; call #restore! " \
|
|
738
|
+
"(or wrap the use in #with_resident) before operating on it"
|
|
739
|
+
end
|
|
740
|
+
|
|
741
|
+
def validate_shape(shape)
|
|
742
|
+
unless shape.is_a?(Array) && shape.all? { |d| d.is_a?(Integer) && d.positive? } && shape.reduce(:*) == @length
|
|
743
|
+
raise ArgumentError, "shape #{shape.inspect} does not match length #{@length}"
|
|
744
|
+
end
|
|
745
|
+
|
|
746
|
+
shape
|
|
747
|
+
end
|
|
748
|
+
|
|
749
|
+
def check_freed!
|
|
750
|
+
raise Error, "buffer has been freed" if @freed
|
|
751
|
+
end
|
|
752
|
+
end
|
|
753
|
+
end
|