tilework 0.18.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE +21 -0
  3. data/lib/tilework/backend/cpu/dsl_codegen.rb +509 -0
  4. data/lib/tilework/backend/cpu/ops_codegen.rb +451 -0
  5. data/lib/tilework/backend/cpu.rb +156 -0
  6. data/lib/tilework/buffer.rb +753 -0
  7. data/lib/tilework/capabilities.rb +183 -0
  8. data/lib/tilework/compat/cuda.h +66 -0
  9. data/lib/tilework/compat/hip.h +35 -0
  10. data/lib/tilework/compat/metal.h +45 -0
  11. data/lib/tilework/conformance.rb +2296 -0
  12. data/lib/tilework/core_ext.rb +21 -0
  13. data/lib/tilework/dialect.rb +67 -0
  14. data/lib/tilework/dsl.rb +722 -0
  15. data/lib/tilework/dtype/bfloat.rb +31 -0
  16. data/lib/tilework/dtype/cast.rb +88 -0
  17. data/lib/tilework/dtype/half.rb +98 -0
  18. data/lib/tilework/graph/builder.rb +1073 -0
  19. data/lib/tilework/graph/fusion.rb +520 -0
  20. data/lib/tilework/graph/grad.rb +443 -0
  21. data/lib/tilework/graph/high_level.rb +266 -0
  22. data/lib/tilework/graph/kernels.rb +403 -0
  23. data/lib/tilework/graph/node.rb +27 -0
  24. data/lib/tilework/graph/op.rb +290 -0
  25. data/lib/tilework/graph/optim.rb +183 -0
  26. data/lib/tilework/graph/plan.rb +1086 -0
  27. data/lib/tilework/graph/pool.rb +178 -0
  28. data/lib/tilework/graph/region.rb +210 -0
  29. data/lib/tilework/graph/residency.rb +449 -0
  30. data/lib/tilework/graph/state.rb +64 -0
  31. data/lib/tilework/graph/tensor.rb +124 -0
  32. data/lib/tilework/graph.rb +213 -0
  33. data/lib/tilework/kernel.rb +163 -0
  34. data/lib/tilework/math/conv.rb +270 -0
  35. data/lib/tilework/math/fft.rb +172 -0
  36. data/lib/tilework/math/nn.rb +1103 -0
  37. data/lib/tilework/math/philox.rb +194 -0
  38. data/lib/tilework/math/random.rb +232 -0
  39. data/lib/tilework/math/tensor_core.rb +269 -0
  40. data/lib/tilework/ops.rb +682 -0
  41. data/lib/tilework/runtime/offload.rb +498 -0
  42. data/lib/tilework/runtime/submission.rb +163 -0
  43. data/lib/tilework/runtime/telemetry.rb +112 -0
  44. data/lib/tilework/scope.rb +33 -0
  45. data/lib/tilework/version.rb +5 -0
  46. data/lib/tilework.rb +345 -0
  47. metadata +92 -0
@@ -0,0 +1,498 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "fileutils"
4
+
5
+ module Tilework
6
+ # Explicit persistence and restoration of device buffers.
7
+ #
8
+ # The purpose is moving what does not fit in VRAM out to SSD, not backup, so
9
+ # offload! always releases VRAM.
10
+ #
11
+ # Computing on an offloaded Buffer raises instead of restoring implicitly:
12
+ # swap-like implicit restores would make performance unpredictable — the
13
+ # same thinking that made to_a an explicit transfer. Buffer#with_resident is
14
+ # the explicit sugar.
15
+ module Offload
16
+ MAGIC = "RMGPUOFF"
17
+ VERSION = 1
18
+ # Header and body both sit on 4096-byte boundaries to suit O_DIRECT. The
19
+ # Ruby fallback writes the same format, so swapping in the C ABI
20
+ # implementation later keeps files compatible.
21
+ HEADER_BYTES = 4096
22
+ ALIGN = 4096
23
+
24
+ DTYPE_CODES = { float32: 0, int32: 1, float16: 2, bfloat16: 3 }.freeze
25
+ CODE_DTYPES = DTYPE_CODES.invert.freeze
26
+
27
+ MAX_DIMS = 8
28
+
29
+ module_function
30
+
31
+ # Non-resident Buffer payloads. The public `Buffer#offload!(path)` API
32
+ # remains path-shaped, but Buffer itself owns a backing object so native
33
+ # host memory can use the identical residency transitions without being
34
+ # disguised as a filesystem path.
35
+ class Backing
36
+ def kind = :backing
37
+
38
+ def diagnostic = kind.to_s
39
+
40
+ def delete; end
41
+ end
42
+
43
+ class FileBacking < Backing
44
+ attr_reader :path, :checksum
45
+
46
+ def initialize(path, checksum: false)
47
+ @path = String(path)
48
+ @checksum = checksum
49
+ end
50
+
51
+ def kind = :file
52
+
53
+ def diagnostic = "file backing at #{@path.inspect}"
54
+
55
+ def write(buffer)
56
+ Offload.write_file(@path, buffer, checksum: @checksum)
57
+ self
58
+ end
59
+
60
+ def metadata
61
+ Offload.read_header(@path)
62
+ end
63
+
64
+ def validate!(buffer)
65
+ meta = metadata
66
+ unless meta[:dtype] == buffer.dtype && meta[:length] == buffer.length
67
+ raise Error, "offload file #{@path} does not match this buffer " \
68
+ "(#{meta[:dtype]} x#{meta[:length]} vs #{buffer.dtype} x#{buffer.length})"
69
+ end
70
+ meta
71
+ end
72
+
73
+ def restore_into(buffer, meta = validate!(buffer))
74
+ Offload.read_body_into(@path, buffer, meta)
75
+ end
76
+
77
+ # Returns a bridge transfer token or nil when the durable file requires
78
+ # the synchronous/checksummed path.
79
+ def begin_restore(handle, bytes, meta = validate_metadata)
80
+ return nil unless Offload.async? && meta[:checksum].nil?
81
+
82
+ Tilework.bridge.transfer_begin(handle, @path, bytes, 0, 1)
83
+ end
84
+
85
+ def delete
86
+ FileUtils.rm_f(@path)
87
+ end
88
+
89
+ private
90
+
91
+ def validate_metadata = metadata
92
+ end
93
+
94
+ # Volatile pageable host storage. Its opaque native handle is intentionally
95
+ # never represented as a path or exposed in diagnostics. Only backends
96
+ # implementing the optional host-backing bridge ABI can construct it.
97
+ class HostBacking < Backing
98
+ Ownership = Struct.new(:handle, :bridge, :freed)
99
+
100
+ attr_reader :dtype, :length, :shape
101
+
102
+ def self.available?
103
+ Tilework.bridge.respond_to?(:host_backing_available?) && Tilework.bridge.host_backing_available?
104
+ end
105
+
106
+ def initialize(dtype:, length:, shape:, bytesize:)
107
+ @dtype, @length, @shape = dtype, length, shape.dup
108
+ bridge = Tilework.bridge
109
+ @handle = bridge.host_backing_alloc(bytesize)
110
+ @ownership = Ownership.new(@handle, bridge, false)
111
+ ObjectSpace.define_finalizer(self, self.class.finalizer(@ownership))
112
+ end
113
+
114
+ def self.finalizer(ownership)
115
+ proc do
116
+ next if ownership.freed
117
+
118
+ ownership.bridge.host_backing_free(ownership.handle)
119
+ ownership.freed = true
120
+ rescue StandardError
121
+ nil # finalizers must never turn GC into an application exception
122
+ end
123
+ end
124
+
125
+ def kind = :host
126
+
127
+ def diagnostic = "pageable host backing"
128
+
129
+ def validate!(buffer)
130
+ return self if buffer.dtype == @dtype && buffer.length == @length && buffer.shape == @shape
131
+
132
+ raise Error, "host backing does not match this buffer"
133
+ end
134
+
135
+ def write(buffer)
136
+ validate!(buffer)
137
+ Tilework.bridge.host_backing_store(@handle, buffer.__handle__, buffer.bytesize, 0)
138
+ self
139
+ end
140
+
141
+ def restore_into(buffer, _meta = self)
142
+ validate!(buffer)
143
+ Tilework.bridge.host_backing_restore(@handle, buffer.__handle__, buffer.bytesize, 0)
144
+ buffer
145
+ end
146
+
147
+ def async_available?
148
+ Tilework.bridge.respond_to?(:host_backing_async_available?) && Tilework.bridge.host_backing_async_available?
149
+ end
150
+
151
+ def begin_restore(handle, bytes)
152
+ return nil unless async_available?
153
+
154
+ [:host, Tilework.bridge.host_backing_transfer_begin(@handle, handle, bytes, 0)]
155
+ end
156
+
157
+ def delete
158
+ return if @ownership.freed
159
+
160
+ @ownership.bridge.host_backing_free(@handle)
161
+ @ownership.freed = true
162
+ end
163
+ end
164
+
165
+ # @return [:staged, :posix] active file-transfer implementation
166
+ #
167
+ # :staged — the bridge's pinned staging (O_DIRECT where possible)
168
+ # :posix — the plain D2H -> Ruby String -> File.write path
169
+ #
170
+ # GPUDirect Storage (:gpudirect) is not implemented; on the test machine
171
+ # cuFile only reaches compatibility mode and gains nothing.
172
+ def transport
173
+ return :posix if ENV["GPU_OFFLOAD_FORCE_POSIX"] == "1"
174
+ return :posix unless Tilework.backend
175
+ return :posix unless Tilework.bridge.respond_to?(:offload_available?) && Tilework.bridge.offload_available?
176
+
177
+ :staged
178
+ end
179
+
180
+ # @return [Boolean] whether the backend supplies a native staged path
181
+ def native? = transport == :staged
182
+
183
+ # @return [Boolean] whether asynchronous restore is supported
184
+ #
185
+ # Whether the async transfer calls (transfer_begin/poll/end) exist. Without
186
+ # them it falls back to the synchronous version and prefetch can no longer
187
+ # hide transfers.
188
+ def async?
189
+ return false if ENV["GPU_OFFLOAD_FORCE_POSIX"] == "1"
190
+ return false unless Tilework.backend
191
+
192
+ Tilework.bridge.respond_to?(:transfer_available?) && Tilework.bridge.transfer_available?
193
+ end
194
+
195
+ # A pending asynchronous buffer restore.
196
+ class Transfer
197
+ def initialize(token, buffer, poll: :transfer_poll, finish: :transfer_end)
198
+ @token = token
199
+ @buffer = buffer
200
+ @poll = poll
201
+ @finish = finish
202
+ @finished = token.nil?
203
+ end
204
+
205
+ # Polls without waiting.
206
+ # @return [Boolean]
207
+ def done?
208
+ return true if @finished
209
+
210
+ Tilework.bridge.public_send(@poll, @token) != 0
211
+ end
212
+
213
+ # Waits for the transfer and makes its buffer resident.
214
+ # @return [Buffer]
215
+ def wait
216
+ return @buffer if @finished
217
+
218
+ Tilework.bridge.public_send(@finish, @token)
219
+ @finished = true
220
+ @buffer.__finish_restore__
221
+ end
222
+ end
223
+
224
+ # Builds the 4096-byte header. No endianness conversion — only x86_64 /
225
+ # aarch64 are targeted, so Buffer's internal representation is written as is.
226
+ def pack_header(dtype:, length:, shape:, checksum: nil)
227
+ code = DTYPE_CODES[dtype] or raise ArgumentError, "cannot offload dtype #{dtype.inspect}"
228
+ raise ArgumentError, "shape has too many dimensions: #{shape.length}" if shape.length > MAX_DIMS
229
+
230
+ dims = shape + [0] * (MAX_DIMS - shape.length)
231
+ flags = checksum ? 1 : 0
232
+ head = [MAGIC, VERSION, code, length, shape.length, *dims, flags, checksum || 0]
233
+ .pack("a8 L L Q L L#{MAX_DIMS} L Q")
234
+ head + ("\0" * (HEADER_BYTES - head.bytesize))
235
+ end
236
+
237
+ def unpack_header(bytes)
238
+ raise Error, "offload file is truncated (#{bytes.bytesize} bytes)" if bytes.bytesize < HEADER_BYTES
239
+
240
+ magic, version, code, length, ndim, *rest = bytes.unpack("a8 L L Q L L#{MAX_DIMS} L Q")
241
+ raise Error, "not a tilework offload file (bad magic)" unless magic == MAGIC
242
+ raise Error, "unsupported offload file version #{version}" unless version == VERSION
243
+
244
+ dtype = CODE_DTYPES[code] or raise Error, "unknown dtype code #{code} in offload file"
245
+ dims = rest.first(MAX_DIMS)
246
+ flags = rest[MAX_DIMS]
247
+ checksum = rest[MAX_DIMS + 1]
248
+ { dtype: dtype, length: length, shape: dims.first(ndim),
249
+ checksum: flags.anybits?(1) ? checksum : nil }
250
+ end
251
+
252
+ # Buffer -> file. Uses the bridge's optional save ABI if implemented,
253
+ # otherwise the plain D2H -> Ruby String -> File.write path. The file
254
+ # contents are the same either way.
255
+ def write_file(path, buffer, checksum: false)
256
+ FileUtils.mkdir_p(File.dirname(path))
257
+
258
+ # A checksum requires looking at the body in Ruby, which cancels the
259
+ # native path's advantage (not routing D2H through a Ruby String), so
260
+ # use the posix path in that case.
261
+ if native? && !checksum
262
+ write_file_native(path, buffer)
263
+ else
264
+ write_file_posix(path, buffer, checksum: checksum)
265
+ end
266
+ path
267
+ end
268
+
269
+ def write_file_posix(path, buffer, checksum: false)
270
+ body = buffer.read_bytes
271
+ sum = checksum ? checksum_of(body) : nil
272
+ padding = (-body.bytesize) % ALIGN
273
+
274
+ File.open(path, "wb") do |f|
275
+ f.write(pack_header(dtype: buffer.dtype, length: buffer.length,
276
+ shape: buffer.shape, checksum: sum))
277
+ f.write(body)
278
+ f.write("\0" * padding) if padding.positive?
279
+ # A checkpoint lost to a power failure is useless, so make it durable.
280
+ # The native path fsyncs too, so both paths are equally durable.
281
+ f.fsync
282
+ end
283
+ end
284
+
285
+ # The header is written first with ordinary I/O and only the body goes
286
+ # through the bridge's O_DIRECT. The file is bit-identical to the posix path's.
287
+ def write_file_native(path, buffer)
288
+ File.open(path, "wb") do |f|
289
+ f.write(pack_header(dtype: buffer.dtype, length: buffer.length, shape: buffer.shape))
290
+ # Reserve the O_DIRECT target (pwrite could extend the file, but
291
+ # truncate explicitly so the size matches the posix path)
292
+ f.truncate(HEADER_BYTES + ((buffer.bytesize + ALIGN - 1) / ALIGN * ALIGN))
293
+ end
294
+ Tilework.bridge.buffer_save(buffer.__handle__, path, buffer.bytesize, 0)
295
+ end
296
+
297
+ # File -> new Buffer. dtype/shape are fully restored from the header, so
298
+ # callers need not remember the shape.
299
+ def read_file(path)
300
+ meta = read_header(path)
301
+ buffer = Buffer.new(meta[:dtype], meta[:length], shape: meta[:shape])
302
+ read_body_into(path, buffer, meta)
303
+ buffer
304
+ end
305
+
306
+ def read_header(path)
307
+ unpack_header(File.binread(path, HEADER_BYTES).to_s)
308
+ end
309
+
310
+ # Reads the body into an existing Buffer. Checksum verification needs the
311
+ # body in Ruby, so that case uses the posix path.
312
+ def read_body_into(path, buffer, meta)
313
+ if native? && meta[:checksum].nil?
314
+ Tilework.bridge.buffer_load(buffer.__handle__, path, buffer.bytesize, 0)
315
+ return buffer
316
+ end
317
+
318
+ body = File.open(path, "rb") do |f|
319
+ f.seek(HEADER_BYTES)
320
+ f.read(buffer.bytesize).to_s
321
+ end
322
+ raise Error, "offload file body is truncated (#{body.bytesize} of #{buffer.bytesize} bytes)" if body.bytesize < buffer.bytesize
323
+
324
+ if meta[:checksum] && checksum_of(body) != meta[:checksum]
325
+ raise Error, "offload file checksum mismatch for #{path} (corrupted?)"
326
+ end
327
+ buffer.write_bytes(body)
328
+ buffer
329
+ end
330
+
331
+ # The checksum is optional: off by default since it is wasted computation
332
+ # for training spills, on for checkpoints. xxh64 is not in the standard
333
+ # library, so FNV-1a folds to 64 bits without adding a dependency.
334
+ def checksum_of(bytes)
335
+ hash = 0xcbf29ce484222325
336
+ bytes.each_byte do |b|
337
+ hash = ((hash ^ b) * 0x100000001b3) & 0xFFFFFFFFFFFFFFFF
338
+ end
339
+ hash
340
+ end
341
+
342
+ def store(dir)
343
+ Store.new(dir)
344
+ end
345
+
346
+ # An alias of pipeline, for when the name should say the block runs once
347
+ # per element (in the family of each_slice / each_with_index). To load one
348
+ # or N buffers at once, use Tilework::Buffer.with_resident instead.
349
+ def each_resident(buffers, depth: 2, write_back: false, &block)
350
+ pipeline(buffers, depth: depth, write_back: write_back, &block)
351
+ end
352
+
353
+ # Makes an offloaded series resident in turn while prefetching ahead.
354
+ #
355
+ # Tilework::Offload.pipeline(layers, depth: 2) do |buf|
356
+ # ... # buf is resident; the next depth-1 are transferring meanwhile
357
+ # end
358
+ #
359
+ # depth: 2 means one element of prefetch. VRAM use scales with one
360
+ # element's size x depth, so choose depth from the free VRAM. A Buffer
361
+ # leaving the block goes back to its offload target, freeing VRAM.
362
+ #
363
+ # Whether transfers are hidden depends on the ratio of compute time to load
364
+ # time: if per-element compute is shorter than its transfer, raising depth
365
+ # adds no bandwidth and the waits remain.
366
+ #
367
+ # write_back: defaults to false. When streaming read-only weights, the
368
+ # offload file does not change, so writing back only rewrites the same
369
+ # contents, and as synchronous I/O it cannot overlap compute and defeats
370
+ # hiding (measured: raising depth stayed at 1.0x). **Set it true only when
371
+ # changes made in the block must be kept.**
372
+ def pipeline(items, depth: 2, write_back: false)
373
+ raise ArgumentError, "depth must be >= 1" unless depth.is_a?(Integer) && depth >= 1
374
+
375
+ # An element may be a single Buffer or an array of Buffers. A real
376
+ # model's "layer" is a group of tensors rather than one weight matrix (in a
377
+ # transformer, heads x {wq,wk,wv,wo} + w1 + w2), and layer-wise streaming
378
+ # needs groups to flow as a unit.
379
+ #
380
+ # Do not use Array() — it would call Buffer#to_a (a read from the device).
381
+ groups = items.map { |item| item.is_a?(Array) ? item : [item] }
382
+
383
+ pending = {}
384
+ results = []
385
+ prefetch = lambda do |i|
386
+ next if i >= groups.length || pending.key?(i)
387
+
388
+ pending[i] = groups[i].reject { |b| b.freed? || b.resident? }.map(&:restore_async!)
389
+ end
390
+
391
+ # At most depth elements occupy VRAM at once (one being computed plus
392
+ # depth-1 prefetching), so depth: 1 means no prefetch and depth: 2 one
393
+ # element ahead. With groups, VRAM is "group size x depth".
394
+ depth.times { |i| prefetch.call(i) }
395
+
396
+ begin
397
+ items.each_with_index do |item, i|
398
+ Offload.wait_all(pending.delete(i) || [])
399
+ groups[i].each { |b| b.restore! unless b.resident? } # backstop for environments without async
400
+ begin
401
+ results << yield(item)
402
+ ensure
403
+ groups[i].each do |b|
404
+ next if b.freed?
405
+
406
+ write_back ? b.offload_back! : b.evict!
407
+ end
408
+ # Issue the next one only after one element's VRAM is free; issuing
409
+ # earlier would exceed the depth limit.
410
+ prefetch.call(i + depth)
411
+ end
412
+ end
413
+ ensure
414
+ # If the block raises (or breaks), prefetched but unconsumed elements
415
+ # would keep holding VRAM. restore_async! allocates device memory before
416
+ # the wait, so they hold it even while resident? is false. Always
417
+ # collect them here.
418
+ pending.each do |i, transfers|
419
+ begin
420
+ Offload.wait_all(transfers)
421
+ rescue StandardError
422
+ nil # a secondary failure during cleanup must not hide the original exception
423
+ end
424
+ groups[i].each { |b| b.evict! unless b.freed? }
425
+ end
426
+ pending.clear
427
+ end
428
+ results
429
+ end
430
+
431
+ # Collects every launched transfer before raising the first exception;
432
+ # leaving early would strand worker threads, so wait on all of them even on failure.
433
+ def wait_all(transfers)
434
+ first_error = nil
435
+ transfers.each do |t|
436
+ t.wait
437
+ rescue StandardError => e
438
+ first_error ||= e
439
+ end
440
+ raise first_error if first_error
441
+ end
442
+
443
+ # A container for when file names should not be the caller's concern. A
444
+ # Store is explicit persistence: it survives the end of a Scope and stays
445
+ # until clear is called.
446
+ class Store
447
+ attr_reader :dir
448
+
449
+ def initialize(dir)
450
+ @dir = dir
451
+ @seq = 0
452
+ @handles = {}
453
+ FileUtils.mkdir_p(dir)
454
+ end
455
+
456
+ # Stores a buffer and returns its generated handle. The buffer's VRAM is
457
+ # released; get it back with the returned handle.
458
+ # @param buffer [Buffer]
459
+ # @param checksum [Boolean]
460
+ # @return [String]
461
+ def put(buffer, checksum: false)
462
+ handle = (@seq += 1)
463
+ path = File.join(@dir, format("%08d.rmgpu", handle))
464
+ buffer.offload!(path, checksum: checksum)
465
+ @handles[handle] = path
466
+ handle
467
+ end
468
+
469
+ # Restores a stored buffer by handle.
470
+ # @param handle [String]
471
+ # @return [Buffer]
472
+ def get(handle)
473
+ path = @handles[handle] or raise Error, "unknown offload handle #{handle.inspect}"
474
+ Offload.read_file(path)
475
+ end
476
+
477
+ # Deletes a stored buffer.
478
+ # @param handle [String]
479
+ # @return [nil]
480
+ def delete(handle)
481
+ path = @handles.delete(handle) or return false
482
+
483
+ FileUtils.rm_f(path)
484
+ true
485
+ end
486
+
487
+ # Deletes every buffer stored by this instance.
488
+ # @return [nil]
489
+ def clear
490
+ @handles.each_value { |path| FileUtils.rm_f(path) }
491
+ @handles.clear
492
+ nil
493
+ end
494
+
495
+ def size = @handles.size
496
+ end
497
+ end
498
+ end
@@ -0,0 +1,163 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Tilework
4
+ # An explicit asynchronous submit boundary for the eager path (#71).
5
+ #
6
+ # `Kernel#launch` always uses `wait: true`, so the host waits on every
7
+ # launch. On hardware that wait is an order of magnitude larger than a small
8
+ # kernel (a no-op kernel on HIP: 162 µs/op synchronous vs 2.4 µs/op queued on
9
+ # the same stream with one final synchronization). Chains of small launches,
10
+ # like elementwise chains and decode loops, are dominated by synchronization
11
+ # rather than arithmetic.
12
+ #
13
+ # job = Tilework.submit { (a * 2 + b).relu }
14
+ # out = job.wait
15
+ #
16
+ # Only inside the block are launches enqueue-only on the same queue;
17
+ # `Job#wait` synchronizes once and returns the block's value. **Semantics
18
+ # outside the block do not change at all** — the point is an explicit
19
+ # boundary, not implicitly making a synchronous API asynchronous.
20
+ #
21
+ # It rides on the same `Ops` execution-context contract as the graph's
22
+ # `Plan#run` (see the `Ops::CONTEXT_KEY` comment), so going asynchronous
23
+ # needs no new machinery. What differs is ownership, which is most of this
24
+ # file:
25
+ #
26
+ # - **The Job holds every output allocated inside.** `Buffer`'s finalizer
27
+ # calls `buffer_free`, so an intermediate created during submit that lost
28
+ # its last reference could vanish under a kernel reading it.
29
+ # - **`free` is deferred until completion**, rather than rejected, so eager
30
+ # code written with `ensure buf.free` still works wrapped in submit
31
+ # (releasing memory a running kernel uses is undefined behavior on CUDA/HIP).
32
+ # - **No pool.** The graph's `Pool` is safe asynchronously because it knows
33
+ # the execution order and can say "the next writer of a slot is queued
34
+ # after its last reader". Eager has no such information.
35
+ module Submission
36
+ # The context plugged into Ops while running. Satisfies the same shape as
37
+ # Graph::Context (`#allocate` / `#async?` / `#synchronous`).
38
+ class Context
39
+ def initialize(job)
40
+ @job = job
41
+ @async = true
42
+ end
43
+
44
+ attr_reader :job
45
+
46
+ def async? = @async
47
+
48
+ def allocate(dtype, length, shape)
49
+ Buffer.new(dtype, length, shape: shape).tap { |buffer| @job.retain(buffer) }
50
+ end
51
+
52
+ # `Buffer#free` inside the block is deferred to the boundary (the Ops.defer_free contract).
53
+ def defer_free(buffer) = @job.defer_free(buffer)
54
+
55
+ # Delegated libraries run synchronously. Sharing the name with
56
+ # Graph::Context is deliberate, so callers that wrap either context the
57
+ # same way (`Plan#dispatch_node`) need no branch.
58
+ def synchronous
59
+ previous = @async
60
+ @async = false
61
+ yield
62
+ ensure
63
+ @async = previous
64
+ end
65
+ end
66
+
67
+ # The return value of `Tilework.submit`.
68
+ class Job
69
+ def initialize
70
+ @value = nil
71
+ @done = false
72
+ @retained = []
73
+ @deferred_free = []
74
+ end
75
+
76
+ # The block's value, waiting first if `wait` has not been called.
77
+ #
78
+ # @return [Object]
79
+ def wait
80
+ return @value if @done
81
+
82
+ Tilework.synchronize
83
+ complete!
84
+ @value
85
+ end
86
+ alias value wait
87
+
88
+ def done? = @done
89
+
90
+ # An output allocated during submit. While the Job holds it the finalizer
91
+ # cannot run, so memory never vanishes under a running kernel.
92
+ def retain(buffer)
93
+ @retained << buffer
94
+ buffer
95
+ end
96
+
97
+ # Defers a `Buffer#free` made during submit until completion.
98
+ def defer_free(buffer)
99
+ @deferred_free << buffer
100
+ buffer
101
+ end
102
+
103
+ # How many frees are pending (for tests and debugging).
104
+ def deferred_free_count = @deferred_free.length
105
+
106
+ # @api private
107
+ def fulfill(value)
108
+ @value = value
109
+ end
110
+
111
+ # @api private -- an exception unwind also drains the queue before getting here.
112
+ def complete!
113
+ return if @done
114
+
115
+ @done = true
116
+ @retained.clear
117
+ pending = @deferred_free
118
+ @deferred_free = []
119
+ pending.each(&:free)
120
+ nil
121
+ end
122
+ end
123
+
124
+ class << self
125
+ # Whether this thread is inside a submit.
126
+ def context
127
+ context = Ops.context
128
+ context if context.is_a?(Context)
129
+ end
130
+
131
+ def active? = !context.nil?
132
+
133
+ # @api private -- the public API is Tilework.submit
134
+ def run(&block)
135
+ # A nested submit opens no new boundary. Synchronizing inside would also
136
+ # wait for the outer submit's queued work, defeating the explicit
137
+ # boundary. The inner Job is returned completed (its value is already final).
138
+ return nested(&block) if active?
139
+
140
+ job = Job.new
141
+ begin
142
+ job.fulfill(Ops.with_context(Context.new(job), &block))
143
+ rescue StandardError, ScriptError
144
+ # Queued kernels may still be running. Drain before freeing and
145
+ # re-raising, so the unwind does not release memory under them.
146
+ Tilework.synchronize
147
+ job.complete!
148
+ raise
149
+ end
150
+ job
151
+ end
152
+
153
+ private
154
+
155
+ def nested
156
+ job = Job.new
157
+ job.fulfill(yield)
158
+ job.complete!
159
+ job
160
+ end
161
+ end
162
+ end
163
+ end