tilework 0.18.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE +21 -0
  3. data/lib/tilework/backend/cpu/dsl_codegen.rb +509 -0
  4. data/lib/tilework/backend/cpu/ops_codegen.rb +451 -0
  5. data/lib/tilework/backend/cpu.rb +156 -0
  6. data/lib/tilework/buffer.rb +753 -0
  7. data/lib/tilework/capabilities.rb +183 -0
  8. data/lib/tilework/compat/cuda.h +66 -0
  9. data/lib/tilework/compat/hip.h +35 -0
  10. data/lib/tilework/compat/metal.h +45 -0
  11. data/lib/tilework/conformance.rb +2296 -0
  12. data/lib/tilework/core_ext.rb +21 -0
  13. data/lib/tilework/dialect.rb +67 -0
  14. data/lib/tilework/dsl.rb +722 -0
  15. data/lib/tilework/dtype/bfloat.rb +31 -0
  16. data/lib/tilework/dtype/cast.rb +88 -0
  17. data/lib/tilework/dtype/half.rb +98 -0
  18. data/lib/tilework/graph/builder.rb +1073 -0
  19. data/lib/tilework/graph/fusion.rb +520 -0
  20. data/lib/tilework/graph/grad.rb +443 -0
  21. data/lib/tilework/graph/high_level.rb +266 -0
  22. data/lib/tilework/graph/kernels.rb +403 -0
  23. data/lib/tilework/graph/node.rb +27 -0
  24. data/lib/tilework/graph/op.rb +290 -0
  25. data/lib/tilework/graph/optim.rb +183 -0
  26. data/lib/tilework/graph/plan.rb +1086 -0
  27. data/lib/tilework/graph/pool.rb +178 -0
  28. data/lib/tilework/graph/region.rb +210 -0
  29. data/lib/tilework/graph/residency.rb +449 -0
  30. data/lib/tilework/graph/state.rb +64 -0
  31. data/lib/tilework/graph/tensor.rb +124 -0
  32. data/lib/tilework/graph.rb +213 -0
  33. data/lib/tilework/kernel.rb +163 -0
  34. data/lib/tilework/math/conv.rb +270 -0
  35. data/lib/tilework/math/fft.rb +172 -0
  36. data/lib/tilework/math/nn.rb +1103 -0
  37. data/lib/tilework/math/philox.rb +194 -0
  38. data/lib/tilework/math/random.rb +232 -0
  39. data/lib/tilework/math/tensor_core.rb +269 -0
  40. data/lib/tilework/ops.rb +682 -0
  41. data/lib/tilework/runtime/offload.rb +498 -0
  42. data/lib/tilework/runtime/submission.rb +163 -0
  43. data/lib/tilework/runtime/telemetry.rb +112 -0
  44. data/lib/tilework/scope.rb +33 -0
  45. data/lib/tilework/version.rb +5 -0
  46. data/lib/tilework.rb +345 -0
  47. metadata +92 -0
@@ -0,0 +1,1086 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Tilework
4
+ module Graph
5
+ # An executable graph with a fixed operation order and buffer lifetime plan.
6
+ #
7
+ # Nodes run through the existing eager layer (Ops), fused where possible.
8
+ # What compile fixes is **order** and **lifetime**:
9
+ #
10
+ # 1. topologically sort after adding mutable state's WAR edges to the value (RAW) edges
11
+ # 2. release intermediates at their last read (state buffers excluded)
12
+ class Plan
13
+ # Owning memory, running synchronously, leaving our execution path and
14
+ # accepting a placed output are all op properties, held by the Graph::Op
15
+ # table. This class consults it and concentrates on order and lifetime.
16
+ attr_reader :order, :nodes, :strict_state, :async
17
+
18
+ def initialize(builder, vram: nil, spill_dir: nil, backing: :file, host_budget: nil)
19
+ @builder = builder
20
+ # The mode is fixed at compile time (so its meaning cannot change between runs).
21
+ @strict_state = Graph.strict_state
22
+ @async = Graph.async
23
+ @pool = Graph.buffer_pool ? Pool.new : nil
24
+ @context = Context.new(pool: @pool, async: @async)
25
+ check_single_leaf!
26
+ @roots = builder.outputs + builder.states.map { |st| st.leaves.first.node }
27
+ @nodes = reachable(@roots)
28
+ @nodes_by_id = @nodes.to_h { |node| [node.id, node] }
29
+ check_dtypes_supported!
30
+ check_requirements!
31
+ @order_deps = order_dependencies
32
+ @order = toposort
33
+ @clusters, @absorbed = plan_fusion
34
+ @placements = plan_placement
35
+ @release_at = plan_releases
36
+ @residency = plan_residency(vram, spill_dir, backing, host_budget)
37
+ end
38
+
39
+ def input_names = @builder.input_nodes.keys
40
+ def states = @builder.states
41
+ def outputs = @builder.outputs
42
+
43
+ # How many random draws this plan makes; 0 means no seed is needed.
44
+ def random_draws = @builder.random_draws
45
+
46
+ # The seed to pass to the next step. RNG inside a plan is a pure function
47
+ # of keys derived from the seed, so **it only needs advancing between
48
+ # steps** (#19 §6). run does not return the seed, which would make the
49
+ # output layout variable; advancing is offered here as a pure function.
50
+ def next_seed(seed) = seed + 1
51
+
52
+ # Per-node key. Mixing in the index scaled by the golden ratio keeps the
53
+ # sequence apart from both neighboring seeds and neighboring nodes (the
54
+ # same trick as Random.next_seed).
55
+ def self.derive_key(seed, index) = (seed ^ ((index + 1) * 0x9E37_79B9_7F4A_7C15)) & 0xFFFF_FFFF_FFFF_FFFF
56
+
57
+ # Runs the plan with named input and index bindings.
58
+ #
59
+ # @param bindings [Hash{Symbol => Buffer, Integer}] values declared with {Builder#input} or {Builder#index}
60
+ # @return [Buffer, Array<Buffer>] graph outputs in {Builder#output} order
61
+ # @note The method does not synchronize; reading an output waits for queued work.
62
+ #
63
+ # plan.run(x: buf, pos: 7)
64
+ #
65
+ # Outputs come in g.output order; a single one is returned as a bare
66
+ # Buffer. State buffers are updated in place in the bound Buffers (not
67
+ # returned).
68
+ #
69
+ # **The plan does not synchronize at its end.** It returns with launches
70
+ # still queued, so rerunning the same plan, as a decode loop does, adds no
71
+ # per-step wait. Reading a result (`#to_a`) waits for the host transfer,
72
+ # so there is one synchronization when the value is actually wanted.
73
+ def run(**bindings)
74
+ values = {}
75
+ output_ids = outputs.map(&:id)
76
+ # Nodes whose placement took effect; for them the write needs no copy.
77
+ @placed = {}
78
+ @sync_stats = { nodes: 0, synchronous: 0, fences: 0 }
79
+ # If there is an enclosing submit boundary, hand ownership to it (#86
80
+ # stage 2). The plan installs its own context, so without this, deferred
81
+ # frees of buffers made in the submit block would land outside the boundary.
82
+ @context.outer = Submission.context
83
+ Ops.with_context(@context) do
84
+ index = 0
85
+ @order.each do |node|
86
+ next if @absorbed[node.id] # computed inside its cluster, not run alone
87
+
88
+ @residency&.ensure!(index)
89
+ values[node.id] = execute_node(node, values, bindings)
90
+ release(index, values, output_ids)
91
+ note_queue_state(node)
92
+ index += 1
93
+ end
94
+ end
95
+ @context.drain_deferred!
96
+ results = outputs.map { |node| values[node.id] }
97
+ results.length == 1 ? results.first : results
98
+ end
99
+
100
+ # The capabilities this plan requires: the interface for saying "this
101
+ # graph does not run on :metal" **before running** (#17's value 5).
102
+ #
103
+ # plan.requires
104
+ # # => [#<Requirement capability=:conv ops=[:conv2d] satisfied=true>]
105
+ #
106
+ # compile fails on anything unsatisfiable, so satisfied=false appears
107
+ # here only if capabilities changed afterwards. Ops with a decomposition
108
+ # are expanded into primitives at record time and never appear here.
109
+ Requirement = Struct.new(:capability, :ops, :satisfied) do
110
+ def satisfied? = satisfied
111
+ def to_s = "#{capability} (#{ops.join(', ')})#{satisfied ? '' : ' — missing'}"
112
+ end
113
+
114
+ def requires
115
+ @nodes.filter_map { |node| [node.spec.requires, node.op] if node.spec.requires }
116
+ .group_by(&:first)
117
+ .map { |capability, pairs| Requirement.new(capability, pairs.map(&:last).uniq, Op.satisfied?(capability)) }
118
+ end
119
+
120
+ # How weights were assigned against the VRAM budget (only when
121
+ # compile(vram:) was given). Tells before running whether it fits and
122
+ # how many bytes move per run — see docs/graph-offload.md.
123
+ #
124
+ # plan.residency
125
+ # # => vram 8.00 GB, reserved 1.07 GB, weights 6.93 GB of 13.47 GB,
126
+ # # paged 18 of 34, bytes/run 7.00 GB (floor 6.55 GB)
127
+ def residency = @residency_report
128
+
129
+ # Bytes actually moved and time waited. Compare bytes_in with
130
+ # residency.floor first.
131
+ def offload_stats = @residency&.stats
132
+
133
+ # Deletes the offload files the plan itself wrote (the companion of
134
+ # free_pool!). Files of weights the user offloaded are left alone.
135
+ def free_offload! = @residency&.free_spills!
136
+
137
+ # How many buffers the pool holds and how many allocations / reuses
138
+ # happened — for measuring whether reuse works (allocated stops growing
139
+ # from the second run on).
140
+ def pool_stats = @pool&.stats
141
+
142
+ # Releases the pool's device memory.
143
+ def free_pool! = @pool&.free_all!
144
+
145
+ # In the latest run, how many nodes ran and how many made the host wait
146
+ # (#86's acceptance criterion: observing fewer host synchronizations per
147
+ # dispatch). `synchronous` counts nodes launched with wait: true, `fences`
148
+ # the `Tilework.synchronize` calls added around them.
149
+ #
150
+ # plan.run(x: buf); plan.sync_stats
151
+ # # => {nodes: 42, synchronous: 0, fences: 0}
152
+ def sync_stats = @sync_stats&.dup
153
+
154
+ # A readable listing of the execution order (for debugging). Nodes fused
155
+ # away are never launched, so they are marked as such.
156
+ def explain
157
+ index = -1
158
+ @order.map do |node|
159
+ next " (fused) #{node}" if @absorbed[node.id]
160
+
161
+ index += 1
162
+ suffix = @clusters[node.id] ? " <- fused #{@clusters[node.id].members.length + 1} nodes" : ""
163
+ "#{index}: #{node}#{suffix}"
164
+ end.join("\n")
165
+ end
166
+
167
+ # Fusion results. nodes counts nodes actually executed (including ones
168
+ # like inputs and constants that launch no kernel), so it is not the
169
+ # launch count itself.
170
+ def fusion_stats
171
+ { clusters: @clusters.length, absorbed: @absorbed.length,
172
+ nodes: @order.count { |node| !@absorbed[node.id] } }
173
+ end
174
+
175
+ def inspect
176
+ "#<Tilework::Graph::Plan #{@order.length} nodes, #{states.length} states, " \
177
+ "#{outputs.length} outputs, strict_state: #{@strict_state.inspect}>"
178
+ end
179
+
180
+ private
181
+
182
+ # --- Compile ---
183
+
184
+ # Each state must end in a single final version; forgetting to rejoin
185
+ # forked writes to disjoint regions with g.merge fails here.
186
+ def check_single_leaf!
187
+ @builder.states.each do |st|
188
+ next if st.leaves.length == 1
189
+
190
+ versions = st.leaves.map { |leaf| "v#{leaf.version}" }.join(", ")
191
+ raise Error, "#{st.name} has #{st.leaves.length} unmerged versions (#{versions}); " \
192
+ "disjoint writes must be joined with g.merge before the plan ends"
193
+ end
194
+ end
195
+
196
+ # Type checking (Builder) and whether those types can be allocated on this
197
+ # backend (here) are separate questions. A float16 graph built on :cpu is
198
+ # reported **at compile time**, all at once, instead of failing at the
199
+ # first Buffer.new mid-run — the smallest form of #17's "know portability
200
+ # before running".
201
+ def check_dtypes_supported!
202
+ supported = Tilework.capabilities.dtypes
203
+ missing = @nodes.map(&:dtype).uniq.reject { |dtype| supported.include?(dtype) }
204
+ return if missing.empty?
205
+
206
+ raise Error, "the graph uses #{missing.map(&:inspect).join(', ')} but the " \
207
+ "#{Tilework.backend.inspect} backend can only allocate " \
208
+ "#{supported.map(&:inspect).join(', ')}"
209
+ end
210
+
211
+ # As with types, whatever can be said before running is said before
212
+ # running. Only "the op is missing and has no decomposition" gets here —
213
+ # ops with a decomposition were expanded into primitives at record time.
214
+ def check_requirements!
215
+ missing = requires.reject(&:satisfied?)
216
+ return if missing.empty?
217
+
218
+ raise Error, "the graph needs #{missing.map(&:to_s).join(', ')} but the " \
219
+ "#{Tilework.backend.inspect} backend does not provide it"
220
+ end
221
+
222
+ def reachable(roots)
223
+ seen = {}
224
+ stack = roots.dup
225
+ while (node = stack.pop)
226
+ next if seen.key?(node.id)
227
+
228
+ seen[node.id] = node
229
+ node.inputs.each { |input| stack.push(input) }
230
+ end
231
+ seen.values.sort_by(&:id)
232
+ end
233
+
234
+ # Version edges give only RAW (write, then read), not WAR — a version is
235
+ # **the same memory**, not a new value, so version N+1's write running
236
+ # while version N's readers are still reading breaks things. There is no
237
+ # value edge for that, so it is added here as an ordering edge:
238
+ #
239
+ # a node overwriting version N runs after the readers of version N and their direct consumers.
240
+ #
241
+ # Direct consumers suffice because a read returns a view (zero copy) and
242
+ # each consumer owns its output buffer; once it has run, nothing aliases
243
+ # the state any more.
244
+ def order_dependencies
245
+ deps = Hash.new { |h, k| h[k] = [] }
246
+ consumers = consumer_index
247
+ @nodes.each do |writer|
248
+ next unless writer.op == :state_write
249
+
250
+ version_node = writer.inputs.first
251
+ consumers[version_node.id].each do |reader|
252
+ next unless reader.op == :state_read
253
+ next if reader.equal?(writer)
254
+
255
+ deps[writer.id] << reader.id
256
+ consumers[reader.id].each do |consumer|
257
+ deps[writer.id] << consumer.id unless consumer.equal?(writer)
258
+ end
259
+ end
260
+ end
261
+ deps.each_value(&:uniq!)
262
+ deps
263
+ end
264
+
265
+ def consumer_index
266
+ index = Hash.new { |h, k| h[k] = [] }
267
+ in_plan = @nodes.each_with_object({}) { |n, h| h[n.id] = true }
268
+ @nodes.each do |node|
269
+ node.inputs.each { |input| index[input.id] << node if in_plan[input.id] }
270
+ end
271
+ index
272
+ end
273
+
274
+ # Topological sort over value edges + ordering edges. Ready nodes come out
275
+ # in id order, so the same graph always yields the same order.
276
+ def toposort
277
+ adjacency = Hash.new { |h, k| h[k] = [] }
278
+ indegree = @nodes.to_h { |node| [node.id, 0] }
279
+ by_id = @nodes.to_h { |node| [node.id, node] }
280
+
281
+ @nodes.each do |node|
282
+ (node.inputs.map(&:id) + @order_deps[node.id]).uniq.each do |from|
283
+ adjacency[from] << node.id
284
+ indegree[node.id] += 1
285
+ end
286
+ end
287
+
288
+ ready = indegree.select { |_, d| d.zero? }.keys.sort
289
+ sorted = []
290
+ until ready.empty?
291
+ id = ready.shift
292
+ sorted << by_id[id]
293
+ adjacency[id].each do |to|
294
+ indegree[to] -= 1
295
+ ready = ready.push(to).sort if indegree[to].zero?
296
+ end
297
+ end
298
+ return sorted if sorted.length == @nodes.length
299
+
300
+ raise Error, cycle_message(@nodes.reject { |n| sorted.include?(n) })
301
+ end
302
+
303
+ # In practice a cycle has only one shape: a read of version N used by a
304
+ # node **after** the one overwriting version N. The read is a view, so by
305
+ # then its value has been overwritten; no order resolves it, so fail.
306
+ def cycle_message(stuck)
307
+ states = stuck.select(&:writes_state?).map { |n| n.state.name }.uniq
308
+ detail = states.empty? ? "" : " (#{states.join(', ')})"
309
+ "cannot order the graph#{detail}: a value read from a state version is used after that " \
310
+ "version was overwritten. A read is a view of the same memory, so the value would be " \
311
+ "clobbered — read the newer version instead, or set " \
312
+ "Tilework::Graph.strict_state = :copy to materialize reads.\n" \
313
+ " stuck: #{stuck.map(&:to_s).join("\n ")}"
314
+ end
315
+
316
+ # The fusion plan. Nodes absorbed into a cluster are no longer
317
+ # materialized, so liveness and execution from here on work in terms of
318
+ # **effective_inputs** (inputs coming from outside the cluster).
319
+ def plan_fusion
320
+ return [{}, {}] unless Graph.fusion
321
+
322
+ # Nodes named by ordering edges are not absorbed: that would move "the
323
+ # moment it finished" to the cluster root and reorder it against state
324
+ # writes. @order_deps is a Hash with a default, and toposort leaves empty
325
+ # arrays behind for every node it looked up — consider **only non-empty
326
+ # edges**.
327
+ pinned = @order_deps.flat_map { |writer, readers| readers.empty? ? [] : [writer, *readers] }
328
+ .to_h { |id| [id, true] }
329
+ Fusion.plan(order: @order, output_ids: outputs.map(&:id).to_h { |id| [id, true] },
330
+ consumers: consumer_index, pinned: pinned)
331
+ end
332
+
333
+ # Output placement (#19 §3). `write_into` needs no copy in principle —
334
+ # making the write target the producer's **output** costs nothing. This
335
+ # picks only the nodes where that is safe.
336
+ #
337
+ # Four conditions, each of which breaks things if dropped:
338
+ #
339
+ # 1. the op allocates its output once and fills it. NN, opaque and
340
+ # reductions allocate several inside, so which one is the result is
341
+ # unknowable
342
+ # 2. the write is the only consumer. Placing a value read elsewhere on
343
+ # the state lets a later write corrupt that reader
344
+ # 3. the write carries no ordering (WAR) edge. **The actual memory
345
+ # mutation moves to the value's node**, so the edge would have to move
346
+ # with it, and if it points at that node's descendants it becomes a
347
+ # cycle; only cases needing no move pass
348
+ # 4. the value's node does not read the same state (it would read and
349
+ # write one region at once)
350
+ #
351
+ # 1 is an op property held by the table (Graph::Op); the other three are
352
+ # about the graph's shape, so they are checked here.
353
+ def plan_placement
354
+ return {} unless Graph.placement
355
+
356
+ consumers = consumer_index
357
+ output_ids = outputs.map(&:id).to_h { |id| [id, true] }
358
+ @nodes.select { |node| node.op == :state_write }.filter_map do |writer|
359
+ value = writer.inputs[1]
360
+ next unless placeable?(value, writer, consumers, output_ids)
361
+
362
+ [value.id, writer]
363
+ end.to_h
364
+ end
365
+
366
+ def placeable?(value, writer, consumers, output_ids)
367
+ return false unless value.spec.placeable?(value)
368
+ return false if output_ids.key?(value.id)
369
+ return false unless consumers[value.id].length == 1
370
+ return false unless @order_deps[writer.id].empty?
371
+
372
+ !reads_state?(value, writer.state)
373
+ end
374
+
375
+ # Whether any node touching the same state precedes producing value.
376
+ def reads_state?(value, state)
377
+ seen = {}
378
+ stack = [value]
379
+ while (node = stack.pop)
380
+ next if seen[node.id]
381
+
382
+ seen[node.id] = true
383
+ return true if node.attrs[:state].equal?(state)
384
+
385
+ node.inputs.each { |input| stack.push(input) }
386
+ end
387
+ false
388
+ end
389
+
390
+ # What is known only at run time (dynamic positions, contiguity) is checked here.
391
+ def placement_for(node, bindings)
392
+ writer = @placements[node.id]
393
+ return nil unless writer
394
+
395
+ region = writer.region.resolve(bindings)
396
+ return nil unless region.contiguous?
397
+
398
+ writer.state.buffer.view(region.offset, region.length, shape: node.shape)
399
+ end
400
+
401
+ # reshape and slice own no memory and just return a view of their input.
402
+ # Reading a view reads the original buffer, so lifetime must be counted
403
+ # on **the owning side**, not the view — otherwise memory still being read
404
+ # goes back to the pool and to the next writer.
405
+ ALIAS_OPS = %i[reshape slice].freeze
406
+
407
+ def alias_root(node)
408
+ node = node.inputs.first while ALIAS_OPS.include?(node.op)
409
+ node
410
+ end
411
+
412
+ # Inputs closed inside a cluster disappear, so liveness uses inputs from outside it.
413
+ def effective_inputs(node)
414
+ cluster = @clusters[node.id]
415
+ cluster ? cluster.inputs : node.inputs
416
+ end
417
+
418
+ # Liveness: release intermediates where they are last read. State buffers
419
+ # (and inputs and constants) are not owned, so they are left alone.
420
+ def plan_releases
421
+ releases = Hash.new { |h, k| h[k] = [] }
422
+ executed = @order.reject { |node| @absorbed[node.id] }
423
+ position = executed.each_with_index.to_h { |node, i| [node.id, i] }
424
+ # reshape is a view, so the output is held by the owning side.
425
+ output_ids = outputs.map { |node| alias_root(node).id }
426
+ last = {}
427
+ executed.each do |node|
428
+ effective_inputs(node).each { |input| last[alias_root(input).id] = position[node.id] }
429
+ end
430
+ @nodes.each do |node|
431
+ next unless node.spec.owned?
432
+ next if output_ids.include?(node.id)
433
+ next if @absorbed[node.id] # never materialized, so nothing to release
434
+ next unless last.key?(node.id)
435
+
436
+ releases[last[node.id]] << node.id
437
+ end
438
+ releases
439
+ end
440
+
441
+ # Assigning weights against the budget (#17 Phase 4 / docs/graph-offload.md).
442
+ # Nothing happens without vram: — paging on its own without being given a
443
+ # budget would hide the performance cliff (the same judgment as #17's
444
+ # "no implicit laziness").
445
+ def plan_residency(vram, spill_dir, backing, host_budget)
446
+ return nil if vram.nil? || !Graph.offload
447
+
448
+ unless %i[file host].include?(backing)
449
+ raise ArgumentError, "backing: must be :file or :host"
450
+ end
451
+ if backing == :host
452
+ raise Error, "pageable host backing is not available on the active backend" unless Tilework.capabilities[:host_backing]
453
+ raise ArgumentError, "host_budget: must be a positive Integer of bytes" unless host_budget.is_a?(Integer) && host_budget.positive?
454
+ end
455
+
456
+ raise ArgumentError, "vram: must be a positive Integer of bytes" unless vram.is_a?(Integer) && vram.positive?
457
+
458
+ executed = @order.reject { |node| @absorbed[node.id] }
459
+ positions = executed.each_with_index.to_h { |node, i| [node.id, i] }
460
+ candidates, pinned = split_consts
461
+ reserved = reserved_bytes(executed) + pinned.sum { |node| bytes_of(node) }
462
+ budget = vram - reserved
463
+ weights = candidates.sum { |node| bytes_of(node) }
464
+ if backing == :host && weights > host_budget
465
+ raise Error, "host backing budget #{host_budget} bytes cannot hold pageable candidates totaling #{weights} bytes"
466
+ end
467
+
468
+ entries = residency_entries(candidates, executed, positions)
469
+ needs = residency_needs(entries, executed, positions)
470
+ check_budget!(entries, needs, budget, vram, reserved, weights)
471
+ # Report even when it fits (being able to say "it fits" is worth something).
472
+ @residency_report = residency_report(vram, reserved, budget, weights, entries, pinned, executed, needs)
473
+ # A loose budget is free — if it fits, the machinery does not exist.
474
+ return nil if budget >= weights
475
+
476
+ Residency.new(entries: entries, needs: needs, steps: executed.length, budget: budget,
477
+ spill_dir: spill_dir || default_spill_dir, backing: backing, report: @residency_report)
478
+ end
479
+
480
+ # Splits consts into pageable and pinned. Two kinds cannot move: views
481
+ # (restore! allocates a new handle, leaving views of the old one dangling;
482
+ # buffer.rb's reject_view! refuses them too) and plan outputs (they must
483
+ # stay resident after run).
484
+ def split_consts
485
+ output_roots = outputs.map { |node| alias_root(node).id }.to_h { |id| [id, true] }
486
+ consts = @nodes.select { |node| node.op == :const && !@absorbed[node.id] }
487
+ .reject { |node| node.attrs[:buffer].freed? }
488
+ consts.partition do |node|
489
+ !node.attrs[:buffer].view? && !output_roots[node.id]
490
+ end
491
+ end
492
+
493
+ def residency_entries(candidates, executed, positions)
494
+ uses = Hash.new { |h, k| h[k] = [] }
495
+ executed.each do |node|
496
+ const_inputs(node).each { |const| uses[const.id] << positions[node.id] }
497
+ end
498
+ candidates.filter_map do |node|
499
+ used = uses[node.id].uniq.sort
500
+ next if used.empty?
501
+
502
+ buffer = node.attrs[:buffer]
503
+ Residency::Entry.new(node.id, buffer, bytes_of(node), used, !buffer.resident?, false)
504
+ end
505
+ end
506
+
507
+ def residency_needs(entries, executed, positions)
508
+ by_node = entries.to_h { |entry| [entry.node_id, entry] }
509
+ needs = Hash.new { |h, k| h[k] = [] }
510
+ executed.each do |node|
511
+ const_inputs(node).each do |const|
512
+ entry = by_node[const.id]
513
+ needs[positions[node.id]] << entry if entry
514
+ end
515
+ end
516
+ needs.each_value(&:uniq!)
517
+ needs
518
+ end
519
+
520
+ # Consts that must be resident when this node runs. Inputs fused away have
521
+ # become the cluster's inputs, so effective_inputs is consulted. reshape /
522
+ # slice create views (Buffer#view requires residency), so they appear here
523
+ # as direct inputs and count as a requirement at that position too.
524
+ def const_inputs(node)
525
+ effective_inputs(node).flat_map { |input| [input, alias_root(input)] }
526
+ .select { |input| input.op == :const }.uniq
527
+ end
528
+
529
+ # The part of the budget weights cannot use. Inputs/outputs and state
530
+ # cannot move, and the pool keeps intermediates, so those count at peak.
531
+ def reserved_bytes(executed)
532
+ io = @builder.input_nodes.each_value.sum { |node| bytes_of(node) } +
533
+ outputs.sum { |node| bytes_of(alias_root(node)) }
534
+ state = states.sum { |st| Residency.bytes_of(st.dtype, st.buffer.length) }
535
+ io + state + peak_intermediate_bytes(executed)
536
+ end
537
+
538
+ # Peak intermediates. The pool keeps slots across runs, so this is
539
+ # steady-state occupancy, not the first run's. Liveness (@release_at)
540
+ # already knows the release points, so this walks the same table in bytes.
541
+ def peak_intermediate_bytes(executed)
542
+ output_ids = outputs.map { |node| alias_root(node).id }.to_h { |id| [id, true] }
543
+ live = 0
544
+ peak = 0
545
+ executed.each_with_index do |node, index|
546
+ live += bytes_of(node) if node.spec.owned? && !output_ids[node.id]
547
+ peak = live if live > peak
548
+ @release_at[index].each do |id|
549
+ released = @nodes.find { |n| n.id == id }
550
+ live -= bytes_of(released) if released && !output_ids[id]
551
+ end
552
+ end
553
+ peak
554
+ end
555
+
556
+ # Fail before running whatever can be (like dtype / requires). Two
557
+ # checks: no room at all, or one node's simultaneous weights do not fit.
558
+ # The latter is "enough overall, but this one line does not pass", so the
559
+ # node is named.
560
+ def check_budget!(entries, needs, budget, vram, reserved, weights)
561
+ largest = entries.map(&:bytes).max || 0
562
+ if budget < largest
563
+ raise Error, "the graph reserves #{Residency.human(reserved)} for inputs, state and " \
564
+ "intermediates out of a #{Residency.human(vram)} budget, leaving " \
565
+ "#{Residency.human(budget)} for #{Residency.human(weights)} of weights; " \
566
+ "the largest single weight is #{Residency.human(largest)}"
567
+ end
568
+
569
+ needs.each do |index, required|
570
+ want = required.sum(&:bytes)
571
+ next if want <= budget
572
+
573
+ node = @order.reject { |n| @absorbed[n.id] }[index]
574
+ raise Error, "step #{index} (#{node}) needs #{Residency.human(want)} of weights at once " \
575
+ "but only #{Residency.human(budget)} is left after reserving " \
576
+ "#{Residency.human(reserved)} of a #{Residency.human(vram)} budget"
577
+ end
578
+ end
579
+
580
+ def residency_report(vram, reserved, budget, weights, entries, pinned, executed, needs)
581
+ # Bytes moved per run. At most budget carries over, so the difference
582
+ # moves under any policy (= floor). Cyclic Belady lands within one slot
583
+ # of it.
584
+ floor = [weights - budget, 0].max
585
+ # In practice one slot more moves — a sequential scan needs one slot for
586
+ # the eviction victim to rotate through. offload_stats[:bytes_in] is the
587
+ # measurement.
588
+ rotating = floor.positive? ? (entries.map(&:bytes).max || 0) : 0
589
+ Residency::Report.new(
590
+ vram: vram, reserved: reserved, budget: budget, weights: weights,
591
+ paged: entries.length, resident_always: pinned.length,
592
+ resident_always_bytes: pinned.sum { |node| bytes_of(node) },
593
+ prefetch_headroom: Residency.prefetch_ceiling(needs, budget),
594
+ bytes_per_run: floor + rotating, floor: floor, steps: executed.length,
595
+ )
596
+ end
597
+
598
+ def bytes_of(node) = Residency.bytes_of(node.dtype, node.length)
599
+
600
+ def default_spill_dir
601
+ require "tmpdir"
602
+ File.join(Dir.tmpdir, "tilework-offload-#{Process.pid}-#{object_id}")
603
+ end
604
+
605
+ # --- Execution ---
606
+
607
+ # Returns last-read intermediates to the pool (frees them without one).
608
+ # Views own nothing, so they never enter the pool.
609
+ def release(index, values, output_ids)
610
+ @release_at[index].each do |id|
611
+ buffer = values[id]
612
+ next unless buffer.is_a?(Buffer) && !output_ids.include?(id)
613
+
614
+ values[id] = nil
615
+ if @pool && !buffer.view?
616
+ if @pool.owns?(buffer)
617
+ @pool.release(buffer)
618
+ elsif @nodes_by_id.fetch(id).op == :opaque
619
+ # opaque may return a value the user made with Buffer.new. If
620
+ # fenced, the node ran synchronously and it can be freed on the
621
+ # spot; with fenced: false launches may still be running, so it
622
+ # is adopted into the pool instead of freed.
623
+ node = @nodes_by_id.fetch(id)
624
+ if node.attrs[:fenced]
625
+ buffer.free
626
+ else
627
+ @pool.release(@pool.adopt(buffer))
628
+ end
629
+ else
630
+ # An ownership violation that should be unreachable. In case the op
631
+ # was asynchronous, wait before freeing so the diagnostic exception
632
+ # does not leave a new leak behind.
633
+ Tilework.synchronize
634
+ buffer.free
635
+ raise Error, "graph node #{@nodes_by_id.fetch(id).op.inspect} returned an intermediate " \
636
+ "buffer that did not come from Ops.allocate_output"
637
+ end
638
+ else
639
+ buffer.free
640
+ end
641
+ end
642
+ end
643
+
644
+ # After a synchronous node the queue is empty; tell residency, so the next
645
+ # eviction can ride on that wait (#17 Phase 4c).
646
+ def note_queue_state(node)
647
+ return unless @residency
648
+ # Nodes that launch nothing (:const / :input / views) leave the queue as
649
+ # it was. Counting them as dirtying it would mean the :const right
650
+ # before every weight-reading matmul prevents any ride-along.
651
+ return unless node.spec.launches?
652
+
653
+ @residency.note_queue(dirty: !Graph.offload_fence_reuse || !drains_queue?(node))
654
+ end
655
+
656
+ # Whether the queue is empty once this node finishes. Synchronous nodes
657
+ # (:nn / whole-buffer reductions / delegated GEMM / opaque) launch with
658
+ # wait: true, so earlier work on the same queue has completed too.
659
+ # With asynchronous execution disabled, every node drains the queue.
660
+ def drains_queue?(node) = !@async || node.spec.synchronous?(node)
661
+
662
+ # Nodes that can stay asynchronous do; nodes that (may) leave our
663
+ # execution path wait on both sides and run synchronously.
664
+ def execute_node(node, values, bindings)
665
+ target = placement_for(node, bindings)
666
+ return dispatch_node(node, values, bindings) unless target
667
+
668
+ result = @context.place(target) { dispatch_node(node, values, bindings) }
669
+ @placed[node.id] = @context.placement_consumed?
670
+ result
671
+ end
672
+
673
+ def dispatch_node(node, values, bindings)
674
+ @sync_stats[:nodes] += 1
675
+ return execute(node, values, bindings) unless synchronous?(node)
676
+
677
+ @sync_stats[:synchronous] += 1
678
+ foreign = foreign?(node)
679
+ if foreign
680
+ @sync_stats[:fences] += 2
681
+ Tilework.synchronize
682
+ end
683
+ result = @context.synchronous { execute(node, values, bindings) }
684
+ Tilework.synchronize if foreign
685
+ result
686
+ end
687
+
688
+ # Whether to run synchronously or wait is an op property (including
689
+ # batching and delegation). Making everything synchronous only when
690
+ # @async is off is this method's job.
691
+ def synchronous?(node) = @async && node.spec.synchronous?(node)
692
+
693
+ def foreign?(node) = node.spec.foreign?(node)
694
+
695
+ # The table holds each op's execution method. All take the same
696
+ # (node, inputs, bindings), so no branching is needed here.
697
+ def execute(node, values, bindings)
698
+ cluster = @clusters[node.id]
699
+ return run_fused(cluster, cluster.inputs.map { |input| values[input.id] }) if cluster
700
+
701
+ inputs = node.inputs.map { |input| values[input.id] }
702
+ send(node.spec.run, node, inputs, bindings)
703
+ end
704
+
705
+ def run_const(node, _inputs, _bindings) = node.attrs[:buffer]
706
+ def run_state_init(node, _inputs, _bindings) = node.state.buffer
707
+ def run_state_merge(node, _inputs, _bindings) = node.state.buffer
708
+
709
+ def run_input(node, _inputs, bindings)
710
+ name = node.attrs[:name]
711
+ buffer = bindings.fetch(name) do
712
+ raise Error, "plan.run is missing input #{name.inspect} (expects #{input_names.inspect})"
713
+ end
714
+ raise TypeError, "input #{name.inspect} must be a Tilework::Buffer, got #{buffer.class}" unless buffer.is_a?(Buffer)
715
+ unless buffer.dtype == node.dtype && buffer.shape == node.shape
716
+ raise Error, "input #{name.inspect} was declared as #{node.shape.inspect} #{node.dtype} " \
717
+ "but got #{buffer.shape.inspect} #{buffer.dtype}"
718
+ end
719
+
720
+ buffer
721
+ end
722
+
723
+ # A read is a view by default = no kernel. It is materialized when
724
+ # - it is a plan output (so later writes do not corrupt it)
725
+ # - strict_state == :copy (for comparison with the pure semantics)
726
+ # - the region is not contiguous (a view cannot express it)
727
+ def run_state_read(node, _inputs, bindings)
728
+ region = node.region.resolve(bindings)
729
+ buffer = node.state.buffer
730
+ materialize = @strict_state == :copy || node.attrs[:materialize] || !region.contiguous?
731
+ unless materialize
732
+ return buffer.view(region.offset, region.length, shape: region.shape)
733
+ end
734
+
735
+ out = Ops.allocate_output(node.dtype, region.length, region.shape)
736
+ region.each_block do |src_offset, dst_offset, len|
737
+ Graph.copy(buffer.view(src_offset, len), out.view(dst_offset, len))
738
+ end
739
+ out
740
+ end
741
+
742
+ def run_state_write(node, inputs, bindings)
743
+ value = inputs[1]
744
+ # If the producer already wrote straight into the target (placement
745
+ # took effect), there is nothing left to copy.
746
+ return node.state.buffer if @placed[node.inputs[1].id]
747
+
748
+ region = node.region.resolve(bindings)
749
+ buffer = node.state.buffer
750
+ region.each_block do |dst_offset, src_offset, len|
751
+ source = value.length == len ? value : value.view(src_offset, len)
752
+ Graph.copy(source, buffer.view(dst_offset, len))
753
+ end
754
+ buffer
755
+ end
756
+
757
+ def run_binary(node, inputs, _bindings)
758
+ op = node.attrs[:op]
759
+ operand = node.attrs.key?(:scalar) ? node.attrs[:scalar] : inputs[1]
760
+ case node.attrs[:kind]
761
+ when :compare then Ops.compare(op, inputs[0], operand)
762
+ else Ops.arithmetic(op, inputs[0], operand, reversed: node.attrs[:reversed] || false)
763
+ end
764
+ end
765
+
766
+ def run_unary(node, inputs, _bindings)
767
+ fn = node.attrs[:fn]
768
+ fn == :abs ? Ops.unary_abs(inputs[0]) : Ops.unary_math(fn, inputs[0])
769
+ end
770
+
771
+ # A whole-buffer reduction takes Ops' device: true path (tree folding). It
772
+ # never reaches the host, so the result passes on as a one-element Buffer.
773
+ def run_reduce(node, inputs, _bindings)
774
+ kind = node.attrs[:kind]
775
+ return launch_axis_reduce(node, inputs[0]) unless node.attrs[:axis].nil?
776
+ return Ops.mean_op(inputs[0], device: true) if kind == :mean
777
+
778
+ Ops.reduce(kind, inputs[0], device: true)
779
+ end
780
+
781
+ def run_arg_reduce(node, inputs, _bindings)
782
+ return launch_axis_reduce(node, inputs[0]) unless node.attrs[:axis].nil?
783
+
784
+ Ops.arg_reduce(node.attrs[:kind], inputs[0], device: true)
785
+ end
786
+
787
+ # An axis reduction is one kernel. The input is viewed as
788
+ # [outer, axis_len, inner], one thread per output element folding axis_len
789
+ # values. outer and inner are the products of the dims before and after
790
+ # the axis, so any rank works the same.
791
+ def launch_axis_reduce(node, input)
792
+ shape = node.inputs.first.shape
793
+ axis = node.attrs[:axis]
794
+ axis_len = shape[axis]
795
+ inner = shape.drop(axis + 1).inject(1, :*)
796
+ kernel =
797
+ if node.op == :arg_reduce
798
+ Kernels.axis_arg_reduce(kind: node.attrs[:kind], in_dtype: input.dtype)
799
+ else
800
+ Kernels.axis_reduce(kind: node.attrs[:kind], in_dtype: input.dtype, out_dtype: node.dtype)
801
+ end
802
+
803
+ out = Ops.allocate_output(node.dtype, node.length, node.shape)
804
+ Ops.launch_kernel(kernel, grid: (node.length / Ops::BLOCK.to_f).ceil, block: Ops::BLOCK,
805
+ args: [flat(input), out, axis_len, inner, node.length])
806
+ out
807
+ end
808
+
809
+ def run_cast(node, inputs, _bindings)
810
+ kernel = Kernels.cast(from: node.inputs.first.dtype, to: node.dtype)
811
+ launch_elementwise(node, kernel, [flat(inputs[0])])
812
+ end
813
+
814
+ # Arguments are always [cond, a, b, out, n]; the side that is a scalar is
815
+ # passed as a value (which one is fixed at compile time).
816
+ def run_where(node, inputs, _bindings)
817
+ a_scalar = node.attrs[:a_scalar]
818
+ b_scalar = node.attrs[:b_scalar]
819
+ kernel = Kernels.select(dtype: node.dtype, a_scalar: !a_scalar.nil?, b_scalar: !b_scalar.nil?)
820
+ rest = inputs.drop(1).map { |buffer| flat(buffer) }
821
+ a = a_scalar || rest.shift
822
+ b = b_scalar || rest.shift
823
+ launch_elementwise(node, kernel, [flat(inputs[0]), a, b])
824
+ end
825
+
826
+ def run_iota(node, _inputs, _bindings)
827
+ kernel = Kernels.iota(shape: node.shape, axis: node.attrs[:axis], dtype: node.dtype)
828
+ launch_elementwise(node, kernel, [])
829
+ end
830
+
831
+ # Only changes the shape. **No kernel runs**; a view of the input buffer is
832
+ # returned. The view points at the original memory, so the original must
833
+ # stay alive until this node's consumers finish (alias_root in plan_releases).
834
+ def run_reshape(node, inputs, _bindings)
835
+ inputs[0].reshape(node.shape)
836
+ end
837
+
838
+ # One thread per output element; shared by the three above and broadcast.
839
+ def launch_elementwise(node, kernel, args)
840
+ out = Ops.allocate_output(node.dtype, node.length, node.shape)
841
+ Ops.launch_kernel(kernel, grid: (node.length / Ops::BLOCK.to_f).ceil, block: Ops::BLOCK,
842
+ args: [*args, out, node.length])
843
+ out
844
+ end
845
+
846
+ # Materializes an index remap (broadcast / permute). Only reached when
847
+ # fusion did not fold it.
848
+ def run_remap(node, inputs, _bindings)
849
+ kernel = Kernels.remap(dims: Kernels.remap_dims(node), dtype: node.dtype, name: node.op.to_s)
850
+ launch_elementwise(node, kernel, [flat(inputs[0])])
851
+ end
852
+
853
+ # An outermost-dimension slice. A view like reshape, so no kernel runs.
854
+ def run_slice(node, inputs, _bindings)
855
+ inputs[0].view(node.attrs[:offset], node.length, shape: node.shape)
856
+ end
857
+
858
+ def run_gather(node, inputs, _bindings)
859
+ source = node.inputs.first
860
+ axis = node.attrs[:axis]
861
+ inner = source.shape.drop(axis + 1).inject(1, :*)
862
+ kernel = Kernels.gather(dtype: node.dtype, axis_len: source.shape[axis],
863
+ idx_len: node.inputs[1].length, inner: inner)
864
+ launch_elementwise(node, kernel, [flat(inputs[0]), flat(inputs[1])])
865
+ end
866
+
867
+ # The inverse of gather: one thread per output element scans the index
868
+ # list (see the note on Kernels.scatter_add).
869
+ def run_scatter_add(node, inputs, _bindings)
870
+ axis = node.attrs[:axis]
871
+ inner = node.shape.drop(axis + 1).inject(1, :*)
872
+ kernel = Kernels.scatter_add(dtype: node.dtype, axis_len: node.shape[axis],
873
+ idx_len: node.inputs[0].length, inner: inner)
874
+ launch_elementwise(node, kernel, [flat(inputs[0]), flat(inputs[1])])
875
+ end
876
+
877
+ def run_pad(node, inputs, _bindings)
878
+ source = node.inputs.first
879
+ strides = Kernels.row_major_strides(source.shape)
880
+ dims = node.shape.each_with_index.map do |dim, d|
881
+ [dim, source.shape[d], node.attrs[:before][d], strides[d]]
882
+ end
883
+ kernel = Kernels.pad(dims: dims, dtype: node.dtype, value: node.attrs[:value])
884
+ out = Ops.allocate_output(node.dtype, node.length, node.shape)
885
+ Ops.launch_kernel(kernel, grid: (node.length / Ops::BLOCK.to_f).ceil, block: Ops::BLOCK,
886
+ args: [flat(inputs[0]), out, node.attrs[:value], node.length])
887
+ out
888
+ end
889
+
890
+ # Kernels see indices as 1D, so multidimensional buffers are passed
891
+ # flattened (as in Graph.copy).
892
+ def flat(buffer) = buffer.shape.length > 1 ? buffer.reshape(buffer.length) : buffer
893
+
894
+ # Runs a fused cluster as one kernel. The output takes the same two forms
895
+ # as everything else (dialect source / :cpu Ruby source), so it rides
896
+ # Kernel's SHA cache as is — clusters with the same expression compile
897
+ # once even across layers.
898
+ def run_fused(cluster, inputs)
899
+ return run_row_fused(cluster, inputs) if cluster.row_wise?
900
+
901
+ kernel = Ops.compiled(
902
+ dialect_name: "fused", dialect_src: Fusion.dialect_source(cluster),
903
+ cpu_name: "fused",
904
+ cpu_src: lambda {
905
+ Ops.codegen.fused(cluster.expr,
906
+ input_dtypes: cluster.inputs.map(&:dtype),
907
+ scalar_dtypes: cluster.scalars.map { |s| s[:dtype] },
908
+ out_dtype: cluster.dtype)
909
+ },
910
+ )
911
+ out = Ops.allocate_output(cluster.dtype, cluster.length, cluster.shape)
912
+ args = inputs + cluster.scalars.map { |s| s[:value] } + [out, cluster.length]
913
+ Ops.launch_kernel(kernel, grid: (cluster.length / Ops::BLOCK.to_f).ceil, block: Ops::BLOCK, args: args)
914
+ out
915
+ end
916
+
917
+ # A row-wise cluster. **The launch domain is rows, not elements** (one
918
+ # thread per row); only the launch differs from elementwise clusters.
919
+ def run_row_fused(cluster, inputs)
920
+ kernel = Ops.compiled(
921
+ dialect_name: "row_fused", dialect_src: Fusion.dialect_source(cluster),
922
+ cpu_name: "row_fused",
923
+ cpu_src: lambda {
924
+ Ops.codegen.row_fused(cluster.expr, reductions: cluster.reductions,
925
+ input_dtypes: cluster.inputs.map(&:dtype),
926
+ scalar_dtypes: cluster.scalars.map { |s| s[:dtype] },
927
+ out_dtype: cluster.dtype)
928
+ },
929
+ )
930
+ out = Ops.allocate_output(cluster.dtype, cluster.length, cluster.shape)
931
+ args = inputs + cluster.scalars.map { |s| s[:value] } + [out, cluster.rows, cluster.cols]
932
+ Ops.launch_kernel(kernel, grid: (cluster.rows / Ops::BLOCK.to_f).ceil, block: Ops::BLOCK, args: args)
933
+ out
934
+ end
935
+
936
+ def run_matmul(node, inputs, _bindings)
937
+ unless node.attrs[:mode] == :batched
938
+ return Ops.matmul_op(inputs[0], inputs[1], out_dtype: node.attrs.fetch(:out_dtype, node.dtype))
939
+ end
940
+
941
+ run_batched_matmul(node, inputs[0], inputs[1])
942
+ end
943
+
944
+ # [b, m, k] x [b, k, n]; a strided batched GEMM does it in one call.
945
+ #
946
+ # The bridge's GEMM is **column-major** (see tensor_core.rb), so row-major
947
+ # OUT = A·B is reinterpreted transposed: OUT^T = B^T · A^T, i.e. swap A
948
+ # and B and swap m and n as column-major arguments. Each ld is its
949
+ # row-major width, which is exactly the column-major leading dimension.
950
+ def run_batched_matmul(node, a, b)
951
+ batch, m, k = a.shape
952
+ n = b.shape[2]
953
+ compute_dtype = node.dtype == :int32 ? :int32 : :float32
954
+ compute_out = Ops.allocate_output(compute_dtype, node.length, node.shape)
955
+ if batched_gemm_usable?(node, a, b)
956
+ TensorCore.gemm_batched(b, a, compute_out,
957
+ m: n, k: k, n: m,
958
+ lda: n, ldb: k, ldc: n,
959
+ stride_a: k * n, stride_b: m * k, stride_c: m * n,
960
+ batch: batch)
961
+ return convert_batched_output(compute_out, node.dtype)
962
+ end
963
+
964
+ # Without a delegate, fall back to per-batch matmuls. The output is
965
+ # allocated here, so one extra copy per batch moves each result into place.
966
+ batch.times do |i|
967
+ part = Ops.matmul_op(a.slice(i).reshape(m, k), b.slice(i).reshape(k, n))
968
+ Graph.copy(part, compute_out.view(i * m * n, m * n))
969
+ part.free
970
+ end
971
+ convert_batched_output(compute_out, node.dtype)
972
+ end
973
+
974
+ def convert_batched_output(out, dtype)
975
+ return out if out.dtype == dtype
976
+
977
+ converted = out.cast(dtype)
978
+ out.free
979
+ converted
980
+ end
981
+
982
+ # gemm_batched applies only when both sides share a float type and the output is float32.
983
+ def batched_gemm_usable?(_node, a, b)
984
+ a.dtype == b.dtype && a.dtype != :int32 && TensorCore.gemm_batched? &&
985
+ Tilework.capabilities.gemm_dtypes.include?(a.dtype)
986
+ end
987
+
988
+ # concat accepts only the outermost dim, so each piece is contiguous in
989
+ # the output; one copy per piece suffices, with no scattered writes.
990
+ def run_concat(node, inputs, _bindings)
991
+ out = Ops.allocate_output(node.dtype, node.length, node.shape)
992
+ axis = node.attrs[:axis] || 0
993
+ return concat_contiguous(out, inputs) if axis.zero?
994
+
995
+ inner = node.shape.drop(axis + 1).inject(1, :*)
996
+ offset = 0
997
+ node.inputs.each_with_index do |piece, i|
998
+ kernel = Kernels.concat_piece(dtype: node.dtype, piece_axis: piece.shape[axis],
999
+ total_axis: node.shape[axis], inner: inner, offset: offset)
1000
+ Ops.launch_kernel(kernel, grid: (piece.length / Ops::BLOCK.to_f).ceil, block: Ops::BLOCK,
1001
+ args: [flat(inputs[i]), out, piece.length])
1002
+ offset += piece.shape[axis]
1003
+ end
1004
+ out
1005
+ end
1006
+
1007
+ # Along the outermost dim pieces are contiguous in the output, so one copy each suffices.
1008
+ def concat_contiguous(out, inputs)
1009
+ offset = 0
1010
+ inputs.each do |buffer|
1011
+ Graph.copy(buffer, out.view(offset, buffer.length))
1012
+ offset += buffer.length
1013
+ end
1014
+ out
1015
+ end
1016
+
1017
+ # Derives each node's key from (plan seed, static index). Philox is
1018
+ # counter-based and takes its key as an argument, so this alone makes RNG
1019
+ # **pure** — independent of execution order, same seed same sequence.
1020
+ def run_random(node, _inputs, bindings)
1021
+ seed = bindings.fetch(:seed) do
1022
+ raise Error, "plan.run needs a seed: this graph draws #{@builder.random_draws} random value(s)"
1023
+ end
1024
+ raise TypeError, "seed must be an Integer, got #{seed.class}" unless seed.is_a?(Integer)
1025
+
1026
+ out = Ops.allocate_output(node.dtype, node.length, node.shape)
1027
+ key = Plan.derive_key(seed, node.attrs[:index])
1028
+ if node.attrs[:kind] == :normal
1029
+ Random.normal!(out, mean: node.attrs[:mean], stddev: node.attrs[:stddev], seed: key)
1030
+ else
1031
+ Random.uniform!(out, seed: key)
1032
+ end
1033
+ out
1034
+ end
1035
+
1036
+ # Calls the same NN function that was recorded. There is no graph-specific
1037
+ # implementation, so values cannot drift from eager (two diverging copies
1038
+ # of the math is the biggest risk #17 named).
1039
+ def run_nn(node, inputs, _bindings)
1040
+ arity = node.attrs[:arity]
1041
+ kwargs = node.attrs[:opts].merge(node.attrs[:keys].zip(inputs.drop(arity)).to_h)
1042
+ result = NN.public_send(node.attrs[:fn], *inputs.first(arity), *node.attrs[:args], **kwargs)
1043
+ check_result!(result, node, "#{node.attrs[:fn]} node")
1044
+ end
1045
+
1046
+ # Calls the delegate (cuDNN / MPSGraph) as is. No graph-specific
1047
+ # implementation, so values cannot drift from eager (same idea as :nn).
1048
+ def run_conv2d(node, inputs, _bindings)
1049
+ result = Conv.forward(inputs[0], inputs[1], pad: node.attrs[:pad], stride: node.attrs[:stride],
1050
+ dilation: node.attrs[:dilation], groups: node.attrs[:groups])
1051
+ check_result!(result, node, "conv2d node")
1052
+ end
1053
+
1054
+ # Backward calls the delegate as is too (same idea as run_conv2d — no
1055
+ # graph-specific implementation, so no drift from eager). inputs[2] is
1056
+ # there only for its shape.
1057
+ def run_conv2d_backward_data(node, inputs, _bindings)
1058
+ result = Conv.backward_data(inputs[0], inputs[1], inputs[2],
1059
+ pad: node.attrs[:pad], stride: node.attrs[:stride],
1060
+ dilation: node.attrs[:dilation], groups: node.attrs[:groups])
1061
+ check_result!(result, node, "conv2d backward data node")
1062
+ end
1063
+
1064
+ def run_conv2d_backward_filter(node, inputs, _bindings)
1065
+ result = Conv.backward_filter(inputs[0], inputs[1], inputs[2],
1066
+ pad: node.attrs[:pad], stride: node.attrs[:stride],
1067
+ dilation: node.attrs[:dilation], groups: node.attrs[:groups])
1068
+ check_result!(result, node, "conv2d backward filter node")
1069
+ end
1070
+
1071
+ def run_opaque(node, inputs, _bindings)
1072
+ result = node.attrs[:body].call(*inputs)
1073
+ check_result!(result, node, "opaque node #{node.attrs[:name].inspect}")
1074
+ end
1075
+
1076
+ def check_result!(result, node, what)
1077
+ unless result.is_a?(Buffer) && result.dtype == node.dtype && result.shape == node.shape
1078
+ got = result.is_a?(Buffer) ? "#{result.shape.inspect} #{result.dtype}" : result.class.to_s
1079
+ raise Error, "#{what} declared #{node.shape.inspect} #{node.dtype} but returned #{got}"
1080
+ end
1081
+
1082
+ result
1083
+ end
1084
+ end
1085
+ end
1086
+ end