tilework 0.18.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE +21 -0
- data/lib/tilework/backend/cpu/dsl_codegen.rb +509 -0
- data/lib/tilework/backend/cpu/ops_codegen.rb +451 -0
- data/lib/tilework/backend/cpu.rb +156 -0
- data/lib/tilework/buffer.rb +753 -0
- data/lib/tilework/capabilities.rb +183 -0
- data/lib/tilework/compat/cuda.h +66 -0
- data/lib/tilework/compat/hip.h +35 -0
- data/lib/tilework/compat/metal.h +45 -0
- data/lib/tilework/conformance.rb +2296 -0
- data/lib/tilework/core_ext.rb +21 -0
- data/lib/tilework/dialect.rb +67 -0
- data/lib/tilework/dsl.rb +722 -0
- data/lib/tilework/dtype/bfloat.rb +31 -0
- data/lib/tilework/dtype/cast.rb +88 -0
- data/lib/tilework/dtype/half.rb +98 -0
- data/lib/tilework/graph/builder.rb +1073 -0
- data/lib/tilework/graph/fusion.rb +520 -0
- data/lib/tilework/graph/grad.rb +443 -0
- data/lib/tilework/graph/high_level.rb +266 -0
- data/lib/tilework/graph/kernels.rb +403 -0
- data/lib/tilework/graph/node.rb +27 -0
- data/lib/tilework/graph/op.rb +290 -0
- data/lib/tilework/graph/optim.rb +183 -0
- data/lib/tilework/graph/plan.rb +1086 -0
- data/lib/tilework/graph/pool.rb +178 -0
- data/lib/tilework/graph/region.rb +210 -0
- data/lib/tilework/graph/residency.rb +449 -0
- data/lib/tilework/graph/state.rb +64 -0
- data/lib/tilework/graph/tensor.rb +124 -0
- data/lib/tilework/graph.rb +213 -0
- data/lib/tilework/kernel.rb +163 -0
- data/lib/tilework/math/conv.rb +270 -0
- data/lib/tilework/math/fft.rb +172 -0
- data/lib/tilework/math/nn.rb +1103 -0
- data/lib/tilework/math/philox.rb +194 -0
- data/lib/tilework/math/random.rb +232 -0
- data/lib/tilework/math/tensor_core.rb +269 -0
- data/lib/tilework/ops.rb +682 -0
- data/lib/tilework/runtime/offload.rb +498 -0
- data/lib/tilework/runtime/submission.rb +163 -0
- data/lib/tilework/runtime/telemetry.rb +112 -0
- data/lib/tilework/scope.rb +33 -0
- data/lib/tilework/version.rb +5 -0
- data/lib/tilework.rb +345 -0
- metadata +92 -0
|
@@ -0,0 +1,1086 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Tilework
|
|
4
|
+
module Graph
|
|
5
|
+
# An executable graph with a fixed operation order and buffer lifetime plan.
|
|
6
|
+
#
|
|
7
|
+
# Nodes run through the existing eager layer (Ops), fused where possible.
|
|
8
|
+
# What compile fixes is **order** and **lifetime**:
|
|
9
|
+
#
|
|
10
|
+
# 1. topologically sort after adding mutable state's WAR edges to the value (RAW) edges
|
|
11
|
+
# 2. release intermediates at their last read (state buffers excluded)
|
|
12
|
+
class Plan
|
|
13
|
+
# Owning memory, running synchronously, leaving our execution path and
|
|
14
|
+
# accepting a placed output are all op properties, held by the Graph::Op
|
|
15
|
+
# table. This class consults it and concentrates on order and lifetime.
|
|
16
|
+
attr_reader :order, :nodes, :strict_state, :async
|
|
17
|
+
|
|
18
|
+
def initialize(builder, vram: nil, spill_dir: nil, backing: :file, host_budget: nil)
|
|
19
|
+
@builder = builder
|
|
20
|
+
# The mode is fixed at compile time (so its meaning cannot change between runs).
|
|
21
|
+
@strict_state = Graph.strict_state
|
|
22
|
+
@async = Graph.async
|
|
23
|
+
@pool = Graph.buffer_pool ? Pool.new : nil
|
|
24
|
+
@context = Context.new(pool: @pool, async: @async)
|
|
25
|
+
check_single_leaf!
|
|
26
|
+
@roots = builder.outputs + builder.states.map { |st| st.leaves.first.node }
|
|
27
|
+
@nodes = reachable(@roots)
|
|
28
|
+
@nodes_by_id = @nodes.to_h { |node| [node.id, node] }
|
|
29
|
+
check_dtypes_supported!
|
|
30
|
+
check_requirements!
|
|
31
|
+
@order_deps = order_dependencies
|
|
32
|
+
@order = toposort
|
|
33
|
+
@clusters, @absorbed = plan_fusion
|
|
34
|
+
@placements = plan_placement
|
|
35
|
+
@release_at = plan_releases
|
|
36
|
+
@residency = plan_residency(vram, spill_dir, backing, host_budget)
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def input_names = @builder.input_nodes.keys
|
|
40
|
+
def states = @builder.states
|
|
41
|
+
def outputs = @builder.outputs
|
|
42
|
+
|
|
43
|
+
# How many random draws this plan makes; 0 means no seed is needed.
|
|
44
|
+
def random_draws = @builder.random_draws
|
|
45
|
+
|
|
46
|
+
# The seed to pass to the next step. RNG inside a plan is a pure function
|
|
47
|
+
# of keys derived from the seed, so **it only needs advancing between
|
|
48
|
+
# steps** (#19 §6). run does not return the seed, which would make the
|
|
49
|
+
# output layout variable; advancing is offered here as a pure function.
|
|
50
|
+
def next_seed(seed) = seed + 1
|
|
51
|
+
|
|
52
|
+
# Per-node key. Mixing in the index scaled by the golden ratio keeps the
|
|
53
|
+
# sequence apart from both neighboring seeds and neighboring nodes (the
|
|
54
|
+
# same trick as Random.next_seed).
|
|
55
|
+
def self.derive_key(seed, index) = (seed ^ ((index + 1) * 0x9E37_79B9_7F4A_7C15)) & 0xFFFF_FFFF_FFFF_FFFF
|
|
56
|
+
|
|
57
|
+
# Runs the plan with named input and index bindings.
|
|
58
|
+
#
|
|
59
|
+
# @param bindings [Hash{Symbol => Buffer, Integer}] values declared with {Builder#input} or {Builder#index}
|
|
60
|
+
# @return [Buffer, Array<Buffer>] graph outputs in {Builder#output} order
|
|
61
|
+
# @note The method does not synchronize; reading an output waits for queued work.
|
|
62
|
+
#
|
|
63
|
+
# plan.run(x: buf, pos: 7)
|
|
64
|
+
#
|
|
65
|
+
# Outputs come in g.output order; a single one is returned as a bare
|
|
66
|
+
# Buffer. State buffers are updated in place in the bound Buffers (not
|
|
67
|
+
# returned).
|
|
68
|
+
#
|
|
69
|
+
# **The plan does not synchronize at its end.** It returns with launches
|
|
70
|
+
# still queued, so rerunning the same plan, as a decode loop does, adds no
|
|
71
|
+
# per-step wait. Reading a result (`#to_a`) waits for the host transfer,
|
|
72
|
+
# so there is one synchronization when the value is actually wanted.
|
|
73
|
+
def run(**bindings)
|
|
74
|
+
values = {}
|
|
75
|
+
output_ids = outputs.map(&:id)
|
|
76
|
+
# Nodes whose placement took effect; for them the write needs no copy.
|
|
77
|
+
@placed = {}
|
|
78
|
+
@sync_stats = { nodes: 0, synchronous: 0, fences: 0 }
|
|
79
|
+
# If there is an enclosing submit boundary, hand ownership to it (#86
|
|
80
|
+
# stage 2). The plan installs its own context, so without this, deferred
|
|
81
|
+
# frees of buffers made in the submit block would land outside the boundary.
|
|
82
|
+
@context.outer = Submission.context
|
|
83
|
+
Ops.with_context(@context) do
|
|
84
|
+
index = 0
|
|
85
|
+
@order.each do |node|
|
|
86
|
+
next if @absorbed[node.id] # computed inside its cluster, not run alone
|
|
87
|
+
|
|
88
|
+
@residency&.ensure!(index)
|
|
89
|
+
values[node.id] = execute_node(node, values, bindings)
|
|
90
|
+
release(index, values, output_ids)
|
|
91
|
+
note_queue_state(node)
|
|
92
|
+
index += 1
|
|
93
|
+
end
|
|
94
|
+
end
|
|
95
|
+
@context.drain_deferred!
|
|
96
|
+
results = outputs.map { |node| values[node.id] }
|
|
97
|
+
results.length == 1 ? results.first : results
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
# The capabilities this plan requires: the interface for saying "this
|
|
101
|
+
# graph does not run on :metal" **before running** (#17's value 5).
|
|
102
|
+
#
|
|
103
|
+
# plan.requires
|
|
104
|
+
# # => [#<Requirement capability=:conv ops=[:conv2d] satisfied=true>]
|
|
105
|
+
#
|
|
106
|
+
# compile fails on anything unsatisfiable, so satisfied=false appears
|
|
107
|
+
# here only if capabilities changed afterwards. Ops with a decomposition
|
|
108
|
+
# are expanded into primitives at record time and never appear here.
|
|
109
|
+
Requirement = Struct.new(:capability, :ops, :satisfied) do
|
|
110
|
+
def satisfied? = satisfied
|
|
111
|
+
def to_s = "#{capability} (#{ops.join(', ')})#{satisfied ? '' : ' — missing'}"
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
def requires
|
|
115
|
+
@nodes.filter_map { |node| [node.spec.requires, node.op] if node.spec.requires }
|
|
116
|
+
.group_by(&:first)
|
|
117
|
+
.map { |capability, pairs| Requirement.new(capability, pairs.map(&:last).uniq, Op.satisfied?(capability)) }
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
# How weights were assigned against the VRAM budget (only when
|
|
121
|
+
# compile(vram:) was given). Tells before running whether it fits and
|
|
122
|
+
# how many bytes move per run — see docs/graph-offload.md.
|
|
123
|
+
#
|
|
124
|
+
# plan.residency
|
|
125
|
+
# # => vram 8.00 GB, reserved 1.07 GB, weights 6.93 GB of 13.47 GB,
|
|
126
|
+
# # paged 18 of 34, bytes/run 7.00 GB (floor 6.55 GB)
|
|
127
|
+
def residency = @residency_report
|
|
128
|
+
|
|
129
|
+
# Bytes actually moved and time waited. Compare bytes_in with
|
|
130
|
+
# residency.floor first.
|
|
131
|
+
def offload_stats = @residency&.stats
|
|
132
|
+
|
|
133
|
+
# Deletes the offload files the plan itself wrote (the companion of
|
|
134
|
+
# free_pool!). Files of weights the user offloaded are left alone.
|
|
135
|
+
def free_offload! = @residency&.free_spills!
|
|
136
|
+
|
|
137
|
+
# How many buffers the pool holds and how many allocations / reuses
|
|
138
|
+
# happened — for measuring whether reuse works (allocated stops growing
|
|
139
|
+
# from the second run on).
|
|
140
|
+
def pool_stats = @pool&.stats
|
|
141
|
+
|
|
142
|
+
# Releases the pool's device memory.
|
|
143
|
+
def free_pool! = @pool&.free_all!
|
|
144
|
+
|
|
145
|
+
# In the latest run, how many nodes ran and how many made the host wait
|
|
146
|
+
# (#86's acceptance criterion: observing fewer host synchronizations per
|
|
147
|
+
# dispatch). `synchronous` counts nodes launched with wait: true, `fences`
|
|
148
|
+
# the `Tilework.synchronize` calls added around them.
|
|
149
|
+
#
|
|
150
|
+
# plan.run(x: buf); plan.sync_stats
|
|
151
|
+
# # => {nodes: 42, synchronous: 0, fences: 0}
|
|
152
|
+
def sync_stats = @sync_stats&.dup
|
|
153
|
+
|
|
154
|
+
# A readable listing of the execution order (for debugging). Nodes fused
|
|
155
|
+
# away are never launched, so they are marked as such.
|
|
156
|
+
def explain
|
|
157
|
+
index = -1
|
|
158
|
+
@order.map do |node|
|
|
159
|
+
next " (fused) #{node}" if @absorbed[node.id]
|
|
160
|
+
|
|
161
|
+
index += 1
|
|
162
|
+
suffix = @clusters[node.id] ? " <- fused #{@clusters[node.id].members.length + 1} nodes" : ""
|
|
163
|
+
"#{index}: #{node}#{suffix}"
|
|
164
|
+
end.join("\n")
|
|
165
|
+
end
|
|
166
|
+
|
|
167
|
+
# Fusion results. nodes counts nodes actually executed (including ones
|
|
168
|
+
# like inputs and constants that launch no kernel), so it is not the
|
|
169
|
+
# launch count itself.
|
|
170
|
+
def fusion_stats
|
|
171
|
+
{ clusters: @clusters.length, absorbed: @absorbed.length,
|
|
172
|
+
nodes: @order.count { |node| !@absorbed[node.id] } }
|
|
173
|
+
end
|
|
174
|
+
|
|
175
|
+
def inspect
|
|
176
|
+
"#<Tilework::Graph::Plan #{@order.length} nodes, #{states.length} states, " \
|
|
177
|
+
"#{outputs.length} outputs, strict_state: #{@strict_state.inspect}>"
|
|
178
|
+
end
|
|
179
|
+
|
|
180
|
+
private
|
|
181
|
+
|
|
182
|
+
# --- Compile ---
|
|
183
|
+
|
|
184
|
+
# Each state must end in a single final version; forgetting to rejoin
|
|
185
|
+
# forked writes to disjoint regions with g.merge fails here.
|
|
186
|
+
def check_single_leaf!
|
|
187
|
+
@builder.states.each do |st|
|
|
188
|
+
next if st.leaves.length == 1
|
|
189
|
+
|
|
190
|
+
versions = st.leaves.map { |leaf| "v#{leaf.version}" }.join(", ")
|
|
191
|
+
raise Error, "#{st.name} has #{st.leaves.length} unmerged versions (#{versions}); " \
|
|
192
|
+
"disjoint writes must be joined with g.merge before the plan ends"
|
|
193
|
+
end
|
|
194
|
+
end
|
|
195
|
+
|
|
196
|
+
# Type checking (Builder) and whether those types can be allocated on this
|
|
197
|
+
# backend (here) are separate questions. A float16 graph built on :cpu is
|
|
198
|
+
# reported **at compile time**, all at once, instead of failing at the
|
|
199
|
+
# first Buffer.new mid-run — the smallest form of #17's "know portability
|
|
200
|
+
# before running".
|
|
201
|
+
def check_dtypes_supported!
|
|
202
|
+
supported = Tilework.capabilities.dtypes
|
|
203
|
+
missing = @nodes.map(&:dtype).uniq.reject { |dtype| supported.include?(dtype) }
|
|
204
|
+
return if missing.empty?
|
|
205
|
+
|
|
206
|
+
raise Error, "the graph uses #{missing.map(&:inspect).join(', ')} but the " \
|
|
207
|
+
"#{Tilework.backend.inspect} backend can only allocate " \
|
|
208
|
+
"#{supported.map(&:inspect).join(', ')}"
|
|
209
|
+
end
|
|
210
|
+
|
|
211
|
+
# As with types, whatever can be said before running is said before
|
|
212
|
+
# running. Only "the op is missing and has no decomposition" gets here —
|
|
213
|
+
# ops with a decomposition were expanded into primitives at record time.
|
|
214
|
+
def check_requirements!
|
|
215
|
+
missing = requires.reject(&:satisfied?)
|
|
216
|
+
return if missing.empty?
|
|
217
|
+
|
|
218
|
+
raise Error, "the graph needs #{missing.map(&:to_s).join(', ')} but the " \
|
|
219
|
+
"#{Tilework.backend.inspect} backend does not provide it"
|
|
220
|
+
end
|
|
221
|
+
|
|
222
|
+
def reachable(roots)
|
|
223
|
+
seen = {}
|
|
224
|
+
stack = roots.dup
|
|
225
|
+
while (node = stack.pop)
|
|
226
|
+
next if seen.key?(node.id)
|
|
227
|
+
|
|
228
|
+
seen[node.id] = node
|
|
229
|
+
node.inputs.each { |input| stack.push(input) }
|
|
230
|
+
end
|
|
231
|
+
seen.values.sort_by(&:id)
|
|
232
|
+
end
|
|
233
|
+
|
|
234
|
+
# Version edges give only RAW (write, then read), not WAR — a version is
|
|
235
|
+
# **the same memory**, not a new value, so version N+1's write running
|
|
236
|
+
# while version N's readers are still reading breaks things. There is no
|
|
237
|
+
# value edge for that, so it is added here as an ordering edge:
|
|
238
|
+
#
|
|
239
|
+
# a node overwriting version N runs after the readers of version N and their direct consumers.
|
|
240
|
+
#
|
|
241
|
+
# Direct consumers suffice because a read returns a view (zero copy) and
|
|
242
|
+
# each consumer owns its output buffer; once it has run, nothing aliases
|
|
243
|
+
# the state any more.
|
|
244
|
+
def order_dependencies
|
|
245
|
+
deps = Hash.new { |h, k| h[k] = [] }
|
|
246
|
+
consumers = consumer_index
|
|
247
|
+
@nodes.each do |writer|
|
|
248
|
+
next unless writer.op == :state_write
|
|
249
|
+
|
|
250
|
+
version_node = writer.inputs.first
|
|
251
|
+
consumers[version_node.id].each do |reader|
|
|
252
|
+
next unless reader.op == :state_read
|
|
253
|
+
next if reader.equal?(writer)
|
|
254
|
+
|
|
255
|
+
deps[writer.id] << reader.id
|
|
256
|
+
consumers[reader.id].each do |consumer|
|
|
257
|
+
deps[writer.id] << consumer.id unless consumer.equal?(writer)
|
|
258
|
+
end
|
|
259
|
+
end
|
|
260
|
+
end
|
|
261
|
+
deps.each_value(&:uniq!)
|
|
262
|
+
deps
|
|
263
|
+
end
|
|
264
|
+
|
|
265
|
+
def consumer_index
|
|
266
|
+
index = Hash.new { |h, k| h[k] = [] }
|
|
267
|
+
in_plan = @nodes.each_with_object({}) { |n, h| h[n.id] = true }
|
|
268
|
+
@nodes.each do |node|
|
|
269
|
+
node.inputs.each { |input| index[input.id] << node if in_plan[input.id] }
|
|
270
|
+
end
|
|
271
|
+
index
|
|
272
|
+
end
|
|
273
|
+
|
|
274
|
+
# Topological sort over value edges + ordering edges. Ready nodes come out
|
|
275
|
+
# in id order, so the same graph always yields the same order.
|
|
276
|
+
def toposort
|
|
277
|
+
adjacency = Hash.new { |h, k| h[k] = [] }
|
|
278
|
+
indegree = @nodes.to_h { |node| [node.id, 0] }
|
|
279
|
+
by_id = @nodes.to_h { |node| [node.id, node] }
|
|
280
|
+
|
|
281
|
+
@nodes.each do |node|
|
|
282
|
+
(node.inputs.map(&:id) + @order_deps[node.id]).uniq.each do |from|
|
|
283
|
+
adjacency[from] << node.id
|
|
284
|
+
indegree[node.id] += 1
|
|
285
|
+
end
|
|
286
|
+
end
|
|
287
|
+
|
|
288
|
+
ready = indegree.select { |_, d| d.zero? }.keys.sort
|
|
289
|
+
sorted = []
|
|
290
|
+
until ready.empty?
|
|
291
|
+
id = ready.shift
|
|
292
|
+
sorted << by_id[id]
|
|
293
|
+
adjacency[id].each do |to|
|
|
294
|
+
indegree[to] -= 1
|
|
295
|
+
ready = ready.push(to).sort if indegree[to].zero?
|
|
296
|
+
end
|
|
297
|
+
end
|
|
298
|
+
return sorted if sorted.length == @nodes.length
|
|
299
|
+
|
|
300
|
+
raise Error, cycle_message(@nodes.reject { |n| sorted.include?(n) })
|
|
301
|
+
end
|
|
302
|
+
|
|
303
|
+
# In practice a cycle has only one shape: a read of version N used by a
|
|
304
|
+
# node **after** the one overwriting version N. The read is a view, so by
|
|
305
|
+
# then its value has been overwritten; no order resolves it, so fail.
|
|
306
|
+
def cycle_message(stuck)
|
|
307
|
+
states = stuck.select(&:writes_state?).map { |n| n.state.name }.uniq
|
|
308
|
+
detail = states.empty? ? "" : " (#{states.join(', ')})"
|
|
309
|
+
"cannot order the graph#{detail}: a value read from a state version is used after that " \
|
|
310
|
+
"version was overwritten. A read is a view of the same memory, so the value would be " \
|
|
311
|
+
"clobbered — read the newer version instead, or set " \
|
|
312
|
+
"Tilework::Graph.strict_state = :copy to materialize reads.\n" \
|
|
313
|
+
" stuck: #{stuck.map(&:to_s).join("\n ")}"
|
|
314
|
+
end
|
|
315
|
+
|
|
316
|
+
# The fusion plan. Nodes absorbed into a cluster are no longer
|
|
317
|
+
# materialized, so liveness and execution from here on work in terms of
|
|
318
|
+
# **effective_inputs** (inputs coming from outside the cluster).
|
|
319
|
+
def plan_fusion
|
|
320
|
+
return [{}, {}] unless Graph.fusion
|
|
321
|
+
|
|
322
|
+
# Nodes named by ordering edges are not absorbed: that would move "the
|
|
323
|
+
# moment it finished" to the cluster root and reorder it against state
|
|
324
|
+
# writes. @order_deps is a Hash with a default, and toposort leaves empty
|
|
325
|
+
# arrays behind for every node it looked up — consider **only non-empty
|
|
326
|
+
# edges**.
|
|
327
|
+
pinned = @order_deps.flat_map { |writer, readers| readers.empty? ? [] : [writer, *readers] }
|
|
328
|
+
.to_h { |id| [id, true] }
|
|
329
|
+
Fusion.plan(order: @order, output_ids: outputs.map(&:id).to_h { |id| [id, true] },
|
|
330
|
+
consumers: consumer_index, pinned: pinned)
|
|
331
|
+
end
|
|
332
|
+
|
|
333
|
+
# Output placement (#19 §3). `write_into` needs no copy in principle —
|
|
334
|
+
# making the write target the producer's **output** costs nothing. This
|
|
335
|
+
# picks only the nodes where that is safe.
|
|
336
|
+
#
|
|
337
|
+
# Four conditions, each of which breaks things if dropped:
|
|
338
|
+
#
|
|
339
|
+
# 1. the op allocates its output once and fills it. NN, opaque and
|
|
340
|
+
# reductions allocate several inside, so which one is the result is
|
|
341
|
+
# unknowable
|
|
342
|
+
# 2. the write is the only consumer. Placing a value read elsewhere on
|
|
343
|
+
# the state lets a later write corrupt that reader
|
|
344
|
+
# 3. the write carries no ordering (WAR) edge. **The actual memory
|
|
345
|
+
# mutation moves to the value's node**, so the edge would have to move
|
|
346
|
+
# with it, and if it points at that node's descendants it becomes a
|
|
347
|
+
# cycle; only cases needing no move pass
|
|
348
|
+
# 4. the value's node does not read the same state (it would read and
|
|
349
|
+
# write one region at once)
|
|
350
|
+
#
|
|
351
|
+
# 1 is an op property held by the table (Graph::Op); the other three are
|
|
352
|
+
# about the graph's shape, so they are checked here.
|
|
353
|
+
def plan_placement
|
|
354
|
+
return {} unless Graph.placement
|
|
355
|
+
|
|
356
|
+
consumers = consumer_index
|
|
357
|
+
output_ids = outputs.map(&:id).to_h { |id| [id, true] }
|
|
358
|
+
@nodes.select { |node| node.op == :state_write }.filter_map do |writer|
|
|
359
|
+
value = writer.inputs[1]
|
|
360
|
+
next unless placeable?(value, writer, consumers, output_ids)
|
|
361
|
+
|
|
362
|
+
[value.id, writer]
|
|
363
|
+
end.to_h
|
|
364
|
+
end
|
|
365
|
+
|
|
366
|
+
def placeable?(value, writer, consumers, output_ids)
|
|
367
|
+
return false unless value.spec.placeable?(value)
|
|
368
|
+
return false if output_ids.key?(value.id)
|
|
369
|
+
return false unless consumers[value.id].length == 1
|
|
370
|
+
return false unless @order_deps[writer.id].empty?
|
|
371
|
+
|
|
372
|
+
!reads_state?(value, writer.state)
|
|
373
|
+
end
|
|
374
|
+
|
|
375
|
+
# Whether any node touching the same state precedes producing value.
|
|
376
|
+
def reads_state?(value, state)
|
|
377
|
+
seen = {}
|
|
378
|
+
stack = [value]
|
|
379
|
+
while (node = stack.pop)
|
|
380
|
+
next if seen[node.id]
|
|
381
|
+
|
|
382
|
+
seen[node.id] = true
|
|
383
|
+
return true if node.attrs[:state].equal?(state)
|
|
384
|
+
|
|
385
|
+
node.inputs.each { |input| stack.push(input) }
|
|
386
|
+
end
|
|
387
|
+
false
|
|
388
|
+
end
|
|
389
|
+
|
|
390
|
+
# What is known only at run time (dynamic positions, contiguity) is checked here.
|
|
391
|
+
def placement_for(node, bindings)
|
|
392
|
+
writer = @placements[node.id]
|
|
393
|
+
return nil unless writer
|
|
394
|
+
|
|
395
|
+
region = writer.region.resolve(bindings)
|
|
396
|
+
return nil unless region.contiguous?
|
|
397
|
+
|
|
398
|
+
writer.state.buffer.view(region.offset, region.length, shape: node.shape)
|
|
399
|
+
end
|
|
400
|
+
|
|
401
|
+
# reshape and slice own no memory and just return a view of their input.
|
|
402
|
+
# Reading a view reads the original buffer, so lifetime must be counted
|
|
403
|
+
# on **the owning side**, not the view — otherwise memory still being read
|
|
404
|
+
# goes back to the pool and to the next writer.
|
|
405
|
+
ALIAS_OPS = %i[reshape slice].freeze
|
|
406
|
+
|
|
407
|
+
def alias_root(node)
|
|
408
|
+
node = node.inputs.first while ALIAS_OPS.include?(node.op)
|
|
409
|
+
node
|
|
410
|
+
end
|
|
411
|
+
|
|
412
|
+
# Inputs closed inside a cluster disappear, so liveness uses inputs from outside it.
|
|
413
|
+
def effective_inputs(node)
|
|
414
|
+
cluster = @clusters[node.id]
|
|
415
|
+
cluster ? cluster.inputs : node.inputs
|
|
416
|
+
end
|
|
417
|
+
|
|
418
|
+
# Liveness: release intermediates where they are last read. State buffers
|
|
419
|
+
# (and inputs and constants) are not owned, so they are left alone.
|
|
420
|
+
def plan_releases
|
|
421
|
+
releases = Hash.new { |h, k| h[k] = [] }
|
|
422
|
+
executed = @order.reject { |node| @absorbed[node.id] }
|
|
423
|
+
position = executed.each_with_index.to_h { |node, i| [node.id, i] }
|
|
424
|
+
# reshape is a view, so the output is held by the owning side.
|
|
425
|
+
output_ids = outputs.map { |node| alias_root(node).id }
|
|
426
|
+
last = {}
|
|
427
|
+
executed.each do |node|
|
|
428
|
+
effective_inputs(node).each { |input| last[alias_root(input).id] = position[node.id] }
|
|
429
|
+
end
|
|
430
|
+
@nodes.each do |node|
|
|
431
|
+
next unless node.spec.owned?
|
|
432
|
+
next if output_ids.include?(node.id)
|
|
433
|
+
next if @absorbed[node.id] # never materialized, so nothing to release
|
|
434
|
+
next unless last.key?(node.id)
|
|
435
|
+
|
|
436
|
+
releases[last[node.id]] << node.id
|
|
437
|
+
end
|
|
438
|
+
releases
|
|
439
|
+
end
|
|
440
|
+
|
|
441
|
+
# Assigning weights against the budget (#17 Phase 4 / docs/graph-offload.md).
|
|
442
|
+
# Nothing happens without vram: — paging on its own without being given a
|
|
443
|
+
# budget would hide the performance cliff (the same judgment as #17's
|
|
444
|
+
# "no implicit laziness").
|
|
445
|
+
def plan_residency(vram, spill_dir, backing, host_budget)
|
|
446
|
+
return nil if vram.nil? || !Graph.offload
|
|
447
|
+
|
|
448
|
+
unless %i[file host].include?(backing)
|
|
449
|
+
raise ArgumentError, "backing: must be :file or :host"
|
|
450
|
+
end
|
|
451
|
+
if backing == :host
|
|
452
|
+
raise Error, "pageable host backing is not available on the active backend" unless Tilework.capabilities[:host_backing]
|
|
453
|
+
raise ArgumentError, "host_budget: must be a positive Integer of bytes" unless host_budget.is_a?(Integer) && host_budget.positive?
|
|
454
|
+
end
|
|
455
|
+
|
|
456
|
+
raise ArgumentError, "vram: must be a positive Integer of bytes" unless vram.is_a?(Integer) && vram.positive?
|
|
457
|
+
|
|
458
|
+
executed = @order.reject { |node| @absorbed[node.id] }
|
|
459
|
+
positions = executed.each_with_index.to_h { |node, i| [node.id, i] }
|
|
460
|
+
candidates, pinned = split_consts
|
|
461
|
+
reserved = reserved_bytes(executed) + pinned.sum { |node| bytes_of(node) }
|
|
462
|
+
budget = vram - reserved
|
|
463
|
+
weights = candidates.sum { |node| bytes_of(node) }
|
|
464
|
+
if backing == :host && weights > host_budget
|
|
465
|
+
raise Error, "host backing budget #{host_budget} bytes cannot hold pageable candidates totaling #{weights} bytes"
|
|
466
|
+
end
|
|
467
|
+
|
|
468
|
+
entries = residency_entries(candidates, executed, positions)
|
|
469
|
+
needs = residency_needs(entries, executed, positions)
|
|
470
|
+
check_budget!(entries, needs, budget, vram, reserved, weights)
|
|
471
|
+
# Report even when it fits (being able to say "it fits" is worth something).
|
|
472
|
+
@residency_report = residency_report(vram, reserved, budget, weights, entries, pinned, executed, needs)
|
|
473
|
+
# A loose budget is free — if it fits, the machinery does not exist.
|
|
474
|
+
return nil if budget >= weights
|
|
475
|
+
|
|
476
|
+
Residency.new(entries: entries, needs: needs, steps: executed.length, budget: budget,
|
|
477
|
+
spill_dir: spill_dir || default_spill_dir, backing: backing, report: @residency_report)
|
|
478
|
+
end
|
|
479
|
+
|
|
480
|
+
# Splits consts into pageable and pinned. Two kinds cannot move: views
|
|
481
|
+
# (restore! allocates a new handle, leaving views of the old one dangling;
|
|
482
|
+
# buffer.rb's reject_view! refuses them too) and plan outputs (they must
|
|
483
|
+
# stay resident after run).
|
|
484
|
+
def split_consts
|
|
485
|
+
output_roots = outputs.map { |node| alias_root(node).id }.to_h { |id| [id, true] }
|
|
486
|
+
consts = @nodes.select { |node| node.op == :const && !@absorbed[node.id] }
|
|
487
|
+
.reject { |node| node.attrs[:buffer].freed? }
|
|
488
|
+
consts.partition do |node|
|
|
489
|
+
!node.attrs[:buffer].view? && !output_roots[node.id]
|
|
490
|
+
end
|
|
491
|
+
end
|
|
492
|
+
|
|
493
|
+
def residency_entries(candidates, executed, positions)
|
|
494
|
+
uses = Hash.new { |h, k| h[k] = [] }
|
|
495
|
+
executed.each do |node|
|
|
496
|
+
const_inputs(node).each { |const| uses[const.id] << positions[node.id] }
|
|
497
|
+
end
|
|
498
|
+
candidates.filter_map do |node|
|
|
499
|
+
used = uses[node.id].uniq.sort
|
|
500
|
+
next if used.empty?
|
|
501
|
+
|
|
502
|
+
buffer = node.attrs[:buffer]
|
|
503
|
+
Residency::Entry.new(node.id, buffer, bytes_of(node), used, !buffer.resident?, false)
|
|
504
|
+
end
|
|
505
|
+
end
|
|
506
|
+
|
|
507
|
+
def residency_needs(entries, executed, positions)
|
|
508
|
+
by_node = entries.to_h { |entry| [entry.node_id, entry] }
|
|
509
|
+
needs = Hash.new { |h, k| h[k] = [] }
|
|
510
|
+
executed.each do |node|
|
|
511
|
+
const_inputs(node).each do |const|
|
|
512
|
+
entry = by_node[const.id]
|
|
513
|
+
needs[positions[node.id]] << entry if entry
|
|
514
|
+
end
|
|
515
|
+
end
|
|
516
|
+
needs.each_value(&:uniq!)
|
|
517
|
+
needs
|
|
518
|
+
end
|
|
519
|
+
|
|
520
|
+
# Consts that must be resident when this node runs. Inputs fused away have
|
|
521
|
+
# become the cluster's inputs, so effective_inputs is consulted. reshape /
|
|
522
|
+
# slice create views (Buffer#view requires residency), so they appear here
|
|
523
|
+
# as direct inputs and count as a requirement at that position too.
|
|
524
|
+
def const_inputs(node)
|
|
525
|
+
effective_inputs(node).flat_map { |input| [input, alias_root(input)] }
|
|
526
|
+
.select { |input| input.op == :const }.uniq
|
|
527
|
+
end
|
|
528
|
+
|
|
529
|
+
# The part of the budget weights cannot use. Inputs/outputs and state
|
|
530
|
+
# cannot move, and the pool keeps intermediates, so those count at peak.
|
|
531
|
+
def reserved_bytes(executed)
|
|
532
|
+
io = @builder.input_nodes.each_value.sum { |node| bytes_of(node) } +
|
|
533
|
+
outputs.sum { |node| bytes_of(alias_root(node)) }
|
|
534
|
+
state = states.sum { |st| Residency.bytes_of(st.dtype, st.buffer.length) }
|
|
535
|
+
io + state + peak_intermediate_bytes(executed)
|
|
536
|
+
end
|
|
537
|
+
|
|
538
|
+
# Peak intermediates. The pool keeps slots across runs, so this is
|
|
539
|
+
# steady-state occupancy, not the first run's. Liveness (@release_at)
|
|
540
|
+
# already knows the release points, so this walks the same table in bytes.
|
|
541
|
+
def peak_intermediate_bytes(executed)
|
|
542
|
+
output_ids = outputs.map { |node| alias_root(node).id }.to_h { |id| [id, true] }
|
|
543
|
+
live = 0
|
|
544
|
+
peak = 0
|
|
545
|
+
executed.each_with_index do |node, index|
|
|
546
|
+
live += bytes_of(node) if node.spec.owned? && !output_ids[node.id]
|
|
547
|
+
peak = live if live > peak
|
|
548
|
+
@release_at[index].each do |id|
|
|
549
|
+
released = @nodes.find { |n| n.id == id }
|
|
550
|
+
live -= bytes_of(released) if released && !output_ids[id]
|
|
551
|
+
end
|
|
552
|
+
end
|
|
553
|
+
peak
|
|
554
|
+
end
|
|
555
|
+
|
|
556
|
+
# Fail before running whatever can be (like dtype / requires). Two
|
|
557
|
+
# checks: no room at all, or one node's simultaneous weights do not fit.
|
|
558
|
+
# The latter is "enough overall, but this one line does not pass", so the
|
|
559
|
+
# node is named.
|
|
560
|
+
def check_budget!(entries, needs, budget, vram, reserved, weights)
|
|
561
|
+
largest = entries.map(&:bytes).max || 0
|
|
562
|
+
if budget < largest
|
|
563
|
+
raise Error, "the graph reserves #{Residency.human(reserved)} for inputs, state and " \
|
|
564
|
+
"intermediates out of a #{Residency.human(vram)} budget, leaving " \
|
|
565
|
+
"#{Residency.human(budget)} for #{Residency.human(weights)} of weights; " \
|
|
566
|
+
"the largest single weight is #{Residency.human(largest)}"
|
|
567
|
+
end
|
|
568
|
+
|
|
569
|
+
needs.each do |index, required|
|
|
570
|
+
want = required.sum(&:bytes)
|
|
571
|
+
next if want <= budget
|
|
572
|
+
|
|
573
|
+
node = @order.reject { |n| @absorbed[n.id] }[index]
|
|
574
|
+
raise Error, "step #{index} (#{node}) needs #{Residency.human(want)} of weights at once " \
|
|
575
|
+
"but only #{Residency.human(budget)} is left after reserving " \
|
|
576
|
+
"#{Residency.human(reserved)} of a #{Residency.human(vram)} budget"
|
|
577
|
+
end
|
|
578
|
+
end
|
|
579
|
+
|
|
580
|
+
def residency_report(vram, reserved, budget, weights, entries, pinned, executed, needs)
|
|
581
|
+
# Bytes moved per run. At most budget carries over, so the difference
|
|
582
|
+
# moves under any policy (= floor). Cyclic Belady lands within one slot
|
|
583
|
+
# of it.
|
|
584
|
+
floor = [weights - budget, 0].max
|
|
585
|
+
# In practice one slot more moves — a sequential scan needs one slot for
|
|
586
|
+
# the eviction victim to rotate through. offload_stats[:bytes_in] is the
|
|
587
|
+
# measurement.
|
|
588
|
+
rotating = floor.positive? ? (entries.map(&:bytes).max || 0) : 0
|
|
589
|
+
Residency::Report.new(
|
|
590
|
+
vram: vram, reserved: reserved, budget: budget, weights: weights,
|
|
591
|
+
paged: entries.length, resident_always: pinned.length,
|
|
592
|
+
resident_always_bytes: pinned.sum { |node| bytes_of(node) },
|
|
593
|
+
prefetch_headroom: Residency.prefetch_ceiling(needs, budget),
|
|
594
|
+
bytes_per_run: floor + rotating, floor: floor, steps: executed.length,
|
|
595
|
+
)
|
|
596
|
+
end
|
|
597
|
+
|
|
598
|
+
def bytes_of(node) = Residency.bytes_of(node.dtype, node.length)
|
|
599
|
+
|
|
600
|
+
def default_spill_dir
|
|
601
|
+
require "tmpdir"
|
|
602
|
+
File.join(Dir.tmpdir, "tilework-offload-#{Process.pid}-#{object_id}")
|
|
603
|
+
end
|
|
604
|
+
|
|
605
|
+
# --- Execution ---
|
|
606
|
+
|
|
607
|
+
# Returns last-read intermediates to the pool (frees them without one).
|
|
608
|
+
# Views own nothing, so they never enter the pool.
|
|
609
|
+
def release(index, values, output_ids)
|
|
610
|
+
@release_at[index].each do |id|
|
|
611
|
+
buffer = values[id]
|
|
612
|
+
next unless buffer.is_a?(Buffer) && !output_ids.include?(id)
|
|
613
|
+
|
|
614
|
+
values[id] = nil
|
|
615
|
+
if @pool && !buffer.view?
|
|
616
|
+
if @pool.owns?(buffer)
|
|
617
|
+
@pool.release(buffer)
|
|
618
|
+
elsif @nodes_by_id.fetch(id).op == :opaque
|
|
619
|
+
# opaque may return a value the user made with Buffer.new. If
|
|
620
|
+
# fenced, the node ran synchronously and it can be freed on the
|
|
621
|
+
# spot; with fenced: false launches may still be running, so it
|
|
622
|
+
# is adopted into the pool instead of freed.
|
|
623
|
+
node = @nodes_by_id.fetch(id)
|
|
624
|
+
if node.attrs[:fenced]
|
|
625
|
+
buffer.free
|
|
626
|
+
else
|
|
627
|
+
@pool.release(@pool.adopt(buffer))
|
|
628
|
+
end
|
|
629
|
+
else
|
|
630
|
+
# An ownership violation that should be unreachable. In case the op
|
|
631
|
+
# was asynchronous, wait before freeing so the diagnostic exception
|
|
632
|
+
# does not leave a new leak behind.
|
|
633
|
+
Tilework.synchronize
|
|
634
|
+
buffer.free
|
|
635
|
+
raise Error, "graph node #{@nodes_by_id.fetch(id).op.inspect} returned an intermediate " \
|
|
636
|
+
"buffer that did not come from Ops.allocate_output"
|
|
637
|
+
end
|
|
638
|
+
else
|
|
639
|
+
buffer.free
|
|
640
|
+
end
|
|
641
|
+
end
|
|
642
|
+
end
|
|
643
|
+
|
|
644
|
+
# After a synchronous node the queue is empty; tell residency, so the next
|
|
645
|
+
# eviction can ride on that wait (#17 Phase 4c).
|
|
646
|
+
def note_queue_state(node)
|
|
647
|
+
return unless @residency
|
|
648
|
+
# Nodes that launch nothing (:const / :input / views) leave the queue as
|
|
649
|
+
# it was. Counting them as dirtying it would mean the :const right
|
|
650
|
+
# before every weight-reading matmul prevents any ride-along.
|
|
651
|
+
return unless node.spec.launches?
|
|
652
|
+
|
|
653
|
+
@residency.note_queue(dirty: !Graph.offload_fence_reuse || !drains_queue?(node))
|
|
654
|
+
end
|
|
655
|
+
|
|
656
|
+
# Whether the queue is empty once this node finishes. Synchronous nodes
|
|
657
|
+
# (:nn / whole-buffer reductions / delegated GEMM / opaque) launch with
|
|
658
|
+
# wait: true, so earlier work on the same queue has completed too.
|
|
659
|
+
# With asynchronous execution disabled, every node drains the queue.
|
|
660
|
+
def drains_queue?(node) = !@async || node.spec.synchronous?(node)
|
|
661
|
+
|
|
662
|
+
# Nodes that can stay asynchronous do; nodes that (may) leave our
|
|
663
|
+
# execution path wait on both sides and run synchronously.
|
|
664
|
+
def execute_node(node, values, bindings)
|
|
665
|
+
target = placement_for(node, bindings)
|
|
666
|
+
return dispatch_node(node, values, bindings) unless target
|
|
667
|
+
|
|
668
|
+
result = @context.place(target) { dispatch_node(node, values, bindings) }
|
|
669
|
+
@placed[node.id] = @context.placement_consumed?
|
|
670
|
+
result
|
|
671
|
+
end
|
|
672
|
+
|
|
673
|
+
def dispatch_node(node, values, bindings)
|
|
674
|
+
@sync_stats[:nodes] += 1
|
|
675
|
+
return execute(node, values, bindings) unless synchronous?(node)
|
|
676
|
+
|
|
677
|
+
@sync_stats[:synchronous] += 1
|
|
678
|
+
foreign = foreign?(node)
|
|
679
|
+
if foreign
|
|
680
|
+
@sync_stats[:fences] += 2
|
|
681
|
+
Tilework.synchronize
|
|
682
|
+
end
|
|
683
|
+
result = @context.synchronous { execute(node, values, bindings) }
|
|
684
|
+
Tilework.synchronize if foreign
|
|
685
|
+
result
|
|
686
|
+
end
|
|
687
|
+
|
|
688
|
+
# Whether to run synchronously or wait is an op property (including
|
|
689
|
+
# batching and delegation). Making everything synchronous only when
|
|
690
|
+
# @async is off is this method's job.
|
|
691
|
+
def synchronous?(node) = @async && node.spec.synchronous?(node)
|
|
692
|
+
|
|
693
|
+
def foreign?(node) = node.spec.foreign?(node)
|
|
694
|
+
|
|
695
|
+
# The table holds each op's execution method. All take the same
|
|
696
|
+
# (node, inputs, bindings), so no branching is needed here.
|
|
697
|
+
def execute(node, values, bindings)
|
|
698
|
+
cluster = @clusters[node.id]
|
|
699
|
+
return run_fused(cluster, cluster.inputs.map { |input| values[input.id] }) if cluster
|
|
700
|
+
|
|
701
|
+
inputs = node.inputs.map { |input| values[input.id] }
|
|
702
|
+
send(node.spec.run, node, inputs, bindings)
|
|
703
|
+
end
|
|
704
|
+
|
|
705
|
+
def run_const(node, _inputs, _bindings) = node.attrs[:buffer]
|
|
706
|
+
def run_state_init(node, _inputs, _bindings) = node.state.buffer
|
|
707
|
+
def run_state_merge(node, _inputs, _bindings) = node.state.buffer
|
|
708
|
+
|
|
709
|
+
def run_input(node, _inputs, bindings)
|
|
710
|
+
name = node.attrs[:name]
|
|
711
|
+
buffer = bindings.fetch(name) do
|
|
712
|
+
raise Error, "plan.run is missing input #{name.inspect} (expects #{input_names.inspect})"
|
|
713
|
+
end
|
|
714
|
+
raise TypeError, "input #{name.inspect} must be a Tilework::Buffer, got #{buffer.class}" unless buffer.is_a?(Buffer)
|
|
715
|
+
unless buffer.dtype == node.dtype && buffer.shape == node.shape
|
|
716
|
+
raise Error, "input #{name.inspect} was declared as #{node.shape.inspect} #{node.dtype} " \
|
|
717
|
+
"but got #{buffer.shape.inspect} #{buffer.dtype}"
|
|
718
|
+
end
|
|
719
|
+
|
|
720
|
+
buffer
|
|
721
|
+
end
|
|
722
|
+
|
|
723
|
+
# A read is a view by default = no kernel. It is materialized when
|
|
724
|
+
# - it is a plan output (so later writes do not corrupt it)
|
|
725
|
+
# - strict_state == :copy (for comparison with the pure semantics)
|
|
726
|
+
# - the region is not contiguous (a view cannot express it)
|
|
727
|
+
def run_state_read(node, _inputs, bindings)
|
|
728
|
+
region = node.region.resolve(bindings)
|
|
729
|
+
buffer = node.state.buffer
|
|
730
|
+
materialize = @strict_state == :copy || node.attrs[:materialize] || !region.contiguous?
|
|
731
|
+
unless materialize
|
|
732
|
+
return buffer.view(region.offset, region.length, shape: region.shape)
|
|
733
|
+
end
|
|
734
|
+
|
|
735
|
+
out = Ops.allocate_output(node.dtype, region.length, region.shape)
|
|
736
|
+
region.each_block do |src_offset, dst_offset, len|
|
|
737
|
+
Graph.copy(buffer.view(src_offset, len), out.view(dst_offset, len))
|
|
738
|
+
end
|
|
739
|
+
out
|
|
740
|
+
end
|
|
741
|
+
|
|
742
|
+
def run_state_write(node, inputs, bindings)
|
|
743
|
+
value = inputs[1]
|
|
744
|
+
# If the producer already wrote straight into the target (placement
|
|
745
|
+
# took effect), there is nothing left to copy.
|
|
746
|
+
return node.state.buffer if @placed[node.inputs[1].id]
|
|
747
|
+
|
|
748
|
+
region = node.region.resolve(bindings)
|
|
749
|
+
buffer = node.state.buffer
|
|
750
|
+
region.each_block do |dst_offset, src_offset, len|
|
|
751
|
+
source = value.length == len ? value : value.view(src_offset, len)
|
|
752
|
+
Graph.copy(source, buffer.view(dst_offset, len))
|
|
753
|
+
end
|
|
754
|
+
buffer
|
|
755
|
+
end
|
|
756
|
+
|
|
757
|
+
def run_binary(node, inputs, _bindings)
|
|
758
|
+
op = node.attrs[:op]
|
|
759
|
+
operand = node.attrs.key?(:scalar) ? node.attrs[:scalar] : inputs[1]
|
|
760
|
+
case node.attrs[:kind]
|
|
761
|
+
when :compare then Ops.compare(op, inputs[0], operand)
|
|
762
|
+
else Ops.arithmetic(op, inputs[0], operand, reversed: node.attrs[:reversed] || false)
|
|
763
|
+
end
|
|
764
|
+
end
|
|
765
|
+
|
|
766
|
+
def run_unary(node, inputs, _bindings)
|
|
767
|
+
fn = node.attrs[:fn]
|
|
768
|
+
fn == :abs ? Ops.unary_abs(inputs[0]) : Ops.unary_math(fn, inputs[0])
|
|
769
|
+
end
|
|
770
|
+
|
|
771
|
+
# A whole-buffer reduction takes Ops' device: true path (tree folding). It
|
|
772
|
+
# never reaches the host, so the result passes on as a one-element Buffer.
|
|
773
|
+
def run_reduce(node, inputs, _bindings)
|
|
774
|
+
kind = node.attrs[:kind]
|
|
775
|
+
return launch_axis_reduce(node, inputs[0]) unless node.attrs[:axis].nil?
|
|
776
|
+
return Ops.mean_op(inputs[0], device: true) if kind == :mean
|
|
777
|
+
|
|
778
|
+
Ops.reduce(kind, inputs[0], device: true)
|
|
779
|
+
end
|
|
780
|
+
|
|
781
|
+
def run_arg_reduce(node, inputs, _bindings)
|
|
782
|
+
return launch_axis_reduce(node, inputs[0]) unless node.attrs[:axis].nil?
|
|
783
|
+
|
|
784
|
+
Ops.arg_reduce(node.attrs[:kind], inputs[0], device: true)
|
|
785
|
+
end
|
|
786
|
+
|
|
787
|
+
# An axis reduction is one kernel. The input is viewed as
|
|
788
|
+
# [outer, axis_len, inner], one thread per output element folding axis_len
|
|
789
|
+
# values. outer and inner are the products of the dims before and after
|
|
790
|
+
# the axis, so any rank works the same.
|
|
791
|
+
def launch_axis_reduce(node, input)
|
|
792
|
+
shape = node.inputs.first.shape
|
|
793
|
+
axis = node.attrs[:axis]
|
|
794
|
+
axis_len = shape[axis]
|
|
795
|
+
inner = shape.drop(axis + 1).inject(1, :*)
|
|
796
|
+
kernel =
|
|
797
|
+
if node.op == :arg_reduce
|
|
798
|
+
Kernels.axis_arg_reduce(kind: node.attrs[:kind], in_dtype: input.dtype)
|
|
799
|
+
else
|
|
800
|
+
Kernels.axis_reduce(kind: node.attrs[:kind], in_dtype: input.dtype, out_dtype: node.dtype)
|
|
801
|
+
end
|
|
802
|
+
|
|
803
|
+
out = Ops.allocate_output(node.dtype, node.length, node.shape)
|
|
804
|
+
Ops.launch_kernel(kernel, grid: (node.length / Ops::BLOCK.to_f).ceil, block: Ops::BLOCK,
|
|
805
|
+
args: [flat(input), out, axis_len, inner, node.length])
|
|
806
|
+
out
|
|
807
|
+
end
|
|
808
|
+
|
|
809
|
+
def run_cast(node, inputs, _bindings)
|
|
810
|
+
kernel = Kernels.cast(from: node.inputs.first.dtype, to: node.dtype)
|
|
811
|
+
launch_elementwise(node, kernel, [flat(inputs[0])])
|
|
812
|
+
end
|
|
813
|
+
|
|
814
|
+
# Arguments are always [cond, a, b, out, n]; the side that is a scalar is
|
|
815
|
+
# passed as a value (which one is fixed at compile time).
|
|
816
|
+
def run_where(node, inputs, _bindings)
|
|
817
|
+
a_scalar = node.attrs[:a_scalar]
|
|
818
|
+
b_scalar = node.attrs[:b_scalar]
|
|
819
|
+
kernel = Kernels.select(dtype: node.dtype, a_scalar: !a_scalar.nil?, b_scalar: !b_scalar.nil?)
|
|
820
|
+
rest = inputs.drop(1).map { |buffer| flat(buffer) }
|
|
821
|
+
a = a_scalar || rest.shift
|
|
822
|
+
b = b_scalar || rest.shift
|
|
823
|
+
launch_elementwise(node, kernel, [flat(inputs[0]), a, b])
|
|
824
|
+
end
|
|
825
|
+
|
|
826
|
+
def run_iota(node, _inputs, _bindings)
|
|
827
|
+
kernel = Kernels.iota(shape: node.shape, axis: node.attrs[:axis], dtype: node.dtype)
|
|
828
|
+
launch_elementwise(node, kernel, [])
|
|
829
|
+
end
|
|
830
|
+
|
|
831
|
+
# Only changes the shape. **No kernel runs**; a view of the input buffer is
|
|
832
|
+
# returned. The view points at the original memory, so the original must
|
|
833
|
+
# stay alive until this node's consumers finish (alias_root in plan_releases).
|
|
834
|
+
def run_reshape(node, inputs, _bindings)
|
|
835
|
+
inputs[0].reshape(node.shape)
|
|
836
|
+
end
|
|
837
|
+
|
|
838
|
+
# One thread per output element; shared by the three above and broadcast.
|
|
839
|
+
def launch_elementwise(node, kernel, args)
|
|
840
|
+
out = Ops.allocate_output(node.dtype, node.length, node.shape)
|
|
841
|
+
Ops.launch_kernel(kernel, grid: (node.length / Ops::BLOCK.to_f).ceil, block: Ops::BLOCK,
|
|
842
|
+
args: [*args, out, node.length])
|
|
843
|
+
out
|
|
844
|
+
end
|
|
845
|
+
|
|
846
|
+
# Materializes an index remap (broadcast / permute). Only reached when
|
|
847
|
+
# fusion did not fold it.
|
|
848
|
+
def run_remap(node, inputs, _bindings)
|
|
849
|
+
kernel = Kernels.remap(dims: Kernels.remap_dims(node), dtype: node.dtype, name: node.op.to_s)
|
|
850
|
+
launch_elementwise(node, kernel, [flat(inputs[0])])
|
|
851
|
+
end
|
|
852
|
+
|
|
853
|
+
# An outermost-dimension slice. A view like reshape, so no kernel runs.
|
|
854
|
+
def run_slice(node, inputs, _bindings)
|
|
855
|
+
inputs[0].view(node.attrs[:offset], node.length, shape: node.shape)
|
|
856
|
+
end
|
|
857
|
+
|
|
858
|
+
def run_gather(node, inputs, _bindings)
|
|
859
|
+
source = node.inputs.first
|
|
860
|
+
axis = node.attrs[:axis]
|
|
861
|
+
inner = source.shape.drop(axis + 1).inject(1, :*)
|
|
862
|
+
kernel = Kernels.gather(dtype: node.dtype, axis_len: source.shape[axis],
|
|
863
|
+
idx_len: node.inputs[1].length, inner: inner)
|
|
864
|
+
launch_elementwise(node, kernel, [flat(inputs[0]), flat(inputs[1])])
|
|
865
|
+
end
|
|
866
|
+
|
|
867
|
+
# The inverse of gather: one thread per output element scans the index
|
|
868
|
+
# list (see the note on Kernels.scatter_add).
|
|
869
|
+
def run_scatter_add(node, inputs, _bindings)
|
|
870
|
+
axis = node.attrs[:axis]
|
|
871
|
+
inner = node.shape.drop(axis + 1).inject(1, :*)
|
|
872
|
+
kernel = Kernels.scatter_add(dtype: node.dtype, axis_len: node.shape[axis],
|
|
873
|
+
idx_len: node.inputs[0].length, inner: inner)
|
|
874
|
+
launch_elementwise(node, kernel, [flat(inputs[0]), flat(inputs[1])])
|
|
875
|
+
end
|
|
876
|
+
|
|
877
|
+
def run_pad(node, inputs, _bindings)
|
|
878
|
+
source = node.inputs.first
|
|
879
|
+
strides = Kernels.row_major_strides(source.shape)
|
|
880
|
+
dims = node.shape.each_with_index.map do |dim, d|
|
|
881
|
+
[dim, source.shape[d], node.attrs[:before][d], strides[d]]
|
|
882
|
+
end
|
|
883
|
+
kernel = Kernels.pad(dims: dims, dtype: node.dtype, value: node.attrs[:value])
|
|
884
|
+
out = Ops.allocate_output(node.dtype, node.length, node.shape)
|
|
885
|
+
Ops.launch_kernel(kernel, grid: (node.length / Ops::BLOCK.to_f).ceil, block: Ops::BLOCK,
|
|
886
|
+
args: [flat(inputs[0]), out, node.attrs[:value], node.length])
|
|
887
|
+
out
|
|
888
|
+
end
|
|
889
|
+
|
|
890
|
+
# Kernels see indices as 1D, so multidimensional buffers are passed
|
|
891
|
+
# flattened (as in Graph.copy).
|
|
892
|
+
def flat(buffer) = buffer.shape.length > 1 ? buffer.reshape(buffer.length) : buffer
|
|
893
|
+
|
|
894
|
+
# Runs a fused cluster as one kernel. The output takes the same two forms
|
|
895
|
+
# as everything else (dialect source / :cpu Ruby source), so it rides
|
|
896
|
+
# Kernel's SHA cache as is — clusters with the same expression compile
|
|
897
|
+
# once even across layers.
|
|
898
|
+
def run_fused(cluster, inputs)
|
|
899
|
+
return run_row_fused(cluster, inputs) if cluster.row_wise?
|
|
900
|
+
|
|
901
|
+
kernel = Ops.compiled(
|
|
902
|
+
dialect_name: "fused", dialect_src: Fusion.dialect_source(cluster),
|
|
903
|
+
cpu_name: "fused",
|
|
904
|
+
cpu_src: lambda {
|
|
905
|
+
Ops.codegen.fused(cluster.expr,
|
|
906
|
+
input_dtypes: cluster.inputs.map(&:dtype),
|
|
907
|
+
scalar_dtypes: cluster.scalars.map { |s| s[:dtype] },
|
|
908
|
+
out_dtype: cluster.dtype)
|
|
909
|
+
},
|
|
910
|
+
)
|
|
911
|
+
out = Ops.allocate_output(cluster.dtype, cluster.length, cluster.shape)
|
|
912
|
+
args = inputs + cluster.scalars.map { |s| s[:value] } + [out, cluster.length]
|
|
913
|
+
Ops.launch_kernel(kernel, grid: (cluster.length / Ops::BLOCK.to_f).ceil, block: Ops::BLOCK, args: args)
|
|
914
|
+
out
|
|
915
|
+
end
|
|
916
|
+
|
|
917
|
+
# A row-wise cluster. **The launch domain is rows, not elements** (one
|
|
918
|
+
# thread per row); only the launch differs from elementwise clusters.
|
|
919
|
+
def run_row_fused(cluster, inputs)
|
|
920
|
+
kernel = Ops.compiled(
|
|
921
|
+
dialect_name: "row_fused", dialect_src: Fusion.dialect_source(cluster),
|
|
922
|
+
cpu_name: "row_fused",
|
|
923
|
+
cpu_src: lambda {
|
|
924
|
+
Ops.codegen.row_fused(cluster.expr, reductions: cluster.reductions,
|
|
925
|
+
input_dtypes: cluster.inputs.map(&:dtype),
|
|
926
|
+
scalar_dtypes: cluster.scalars.map { |s| s[:dtype] },
|
|
927
|
+
out_dtype: cluster.dtype)
|
|
928
|
+
},
|
|
929
|
+
)
|
|
930
|
+
out = Ops.allocate_output(cluster.dtype, cluster.length, cluster.shape)
|
|
931
|
+
args = inputs + cluster.scalars.map { |s| s[:value] } + [out, cluster.rows, cluster.cols]
|
|
932
|
+
Ops.launch_kernel(kernel, grid: (cluster.rows / Ops::BLOCK.to_f).ceil, block: Ops::BLOCK, args: args)
|
|
933
|
+
out
|
|
934
|
+
end
|
|
935
|
+
|
|
936
|
+
def run_matmul(node, inputs, _bindings)
|
|
937
|
+
unless node.attrs[:mode] == :batched
|
|
938
|
+
return Ops.matmul_op(inputs[0], inputs[1], out_dtype: node.attrs.fetch(:out_dtype, node.dtype))
|
|
939
|
+
end
|
|
940
|
+
|
|
941
|
+
run_batched_matmul(node, inputs[0], inputs[1])
|
|
942
|
+
end
|
|
943
|
+
|
|
944
|
+
# [b, m, k] x [b, k, n]; a strided batched GEMM does it in one call.
|
|
945
|
+
#
|
|
946
|
+
# The bridge's GEMM is **column-major** (see tensor_core.rb), so row-major
|
|
947
|
+
# OUT = A·B is reinterpreted transposed: OUT^T = B^T · A^T, i.e. swap A
|
|
948
|
+
# and B and swap m and n as column-major arguments. Each ld is its
|
|
949
|
+
# row-major width, which is exactly the column-major leading dimension.
|
|
950
|
+
def run_batched_matmul(node, a, b)
|
|
951
|
+
batch, m, k = a.shape
|
|
952
|
+
n = b.shape[2]
|
|
953
|
+
compute_dtype = node.dtype == :int32 ? :int32 : :float32
|
|
954
|
+
compute_out = Ops.allocate_output(compute_dtype, node.length, node.shape)
|
|
955
|
+
if batched_gemm_usable?(node, a, b)
|
|
956
|
+
TensorCore.gemm_batched(b, a, compute_out,
|
|
957
|
+
m: n, k: k, n: m,
|
|
958
|
+
lda: n, ldb: k, ldc: n,
|
|
959
|
+
stride_a: k * n, stride_b: m * k, stride_c: m * n,
|
|
960
|
+
batch: batch)
|
|
961
|
+
return convert_batched_output(compute_out, node.dtype)
|
|
962
|
+
end
|
|
963
|
+
|
|
964
|
+
# Without a delegate, fall back to per-batch matmuls. The output is
|
|
965
|
+
# allocated here, so one extra copy per batch moves each result into place.
|
|
966
|
+
batch.times do |i|
|
|
967
|
+
part = Ops.matmul_op(a.slice(i).reshape(m, k), b.slice(i).reshape(k, n))
|
|
968
|
+
Graph.copy(part, compute_out.view(i * m * n, m * n))
|
|
969
|
+
part.free
|
|
970
|
+
end
|
|
971
|
+
convert_batched_output(compute_out, node.dtype)
|
|
972
|
+
end
|
|
973
|
+
|
|
974
|
+
def convert_batched_output(out, dtype)
|
|
975
|
+
return out if out.dtype == dtype
|
|
976
|
+
|
|
977
|
+
converted = out.cast(dtype)
|
|
978
|
+
out.free
|
|
979
|
+
converted
|
|
980
|
+
end
|
|
981
|
+
|
|
982
|
+
# gemm_batched applies only when both sides share a float type and the output is float32.
|
|
983
|
+
def batched_gemm_usable?(_node, a, b)
|
|
984
|
+
a.dtype == b.dtype && a.dtype != :int32 && TensorCore.gemm_batched? &&
|
|
985
|
+
Tilework.capabilities.gemm_dtypes.include?(a.dtype)
|
|
986
|
+
end
|
|
987
|
+
|
|
988
|
+
# concat accepts only the outermost dim, so each piece is contiguous in
|
|
989
|
+
# the output; one copy per piece suffices, with no scattered writes.
|
|
990
|
+
def run_concat(node, inputs, _bindings)
|
|
991
|
+
out = Ops.allocate_output(node.dtype, node.length, node.shape)
|
|
992
|
+
axis = node.attrs[:axis] || 0
|
|
993
|
+
return concat_contiguous(out, inputs) if axis.zero?
|
|
994
|
+
|
|
995
|
+
inner = node.shape.drop(axis + 1).inject(1, :*)
|
|
996
|
+
offset = 0
|
|
997
|
+
node.inputs.each_with_index do |piece, i|
|
|
998
|
+
kernel = Kernels.concat_piece(dtype: node.dtype, piece_axis: piece.shape[axis],
|
|
999
|
+
total_axis: node.shape[axis], inner: inner, offset: offset)
|
|
1000
|
+
Ops.launch_kernel(kernel, grid: (piece.length / Ops::BLOCK.to_f).ceil, block: Ops::BLOCK,
|
|
1001
|
+
args: [flat(inputs[i]), out, piece.length])
|
|
1002
|
+
offset += piece.shape[axis]
|
|
1003
|
+
end
|
|
1004
|
+
out
|
|
1005
|
+
end
|
|
1006
|
+
|
|
1007
|
+
# Along the outermost dim pieces are contiguous in the output, so one copy each suffices.
|
|
1008
|
+
def concat_contiguous(out, inputs)
|
|
1009
|
+
offset = 0
|
|
1010
|
+
inputs.each do |buffer|
|
|
1011
|
+
Graph.copy(buffer, out.view(offset, buffer.length))
|
|
1012
|
+
offset += buffer.length
|
|
1013
|
+
end
|
|
1014
|
+
out
|
|
1015
|
+
end
|
|
1016
|
+
|
|
1017
|
+
# Derives each node's key from (plan seed, static index). Philox is
|
|
1018
|
+
# counter-based and takes its key as an argument, so this alone makes RNG
|
|
1019
|
+
# **pure** — independent of execution order, same seed same sequence.
|
|
1020
|
+
def run_random(node, _inputs, bindings)
|
|
1021
|
+
seed = bindings.fetch(:seed) do
|
|
1022
|
+
raise Error, "plan.run needs a seed: this graph draws #{@builder.random_draws} random value(s)"
|
|
1023
|
+
end
|
|
1024
|
+
raise TypeError, "seed must be an Integer, got #{seed.class}" unless seed.is_a?(Integer)
|
|
1025
|
+
|
|
1026
|
+
out = Ops.allocate_output(node.dtype, node.length, node.shape)
|
|
1027
|
+
key = Plan.derive_key(seed, node.attrs[:index])
|
|
1028
|
+
if node.attrs[:kind] == :normal
|
|
1029
|
+
Random.normal!(out, mean: node.attrs[:mean], stddev: node.attrs[:stddev], seed: key)
|
|
1030
|
+
else
|
|
1031
|
+
Random.uniform!(out, seed: key)
|
|
1032
|
+
end
|
|
1033
|
+
out
|
|
1034
|
+
end
|
|
1035
|
+
|
|
1036
|
+
# Calls the same NN function that was recorded. There is no graph-specific
|
|
1037
|
+
# implementation, so values cannot drift from eager (two diverging copies
|
|
1038
|
+
# of the math is the biggest risk #17 named).
|
|
1039
|
+
def run_nn(node, inputs, _bindings)
|
|
1040
|
+
arity = node.attrs[:arity]
|
|
1041
|
+
kwargs = node.attrs[:opts].merge(node.attrs[:keys].zip(inputs.drop(arity)).to_h)
|
|
1042
|
+
result = NN.public_send(node.attrs[:fn], *inputs.first(arity), *node.attrs[:args], **kwargs)
|
|
1043
|
+
check_result!(result, node, "#{node.attrs[:fn]} node")
|
|
1044
|
+
end
|
|
1045
|
+
|
|
1046
|
+
# Calls the delegate (cuDNN / MPSGraph) as is. No graph-specific
|
|
1047
|
+
# implementation, so values cannot drift from eager (same idea as :nn).
|
|
1048
|
+
def run_conv2d(node, inputs, _bindings)
|
|
1049
|
+
result = Conv.forward(inputs[0], inputs[1], pad: node.attrs[:pad], stride: node.attrs[:stride],
|
|
1050
|
+
dilation: node.attrs[:dilation], groups: node.attrs[:groups])
|
|
1051
|
+
check_result!(result, node, "conv2d node")
|
|
1052
|
+
end
|
|
1053
|
+
|
|
1054
|
+
# Backward calls the delegate as is too (same idea as run_conv2d — no
|
|
1055
|
+
# graph-specific implementation, so no drift from eager). inputs[2] is
|
|
1056
|
+
# there only for its shape.
|
|
1057
|
+
def run_conv2d_backward_data(node, inputs, _bindings)
|
|
1058
|
+
result = Conv.backward_data(inputs[0], inputs[1], inputs[2],
|
|
1059
|
+
pad: node.attrs[:pad], stride: node.attrs[:stride],
|
|
1060
|
+
dilation: node.attrs[:dilation], groups: node.attrs[:groups])
|
|
1061
|
+
check_result!(result, node, "conv2d backward data node")
|
|
1062
|
+
end
|
|
1063
|
+
|
|
1064
|
+
def run_conv2d_backward_filter(node, inputs, _bindings)
|
|
1065
|
+
result = Conv.backward_filter(inputs[0], inputs[1], inputs[2],
|
|
1066
|
+
pad: node.attrs[:pad], stride: node.attrs[:stride],
|
|
1067
|
+
dilation: node.attrs[:dilation], groups: node.attrs[:groups])
|
|
1068
|
+
check_result!(result, node, "conv2d backward filter node")
|
|
1069
|
+
end
|
|
1070
|
+
|
|
1071
|
+
def run_opaque(node, inputs, _bindings)
|
|
1072
|
+
result = node.attrs[:body].call(*inputs)
|
|
1073
|
+
check_result!(result, node, "opaque node #{node.attrs[:name].inspect}")
|
|
1074
|
+
end
|
|
1075
|
+
|
|
1076
|
+
def check_result!(result, node, what)
|
|
1077
|
+
unless result.is_a?(Buffer) && result.dtype == node.dtype && result.shape == node.shape
|
|
1078
|
+
got = result.is_a?(Buffer) ? "#{result.shape.inspect} #{result.dtype}" : result.class.to_s
|
|
1079
|
+
raise Error, "#{what} declared #{node.shape.inspect} #{node.dtype} but returned #{got}"
|
|
1080
|
+
end
|
|
1081
|
+
|
|
1082
|
+
result
|
|
1083
|
+
end
|
|
1084
|
+
end
|
|
1085
|
+
end
|
|
1086
|
+
end
|