tilework 0.18.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE +21 -0
- data/lib/tilework/backend/cpu/dsl_codegen.rb +509 -0
- data/lib/tilework/backend/cpu/ops_codegen.rb +451 -0
- data/lib/tilework/backend/cpu.rb +156 -0
- data/lib/tilework/buffer.rb +753 -0
- data/lib/tilework/capabilities.rb +183 -0
- data/lib/tilework/compat/cuda.h +66 -0
- data/lib/tilework/compat/hip.h +35 -0
- data/lib/tilework/compat/metal.h +45 -0
- data/lib/tilework/conformance.rb +2296 -0
- data/lib/tilework/core_ext.rb +21 -0
- data/lib/tilework/dialect.rb +67 -0
- data/lib/tilework/dsl.rb +722 -0
- data/lib/tilework/dtype/bfloat.rb +31 -0
- data/lib/tilework/dtype/cast.rb +88 -0
- data/lib/tilework/dtype/half.rb +98 -0
- data/lib/tilework/graph/builder.rb +1073 -0
- data/lib/tilework/graph/fusion.rb +520 -0
- data/lib/tilework/graph/grad.rb +443 -0
- data/lib/tilework/graph/high_level.rb +266 -0
- data/lib/tilework/graph/kernels.rb +403 -0
- data/lib/tilework/graph/node.rb +27 -0
- data/lib/tilework/graph/op.rb +290 -0
- data/lib/tilework/graph/optim.rb +183 -0
- data/lib/tilework/graph/plan.rb +1086 -0
- data/lib/tilework/graph/pool.rb +178 -0
- data/lib/tilework/graph/region.rb +210 -0
- data/lib/tilework/graph/residency.rb +449 -0
- data/lib/tilework/graph/state.rb +64 -0
- data/lib/tilework/graph/tensor.rb +124 -0
- data/lib/tilework/graph.rb +213 -0
- data/lib/tilework/kernel.rb +163 -0
- data/lib/tilework/math/conv.rb +270 -0
- data/lib/tilework/math/fft.rb +172 -0
- data/lib/tilework/math/nn.rb +1103 -0
- data/lib/tilework/math/philox.rb +194 -0
- data/lib/tilework/math/random.rb +232 -0
- data/lib/tilework/math/tensor_core.rb +269 -0
- data/lib/tilework/ops.rb +682 -0
- data/lib/tilework/runtime/offload.rb +498 -0
- data/lib/tilework/runtime/submission.rb +163 -0
- data/lib/tilework/runtime/telemetry.rb +112 -0
- data/lib/tilework/scope.rb +33 -0
- data/lib/tilework/version.rb +5 -0
- data/lib/tilework.rb +345 -0
- metadata +92 -0
|
@@ -0,0 +1,1073 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Tilework
|
|
4
|
+
module Graph
|
|
5
|
+
# Records graph inputs, operations, mutable state, and outputs without touching a device.
|
|
6
|
+
class Builder
|
|
7
|
+
# float16 appears **only as a storage type**, as in Ops: arithmetic always
|
|
8
|
+
# widens to float and rounds only on write-back. The rule differs per
|
|
9
|
+
# expression and is mirrored in result_dtype / unary_math / reduce /
|
|
10
|
+
# matmul — if this drifts from Ops, the same expression gets a different
|
|
11
|
+
# type only in the graph.
|
|
12
|
+
#
|
|
13
|
+
# Whether a dtype can be allocated depends on the backend, so only typing
|
|
14
|
+
# is checked here; Plan checks against capabilities[:dtypes] at compile time.
|
|
15
|
+
DTYPES = %i[float32 int32 float16 bfloat16].freeze
|
|
16
|
+
|
|
17
|
+
attr_reader :nodes, :outputs, :states, :writers
|
|
18
|
+
|
|
19
|
+
def initialize(differentiable: false)
|
|
20
|
+
# A differentiable graph records high-level ops as their decompositions,
|
|
21
|
+
# since gradients follow from primitive vjps alone (docs/autograd.md).
|
|
22
|
+
@differentiable = differentiable
|
|
23
|
+
@nodes = []
|
|
24
|
+
@cse = {}
|
|
25
|
+
@outputs = []
|
|
26
|
+
@inputs = {}
|
|
27
|
+
@states = []
|
|
28
|
+
# Version node id -> nodes overwriting that version; for linearity checks and WAR edges.
|
|
29
|
+
@writers = Hash.new { |h, k| h[k] = [] }
|
|
30
|
+
@random_draws = 0
|
|
31
|
+
# Regions rebuilt for backward (g.checkpoint).
|
|
32
|
+
@checkpoints = []
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
# --- Inputs and constants ---
|
|
36
|
+
|
|
37
|
+
# Declares a named value supplied to {Plan#run}.
|
|
38
|
+
#
|
|
39
|
+
# @param name [Symbol, String] binding name
|
|
40
|
+
# @param shape [Array<Integer>] logical tensor shape
|
|
41
|
+
# @param dtype [:float32, :float16, :int32]
|
|
42
|
+
# @return [Tensor]
|
|
43
|
+
def input(name, shape, dtype: :float32)
|
|
44
|
+
name = name.to_sym
|
|
45
|
+
raise Error, "input #{name.inspect} is already declared" if @inputs.key?(name)
|
|
46
|
+
# plan.run(seed:) feeds RNG keys, so :seed cannot be an input name.
|
|
47
|
+
raise Error, "input name :seed is reserved for the plan's RNG seed" if name == :seed
|
|
48
|
+
|
|
49
|
+
check_dtype!(dtype)
|
|
50
|
+
node = record(:input, attrs: { name: name }, dtype: dtype, shape: normalize_shape(shape))
|
|
51
|
+
@inputs[name] = node
|
|
52
|
+
Tensor.new(self, node)
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
# Binds an existing buffer as an immutable graph constant.
|
|
56
|
+
#
|
|
57
|
+
# @param buffer [Buffer]
|
|
58
|
+
# @return [Tensor]
|
|
59
|
+
def const(buffer)
|
|
60
|
+
raise TypeError, "const requires a Tilework::Buffer, got #{buffer.class}" unless buffer.is_a?(Buffer)
|
|
61
|
+
|
|
62
|
+
check_dtype!(buffer.dtype)
|
|
63
|
+
node = record(:const, attrs: { buffer: buffer }, dtype: buffer.dtype, shape: buffer.shape,
|
|
64
|
+
cse: [:const, buffer.object_id])
|
|
65
|
+
Tensor.new(self, node)
|
|
66
|
+
end
|
|
67
|
+
alias param const
|
|
68
|
+
|
|
69
|
+
# --- Mutable state ---
|
|
70
|
+
|
|
71
|
+
# Declares a buffer as mutable state that survives across plan runs.
|
|
72
|
+
#
|
|
73
|
+
# @param buffer [Buffer]
|
|
74
|
+
# @param name [Symbol, String, nil] diagnostic name
|
|
75
|
+
# @return [StateRef] version-zero state reference
|
|
76
|
+
def state(buffer, name: nil)
|
|
77
|
+
raise TypeError, "state requires a Tilework::Buffer, got #{buffer.class}" unless buffer.is_a?(Buffer)
|
|
78
|
+
|
|
79
|
+
check_dtype!(buffer.dtype)
|
|
80
|
+
st = State.new(@states.length, buffer, name)
|
|
81
|
+
@states << st
|
|
82
|
+
node = record(:state_init, attrs: { state: st, version: 0 }, dtype: st.dtype, shape: st.shape)
|
|
83
|
+
ref = StateRef.new(self, st, 0, node)
|
|
84
|
+
st.leaves << ref
|
|
85
|
+
ref
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
# Records a write and returns a new state version.
|
|
89
|
+
#
|
|
90
|
+
# @param ref [StateRef]
|
|
91
|
+
# @param value [Tensor]
|
|
92
|
+
# @param at [Integer, Range, Region, Index, nil] target region
|
|
93
|
+
# @return [StateRef]
|
|
94
|
+
#
|
|
95
|
+
# kv1 = g.write_into(kv0, at: g.index(:pos), value: k_new)
|
|
96
|
+
#
|
|
97
|
+
# at: takes an Integer / Range (outermost dim), a Region, or a run-time
|
|
98
|
+
# index. One version may be written twice only when the regions are
|
|
99
|
+
# **provably disjoint** (they must then be rejoined with g.merge).
|
|
100
|
+
def write_into(ref, value:, at: nil)
|
|
101
|
+
ref = check_ref!(ref)
|
|
102
|
+
region = to_region(ref, at)
|
|
103
|
+
value = check_tensor!(value, "write_into value")
|
|
104
|
+
st = ref.state
|
|
105
|
+
|
|
106
|
+
unless value.dtype == st.dtype
|
|
107
|
+
raise Error, "cannot write a #{value.dtype} value into #{st.name} (#{st.dtype})"
|
|
108
|
+
end
|
|
109
|
+
unless value.length == region.length
|
|
110
|
+
raise Error, "value has #{value.length} elements but #{region.inspect} covers #{region.length}"
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
@writers[ref.node.id].each do |other|
|
|
114
|
+
next if region.disjoint?(other.region)
|
|
115
|
+
|
|
116
|
+
raise Error, "#{st.name} v#{ref.version} is written twice by regions that may overlap " \
|
|
117
|
+
"(#{other.region.inspect} and #{region.inspect}); a version may only fork into " \
|
|
118
|
+
"provably disjoint writes"
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
version = st.next_version
|
|
122
|
+
node = record(:state_write, inputs: [ref.node, value.node],
|
|
123
|
+
attrs: { state: st, region: region, version: version },
|
|
124
|
+
dtype: st.dtype, shape: st.shape)
|
|
125
|
+
@writers[ref.node.id] << node
|
|
126
|
+
st.leaves.delete_if { |leaf| leaf.node.equal?(ref.node) }
|
|
127
|
+
StateRef.new(self, st, version, node).tap { |new_ref| st.leaves << new_ref }
|
|
128
|
+
end
|
|
129
|
+
|
|
130
|
+
# Reads all or part of a state version as a tensor.
|
|
131
|
+
#
|
|
132
|
+
# @param ref [StateRef]
|
|
133
|
+
# @param at [Integer, Range, Region, Index, nil]
|
|
134
|
+
# @return [Tensor]
|
|
135
|
+
def read(ref, at = nil)
|
|
136
|
+
ref = check_ref!(ref)
|
|
137
|
+
region = to_region(ref, at)
|
|
138
|
+
node = record(:state_read, inputs: [ref.node],
|
|
139
|
+
attrs: { state: ref.state, region: region, version: ref.version },
|
|
140
|
+
dtype: ref.state.dtype, shape: region.shape,
|
|
141
|
+
# The version is part of the value's identity; that alone
|
|
142
|
+
# keeps reads on either side of a write from merging in CSE.
|
|
143
|
+
cse: [:state_read, ref.state.id, ref.version, region.key])
|
|
144
|
+
Tensor.new(self, node)
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
# Rejoins writes to disjoint regions. **It writes no memory** (it only
|
|
148
|
+
# places a join point in the ordering), so it carries no WAR constraint.
|
|
149
|
+
def merge(*refs)
|
|
150
|
+
refs = refs.flatten.map { |r| check_ref!(r) }
|
|
151
|
+
raise ArgumentError, "merge needs at least two versions" if refs.length < 2
|
|
152
|
+
|
|
153
|
+
st = refs.first.state
|
|
154
|
+
unless refs.all? { |r| r.state.equal?(st) }
|
|
155
|
+
raise Error, "cannot merge versions of different states (#{refs.map { |r| r.state.name }.uniq.join(', ')})"
|
|
156
|
+
end
|
|
157
|
+
refs.each do |r|
|
|
158
|
+
next if st.leaves.any? { |leaf| leaf.node.equal?(r.node) }
|
|
159
|
+
|
|
160
|
+
raise Error, "#{st.name} v#{r.version} is not a current version of #{st.name}; " \
|
|
161
|
+
"only the versions produced by the latest writes can be merged"
|
|
162
|
+
end
|
|
163
|
+
|
|
164
|
+
version = st.next_version
|
|
165
|
+
node = record(:state_merge, inputs: refs.map(&:node),
|
|
166
|
+
attrs: { state: st, version: version },
|
|
167
|
+
dtype: st.dtype, shape: st.shape)
|
|
168
|
+
st.leaves.reject! { |leaf| refs.any? { |r| r.node.equal?(leaf.node) } }
|
|
169
|
+
StateRef.new(self, st, version, node).tap { |new_ref| st.leaves << new_ref }
|
|
170
|
+
end
|
|
171
|
+
|
|
172
|
+
# Creates a runtime outer-dimension index binding.
|
|
173
|
+
#
|
|
174
|
+
# @param name [Symbol, String] binding name passed to {Plan#run}
|
|
175
|
+
# @param count [Integer] contiguous outer-dimension elements
|
|
176
|
+
# @return [Index]
|
|
177
|
+
#
|
|
178
|
+
# g.write_into(kv, at: g.index(:pos), value: k) # plan.run(pos: 7)
|
|
179
|
+
def index(name, count: 1)
|
|
180
|
+
Index.new(name.to_sym, count)
|
|
181
|
+
end
|
|
182
|
+
|
|
183
|
+
# --- Operations ---
|
|
184
|
+
|
|
185
|
+
def arithmetic(op, tensor, other, reversed: false)
|
|
186
|
+
case other
|
|
187
|
+
when Tensor
|
|
188
|
+
dtype = result_dtype(tensor.dtype, other.dtype)
|
|
189
|
+
binary_node(op, tensor, other, dtype: dtype, kind: :arith)
|
|
190
|
+
when Numeric
|
|
191
|
+
dtype = result_dtype(tensor.dtype, other)
|
|
192
|
+
scalar_node(op, tensor, other, dtype: dtype, kind: :arith, reversed: reversed)
|
|
193
|
+
else
|
|
194
|
+
raise TypeError, "cannot combine Tilework::Graph::Tensor with #{other.class}"
|
|
195
|
+
end
|
|
196
|
+
end
|
|
197
|
+
|
|
198
|
+
def compare(op, tensor, other)
|
|
199
|
+
case other
|
|
200
|
+
when Tensor then binary_node(op, tensor, other, dtype: :int32, kind: :compare)
|
|
201
|
+
when Numeric then scalar_node(op, tensor, other, dtype: :int32, kind: :compare, reversed: false)
|
|
202
|
+
else
|
|
203
|
+
raise TypeError, "cannot compare Tilework::Graph::Tensor with #{other.class}"
|
|
204
|
+
end
|
|
205
|
+
end
|
|
206
|
+
|
|
207
|
+
def unary_math(fn, tensor)
|
|
208
|
+
# As in Ops.unary_math, half stays half (the point is saving bandwidth,
|
|
209
|
+
# so widening to float32 here would defeat it).
|
|
210
|
+
dtype = Ops::LOW_PRECISION_DTYPES.include?(tensor.dtype) ? tensor.dtype : :float32
|
|
211
|
+
node = record(:unary, inputs: [tensor.node], attrs: { fn: fn },
|
|
212
|
+
dtype: dtype, shape: tensor.shape, cse: [:unary, fn, dtype])
|
|
213
|
+
Tensor.new(self, node)
|
|
214
|
+
end
|
|
215
|
+
|
|
216
|
+
def unary_abs(tensor)
|
|
217
|
+
node = record(:unary, inputs: [tensor.node], attrs: { fn: :abs },
|
|
218
|
+
dtype: tensor.dtype, shape: tensor.shape, cse: [:unary, :abs, tensor.dtype])
|
|
219
|
+
Tensor.new(self, node)
|
|
220
|
+
end
|
|
221
|
+
|
|
222
|
+
# Reductions. With axis: only that axis folds; without it everything folds
|
|
223
|
+
# to shape [1].
|
|
224
|
+
#
|
|
225
|
+
# Eager `buffer.sum` returns a host number, which would end the plan
|
|
226
|
+
# there (reading back to the host drains the queue). A graph needs to
|
|
227
|
+
# carry it as a value, so a whole-buffer reduction takes Ops' device: true
|
|
228
|
+
# path and an axis reduction a kernel with one thread per output element.
|
|
229
|
+
def reduce(kind, tensor, axis: nil, keepdims: false)
|
|
230
|
+
# As in Ops.accumulator_dtype: never accumulate in half (around 2048
|
|
231
|
+
# adding stops increasing the sum).
|
|
232
|
+
dtype = kind == :mean || Ops::LOW_PRECISION_DTYPES.include?(tensor.dtype) ? :float32 : tensor.dtype
|
|
233
|
+
reduce_node(:reduce, kind, tensor, axis, keepdims, dtype)
|
|
234
|
+
end
|
|
235
|
+
|
|
236
|
+
# A reduction returning positions rather than values, hence int32.
|
|
237
|
+
def arg_reduce(kind, tensor, axis: nil, keepdims: false)
|
|
238
|
+
reduce_node(:arg_reduce, kind, tensor, axis, keepdims, :int32)
|
|
239
|
+
end
|
|
240
|
+
|
|
241
|
+
# Cast. It **only changes the storage type**, not the meaning (float ->
|
|
242
|
+
# int truncates toward zero as in C). float16 appears only as a storage
|
|
243
|
+
# type, as in Ops, so this is for placing conversions explicitly at
|
|
244
|
+
# boundaries, not in the middle of computation.
|
|
245
|
+
def cast(tensor, dtype)
|
|
246
|
+
tensor = check_tensor!(tensor, "cast")
|
|
247
|
+
check_dtype!(dtype)
|
|
248
|
+
return tensor if tensor.dtype == dtype
|
|
249
|
+
|
|
250
|
+
node = record(:cast, inputs: [tensor.node], attrs: {}, dtype: dtype,
|
|
251
|
+
shape: tensor.shape, cse: [:cast, dtype])
|
|
252
|
+
Tensor.new(self, node)
|
|
253
|
+
end
|
|
254
|
+
|
|
255
|
+
# Select by condition. cond is a 0/1 mask (a comparison's output as is).
|
|
256
|
+
#
|
|
257
|
+
# g.where(x > 0, x, 0.0) # relu
|
|
258
|
+
# g.where(mask.eq(0), scores, -1e9)
|
|
259
|
+
#
|
|
260
|
+
# a / b may be Tensors or scalars. All three operands broadcast by the
|
|
261
|
+
# same rule as binary ops.
|
|
262
|
+
def where(cond, a, b)
|
|
263
|
+
cond = check_tensor!(cond, "where condition")
|
|
264
|
+
dtype = where_dtype(a, b)
|
|
265
|
+
shape = where_shape!(cond, a, b)
|
|
266
|
+
|
|
267
|
+
attrs = {}
|
|
268
|
+
inputs = [broadcast(cond, shape).node]
|
|
269
|
+
{ a: a, b: b }.each do |key, value|
|
|
270
|
+
if value.is_a?(Numeric)
|
|
271
|
+
attrs[:"#{key}_scalar"] = dtype == :int32 ? value.to_i : value.to_f
|
|
272
|
+
else
|
|
273
|
+
inputs << broadcast(check_tensor!(value, "where #{key}"), shape).node
|
|
274
|
+
end
|
|
275
|
+
end
|
|
276
|
+
|
|
277
|
+
node = record(:where, inputs: inputs, attrs: attrs, dtype: dtype, shape: shape,
|
|
278
|
+
cse: [:where, dtype, attrs])
|
|
279
|
+
Tensor.new(self, node)
|
|
280
|
+
end
|
|
281
|
+
|
|
282
|
+
# The index along an axis itself, so masks and position embeddings can be
|
|
283
|
+
# built inside the graph without bringing constants from the host.
|
|
284
|
+
#
|
|
285
|
+
# pos = g.iota([n, n], axis: 0)
|
|
286
|
+
# causal = g.where(pos >= g.iota([n, n], axis: 1), scores, -1e9)
|
|
287
|
+
def iota(shape, axis: -1, dtype: :int32)
|
|
288
|
+
shape = normalize_shape(shape)
|
|
289
|
+
unless %i[int32 float32].include?(dtype)
|
|
290
|
+
raise Error, "iota supports :int32 or :float32 (got #{dtype.inspect})"
|
|
291
|
+
end
|
|
292
|
+
|
|
293
|
+
axis = normalize_axis!(axis, shape)
|
|
294
|
+
node = record(:iota, attrs: { axis: axis }, dtype: dtype, shape: shape,
|
|
295
|
+
cse: [:iota, axis, dtype, shape])
|
|
296
|
+
Tensor.new(self, node)
|
|
297
|
+
end
|
|
298
|
+
|
|
299
|
+
# --- Shape operations ---
|
|
300
|
+
|
|
301
|
+
# Explicit broadcast. **The rule is NumPy's right alignment**: dims match
|
|
302
|
+
# or one of them is 1.
|
|
303
|
+
#
|
|
304
|
+
# It is inserted as a node at record time, so downstream (fusion,
|
|
305
|
+
# placement, liveness) need not know the implicit rule — they see one
|
|
306
|
+
# ordinary node. Fusion folds it into the consumer's index where it can;
|
|
307
|
+
# otherwise it is materialized.
|
|
308
|
+
def broadcast(tensor, shape)
|
|
309
|
+
tensor = check_tensor!(tensor, "broadcast")
|
|
310
|
+
shape = normalize_shape(shape)
|
|
311
|
+
return tensor if tensor.shape == shape
|
|
312
|
+
|
|
313
|
+
unless broadcastable?(tensor.shape, shape)
|
|
314
|
+
raise ArgumentError, "cannot broadcast #{tensor.shape.inspect} to #{shape.inspect}"
|
|
315
|
+
end
|
|
316
|
+
|
|
317
|
+
node = record(:broadcast, inputs: [tensor.node], attrs: {}, dtype: tensor.dtype,
|
|
318
|
+
shape: shape, cse: [:broadcast, shape])
|
|
319
|
+
Tensor.new(self, node)
|
|
320
|
+
end
|
|
321
|
+
|
|
322
|
+
# Axis permutation. axes lists, for each output dim d, which input dim it is.
|
|
323
|
+
#
|
|
324
|
+
# x.permute(1, 0) # 2D transpose
|
|
325
|
+
# x.permute(1, 0, 2) # [n, h, dh] -> [h, n, dh]
|
|
326
|
+
def permute(tensor, *axes)
|
|
327
|
+
tensor = check_tensor!(tensor, "permute")
|
|
328
|
+
axes = axes.first if axes.length == 1 && axes.first.is_a?(Array)
|
|
329
|
+
rank = tensor.shape.length
|
|
330
|
+
unless axes.sort == (0...rank).to_a
|
|
331
|
+
raise ArgumentError, "permute needs each axis of #{tensor.shape.inspect} exactly once, " \
|
|
332
|
+
"got #{axes.inspect}"
|
|
333
|
+
end
|
|
334
|
+
return tensor if axes == (0...rank).to_a
|
|
335
|
+
|
|
336
|
+
shape = axes.map { |axis| tensor.shape[axis] }
|
|
337
|
+
node = record(:permute, inputs: [tensor.node], attrs: { axes: axes }, dtype: tensor.dtype,
|
|
338
|
+
shape: shape, cse: [:permute, axes])
|
|
339
|
+
Tensor.new(self, node)
|
|
340
|
+
end
|
|
341
|
+
|
|
342
|
+
# A slice of the outermost dim. **No kernel runs** — as with Buffer#slice
|
|
343
|
+
# the slice is contiguous, so a view expresses it. Slicing an inner dim is
|
|
344
|
+
# not contiguous, so it is not accepted here (docs/graph-ir.md).
|
|
345
|
+
def slice(tensor, range)
|
|
346
|
+
tensor = check_tensor!(tensor, "slice")
|
|
347
|
+
raise Error, "slice needs at least 2 dimensions, got #{tensor.shape.inspect}" if tensor.shape.length < 2
|
|
348
|
+
|
|
349
|
+
first, count = slice_range!(range, tensor.shape[0])
|
|
350
|
+
return tensor if first.zero? && count == tensor.shape[0]
|
|
351
|
+
|
|
352
|
+
shape = [count, *tensor.shape.drop(1)]
|
|
353
|
+
node = record(:slice, inputs: [tensor.node], attrs: { offset: first * shape.drop(1).inject(1, :*) },
|
|
354
|
+
dtype: tensor.dtype, shape: shape, cse: [:slice, first, count])
|
|
355
|
+
Tensor.new(self, node)
|
|
356
|
+
end
|
|
357
|
+
|
|
358
|
+
# Table lookup. indices is an int32 Tensor whose elements point at
|
|
359
|
+
# positions along axis.
|
|
360
|
+
#
|
|
361
|
+
# g.gather(table, ids) # embedding: [vocab, d] indexed by [n] gives [n, d]
|
|
362
|
+
#
|
|
363
|
+
# Out-of-range indices are clamped to the ends (in the kernel).
|
|
364
|
+
def gather(source, indices, axis: 0)
|
|
365
|
+
source = check_tensor!(source, "gather source")
|
|
366
|
+
indices = check_tensor!(indices, "gather indices")
|
|
367
|
+
raise Error, "gather indices must be :int32, got #{indices.dtype.inspect}" unless indices.dtype == :int32
|
|
368
|
+
|
|
369
|
+
axis = normalize_axis!(axis, source.shape)
|
|
370
|
+
shape = source.shape.take(axis) + indices.shape + source.shape.drop(axis + 1)
|
|
371
|
+
node = record(:gather, inputs: [source.node, indices.node], attrs: { axis: axis },
|
|
372
|
+
dtype: source.dtype, shape: shape, cse: [:gather, axis])
|
|
373
|
+
Tensor.new(self, node)
|
|
374
|
+
end
|
|
375
|
+
|
|
376
|
+
# The inverse of table lookup (gather's vjp). shape is the shape added
|
|
377
|
+
# back into (gather's source).
|
|
378
|
+
#
|
|
379
|
+
# It has no forward counterpart, so it is not on the Tensor surface — only
|
|
380
|
+
# Grad's gather row uses it.
|
|
381
|
+
def scatter_add(indices, values, shape:, axis: 0)
|
|
382
|
+
indices = check_tensor!(indices, "scatter_add indices")
|
|
383
|
+
values = check_tensor!(values, "scatter_add values")
|
|
384
|
+
shape = normalize_shape(shape)
|
|
385
|
+
axis = normalize_axis!(axis, shape)
|
|
386
|
+
node = record(:scatter_add, inputs: [indices.node, values.node], attrs: { axis: axis },
|
|
387
|
+
dtype: values.dtype, shape: shape, cse: [:scatter_add, axis, shape])
|
|
388
|
+
Tensor.new(self, node)
|
|
389
|
+
end
|
|
390
|
+
|
|
391
|
+
# Fills a border. before / after are the amounts added per dim.
|
|
392
|
+
#
|
|
393
|
+
# g.pad(x, before: [0, 1], after: [0, 1], value: 0.0)
|
|
394
|
+
def pad(tensor, before:, after: nil, value: 0.0)
|
|
395
|
+
tensor = check_tensor!(tensor, "pad")
|
|
396
|
+
rank = tensor.shape.length
|
|
397
|
+
before = Array.new(rank, before) if before.is_a?(Integer)
|
|
398
|
+
after = after.nil? ? before : (after.is_a?(Integer) ? Array.new(rank, after) : after)
|
|
399
|
+
unless before.length == rank && after.length == rank &&
|
|
400
|
+
(before + after).all? { |v| v.is_a?(Integer) && !v.negative? }
|
|
401
|
+
raise ArgumentError, "pad needs a non-negative Integer per dimension of #{tensor.shape.inspect}"
|
|
402
|
+
end
|
|
403
|
+
return tensor if (before + after).all?(&:zero?)
|
|
404
|
+
|
|
405
|
+
shape = tensor.shape.each_with_index.map { |dim, d| dim + before[d] + after[d] }
|
|
406
|
+
value = tensor.dtype == :int32 ? value.to_i : value.to_f
|
|
407
|
+
node = record(:pad, inputs: [tensor.node], attrs: { before: before, value: value },
|
|
408
|
+
dtype: tensor.dtype, shape: shape, cse: [:pad, before, after, value])
|
|
409
|
+
Tensor.new(self, node)
|
|
410
|
+
end
|
|
411
|
+
|
|
412
|
+
# Changes only the shape, keeping element order. **No kernel runs** — at
|
|
413
|
+
# run time it is a view of the input buffer (like Buffer#reshape). A
|
|
414
|
+
# single -1 is inferred.
|
|
415
|
+
def reshape(tensor, *dims)
|
|
416
|
+
tensor = check_tensor!(tensor, "reshape")
|
|
417
|
+
dims = dims.first if dims.length == 1 && dims.first.is_a?(Array)
|
|
418
|
+
shape = resolve_reshape(tensor, dims)
|
|
419
|
+
return tensor if tensor.shape == shape
|
|
420
|
+
|
|
421
|
+
node = record(:reshape, inputs: [tensor.node], attrs: {}, dtype: tensor.dtype,
|
|
422
|
+
shape: shape, cse: [:reshape, shape])
|
|
423
|
+
Tensor.new(self, node)
|
|
424
|
+
end
|
|
425
|
+
|
|
426
|
+
# Accepts **batches** as well as 2D pairs:
|
|
427
|
+
#
|
|
428
|
+
# [m, k] x [k, n] -> [m, n] same as eager matmul
|
|
429
|
+
# [b, m, k] x [b, k, n] -> [b, m, n] one per batch element
|
|
430
|
+
#
|
|
431
|
+
# The latter has no eager counterpart (`Buffer#matmul` is 2D only). It
|
|
432
|
+
# becomes one strided batched GEMM when available and per-batch matmuls
|
|
433
|
+
# otherwise; conformance checks that the path does not change values.
|
|
434
|
+
def matmul(a, b, out_dtype: nil)
|
|
435
|
+
b = check_tensor!(b, "matmul")
|
|
436
|
+
mode, shape = matmul_shape(a, b)
|
|
437
|
+
out_dtype ||= promote(a.dtype, b.dtype)
|
|
438
|
+
allowed = [promote(a.dtype, b.dtype)]
|
|
439
|
+
allowed << a.dtype if a.dtype == b.dtype && Ops::LOW_PRECISION_DTYPES.include?(a.dtype)
|
|
440
|
+
unless allowed.include?(out_dtype)
|
|
441
|
+
raise ArgumentError, "matmul out_dtype must be #{allowed.map(&:inspect).join(' or ')} " \
|
|
442
|
+
"for #{a.dtype.inspect} x #{b.dtype.inspect} inputs"
|
|
443
|
+
end
|
|
444
|
+
node = record(:matmul, inputs: [a.node, b.node], attrs: { mode: mode, out_dtype: out_dtype },
|
|
445
|
+
dtype: out_dtype, shape: shape, cse: [:matmul, mode, out_dtype])
|
|
446
|
+
Tensor.new(self, node)
|
|
447
|
+
end
|
|
448
|
+
|
|
449
|
+
# Concatenation along axis:. Along the outermost dim each piece lands
|
|
450
|
+
# contiguously in the output and one copy per piece suffices; along an
|
|
451
|
+
# inner dim pieces are scattered, so a kernel writes each into its slot.
|
|
452
|
+
def concat(*tensors, axis: 0)
|
|
453
|
+
tensors = tensors.flatten.map { |t| check_tensor!(t, "concat") }
|
|
454
|
+
raise ArgumentError, "concat needs at least two tensors, got #{tensors.length}" if tensors.length < 2
|
|
455
|
+
|
|
456
|
+
dtype = tensors.first.dtype
|
|
457
|
+
axis = normalize_axis!(axis, tensors.first.shape)
|
|
458
|
+
tensors.each do |t|
|
|
459
|
+
same = t.dtype == dtype && t.shape.length == tensors.first.shape.length &&
|
|
460
|
+
t.shape.each_with_index.all? { |dim, d| d == axis || dim == tensors.first.shape[d] }
|
|
461
|
+
next if same
|
|
462
|
+
|
|
463
|
+
raise ArgumentError, "concat needs the same dtype and the same dims outside axis #{axis}: " \
|
|
464
|
+
"#{tensors.map { |o| "#{o.shape.inspect} #{o.dtype}" }.join(' , ')}"
|
|
465
|
+
end
|
|
466
|
+
|
|
467
|
+
shape = tensors.first.shape.dup
|
|
468
|
+
shape[axis] = tensors.sum { |t| t.shape[axis] }
|
|
469
|
+
node = record(:concat, inputs: tensors.map(&:node), attrs: { axis: axis }, dtype: dtype,
|
|
470
|
+
shape: shape, cse: [:concat, dtype, axis])
|
|
471
|
+
Tensor.new(self, node)
|
|
472
|
+
end
|
|
473
|
+
|
|
474
|
+
# Built-ins the graph **does not decompose**, like NN's fused kernels. The
|
|
475
|
+
# recording entry point is Graph.record_nn (one line on the NN side), so
|
|
476
|
+
# this is not normally called directly.
|
|
477
|
+
#
|
|
478
|
+
# Unlike opaque, it knows by name what it is, so CSE applies and fusion can
|
|
479
|
+
# treat it as a partner. Weights passed as Buffers (gamma / bias ...) are
|
|
480
|
+
# lifted to const nodes like g.param, so writing a graph needs no rewrapping
|
|
481
|
+
# of weights.
|
|
482
|
+
def nn(fn, inputs, shape:, dtype:, tensors: {}, args: [], opts: {})
|
|
483
|
+
check_dtype!(dtype)
|
|
484
|
+
positional = inputs.map { |value| lift(value, "#{fn} input") }
|
|
485
|
+
keyed = tensors.reject { |_, value| value.nil? }
|
|
486
|
+
.to_h { |key, value| [key, lift(value, "#{fn} #{key}")] }
|
|
487
|
+
# A differentiable graph records decompositions — except that **functions
|
|
488
|
+
# with a hand-written vjp stay :nn**. They differentiate without
|
|
489
|
+
# decomposing, so there is no reason to give up the fused forward kernel
|
|
490
|
+
# (Grad::NN_REGISTRY).
|
|
491
|
+
if HighLevel.lower?(fn) || (@differentiable && HighLevel.defined?(fn) && !Grad.nn_defined?(fn))
|
|
492
|
+
return lowered(fn, positional, keyed, args, opts, shape, dtype)
|
|
493
|
+
end
|
|
494
|
+
|
|
495
|
+
node = record(:nn, inputs: (positional + keyed.values).map(&:node),
|
|
496
|
+
attrs: { fn: fn, arity: positional.length, keys: keyed.keys,
|
|
497
|
+
args: args, opts: opts },
|
|
498
|
+
dtype: dtype, shape: normalize_shape(shape),
|
|
499
|
+
cse: [:nn, fn, args, keyed.keys, opts])
|
|
500
|
+
Tensor.new(self, node)
|
|
501
|
+
end
|
|
502
|
+
|
|
503
|
+
# Expands a high-level op into primitives (Graph.lower_high_level).
|
|
504
|
+
# Whether to expand is decided **at record time**, so one plan never mixes
|
|
505
|
+
# the two.
|
|
506
|
+
#
|
|
507
|
+
# The declared shape and type are checked here, so a wrong decomposition
|
|
508
|
+
# fails at record time instead of surviving into execution or numeric
|
|
509
|
+
# comparison. Only the type is coerced — the hand-written kernel also
|
|
510
|
+
# computes in float and rounds to x's type on write-back, so the trailing
|
|
511
|
+
# cast reproduces that write-back.
|
|
512
|
+
def lowered(fn, inputs, tensors, args, opts, shape, dtype)
|
|
513
|
+
result = HighLevel.lower(self, fn, inputs, tensors: tensors, args: args, opts: opts)
|
|
514
|
+
result = check_tensor!(result, "the decomposition of #{fn}")
|
|
515
|
+
unless result.shape == shape
|
|
516
|
+
raise Error, "the decomposition of #{fn} produced #{result.shape.inspect} " \
|
|
517
|
+
"but #{fn} declares #{shape.inspect}"
|
|
518
|
+
end
|
|
519
|
+
|
|
520
|
+
cast(result, dtype)
|
|
521
|
+
end
|
|
522
|
+
|
|
523
|
+
# 2D convolution. Conv.forward calls this, so it is not normally called
|
|
524
|
+
# directly.
|
|
525
|
+
#
|
|
526
|
+
# Without convolution on this backend (caps.conv is nil) it is **expanded
|
|
527
|
+
# into primitives** at record time — only in the graph layer can
|
|
528
|
+
# convolution be written anywhere. The expansion is graph/high_level.rb's :conv2d.
|
|
529
|
+
def conv2d(input, weight, shape:, pad:, stride:, dilation: [1, 1], groups: 1)
|
|
530
|
+
input = lift(input, "conv2d input")
|
|
531
|
+
weight = lift(weight, "conv2d weight")
|
|
532
|
+
opts = { pad: pad, stride: stride, dilation: dilation, groups: groups }
|
|
533
|
+
if HighLevel.lower?(:conv2d) || !Tilework.capabilities.conv_dtypes.include?(input.dtype)
|
|
534
|
+
return lowered(:conv2d, [input, weight], {}, [], opts, shape, input.dtype)
|
|
535
|
+
end
|
|
536
|
+
|
|
537
|
+
node = record(:conv2d, inputs: [input.node, weight.node], attrs: opts,
|
|
538
|
+
dtype: input.dtype, shape: normalize_shape(shape),
|
|
539
|
+
cse: [:conv2d, pad, stride, dilation, groups])
|
|
540
|
+
Tensor.new(self, node)
|
|
541
|
+
end
|
|
542
|
+
|
|
543
|
+
# Convolution backward (#22 Step 2). Calls cuDNN / MPSGraph's dedicated
|
|
544
|
+
# kernels as is — when forward is delegated, delegating backward is
|
|
545
|
+
# fastest (differentiating the im2col decomposition doubles the expansion).
|
|
546
|
+
#
|
|
547
|
+
# like is an input used only for its shape (as in Conv.backward_data /
|
|
548
|
+
# backward_filter's arguments). It must be among the node's inputs, or
|
|
549
|
+
# liveness would release that buffer early.
|
|
550
|
+
def conv2d_backward(kind, primary, grad_output, like, pad:, stride:, dilation: [1, 1], groups: 1)
|
|
551
|
+
node = record(:"conv2d_backward_#{kind}",
|
|
552
|
+
inputs: [primary.node, grad_output.node, like.node],
|
|
553
|
+
attrs: { pad: pad, stride: stride, dilation: dilation, groups: groups },
|
|
554
|
+
dtype: like.dtype, shape: like.shape,
|
|
555
|
+
cse: [:"conv2d_backward_#{kind}", pad, stride, dilation, groups])
|
|
556
|
+
Tensor.new(self, node)
|
|
557
|
+
end
|
|
558
|
+
|
|
559
|
+
# --- Random numbers ---
|
|
560
|
+
#
|
|
561
|
+
# Counter-based RNG (Philox) takes its key as an argument, so **deriving
|
|
562
|
+
# each node's key from the plan seed and a static index makes it pure**:
|
|
563
|
+
# independent of execution order, same seed same sequence. That is why
|
|
564
|
+
# RNG is not mutable state (g.state) — #19 §6's conclusion.
|
|
565
|
+
#
|
|
566
|
+
# plan.run(x: buf, seed: 42)
|
|
567
|
+
#
|
|
568
|
+
# The same expression must draw **different values each time**, so draws
|
|
569
|
+
# are exempt from CSE.
|
|
570
|
+
def random(shape, dtype: :float32) = random_node(:uniform, shape, dtype, {})
|
|
571
|
+
|
|
572
|
+
def randn(shape, mean: 0.0, stddev: 1.0, dtype: :float32)
|
|
573
|
+
random_node(:normal, shape, dtype, { mean: mean.to_f, stddev: stddev.to_f })
|
|
574
|
+
end
|
|
575
|
+
|
|
576
|
+
# How many draws the plan makes; the step for advancing the seed (Plan#next_seed).
|
|
577
|
+
def random_draws = @random_draws
|
|
578
|
+
|
|
579
|
+
# A node whose contents are not inspected: the exit for things the graph
|
|
580
|
+
# cannot (or must not) decompose, like Tilework.kernel {} DSL kernels or
|
|
581
|
+
# NN's fused functions. body takes the input Buffers and returns a Buffer
|
|
582
|
+
# as declared.
|
|
583
|
+
#
|
|
584
|
+
# By default it synchronizes on both sides of the block (what it does is
|
|
585
|
+
# unknown). When **the caller can declare** the block stays inside our
|
|
586
|
+
# queue via `Ops.launch_kernel`, passing `fenced: false` removes the fence
|
|
587
|
+
# so it joins its neighbors' submission (#86). Never pass it for a block
|
|
588
|
+
# calling a library that creates its own command buffer / stream.
|
|
589
|
+
def opaque(*inputs, shape:, dtype: :float32, name: "opaque", fenced: true, &body)
|
|
590
|
+
raise ArgumentError, "opaque requires a block" if body.nil?
|
|
591
|
+
|
|
592
|
+
check_dtype!(dtype)
|
|
593
|
+
inputs = inputs.map { |t| check_tensor!(t, "opaque input") }
|
|
594
|
+
node = record(:opaque, inputs: inputs.map(&:node), attrs: { name: name, body: body, fenced: fenced },
|
|
595
|
+
dtype: dtype, shape: normalize_shape(shape))
|
|
596
|
+
Tensor.new(self, node)
|
|
597
|
+
end
|
|
598
|
+
|
|
599
|
+
# A tensor filled with one value. To **avoid bringing a buffer from the
|
|
600
|
+
# host** it adds iota multiplied by 0 (both nodes are elementwise, so
|
|
601
|
+
# fusion folds them into one). Used for the backward seed (the loss
|
|
602
|
+
# cotangent 1.0) and zero gradients of parameters the loss does not use.
|
|
603
|
+
def fill(shape, value, dtype: :float32)
|
|
604
|
+
base = (iota(shape, axis: 0, dtype: :float32) * 0.0) + value.to_f
|
|
605
|
+
dtype == :float32 ? base : cast(base, dtype)
|
|
606
|
+
end
|
|
607
|
+
|
|
608
|
+
# --- checkpointing (#22 Step 5) -----------------------------------------
|
|
609
|
+
|
|
610
|
+
# Does **not keep** intermediates made inside the block for backward;
|
|
611
|
+
# instead the same computation is recorded again when backward is recorded.
|
|
612
|
+
#
|
|
613
|
+
# h = g.checkpoint { Tilework::NN.gelu(x.matmul(w1)).matmul(w2) }
|
|
614
|
+
#
|
|
615
|
+
# Backward is a peak-memory problem, and its cause is "forward
|
|
616
|
+
# intermediates live to the end" (vjps read them, so liveness rightly
|
|
617
|
+
# extends them). With checkpoint, the region's intermediates are read only
|
|
618
|
+
# inside forward, so **they die when the region ends**; backward reads the
|
|
619
|
+
# rebuilt values.
|
|
620
|
+
#
|
|
621
|
+
# **In a graph, "recomputation" is not a run-time mechanism, just more
|
|
622
|
+
# nodes.** The region's nodes are duplicated under new ids and backward's
|
|
623
|
+
# inputs are switched to them. The copies bypass CSE (going through it
|
|
624
|
+
# would fold them back onto the originals, doing nothing).
|
|
625
|
+
#
|
|
626
|
+
# Three kinds are never duplicated: opaque (its contents are unknown —
|
|
627
|
+
# rebuilding reruns the user's block), state reads/writes (memory, not
|
|
628
|
+
# values) and inputs/constants (already live). Their original values stay
|
|
629
|
+
# alive, so correctness is unaffected; only the saving shrinks.
|
|
630
|
+
#
|
|
631
|
+
# **Random draws can be duplicated.** The key comes from (plan seed, the
|
|
632
|
+
# node's static index) and the index lives in attrs, so a copy **redraws
|
|
633
|
+
# the same sequence** — #19's choice not to make RNG mutable state is what
|
|
634
|
+
# guarantees recomputation gives the same values. (Assigning indices
|
|
635
|
+
# dynamically by recording order would break this silently.)
|
|
636
|
+
NEVER_RECOMPUTE = %i[input const opaque state_init state_read
|
|
637
|
+
state_write state_merge].freeze
|
|
638
|
+
|
|
639
|
+
Checkpoint = Struct.new(:from, :to)
|
|
640
|
+
|
|
641
|
+
def checkpoint
|
|
642
|
+
from = @nodes.length
|
|
643
|
+
result = check_tensor!(yield, "the checkpoint block")
|
|
644
|
+
@checkpoints << Checkpoint.new(from, @nodes.length)
|
|
645
|
+
result
|
|
646
|
+
end
|
|
647
|
+
|
|
648
|
+
# --- Backward (#22) -----------------------------------------------------
|
|
649
|
+
|
|
650
|
+
# Records the gradients of loss with respect to each wrt: tensor **as nodes
|
|
651
|
+
# of the same graph**. They come back as ordinary Tensors, usable as outputs
|
|
652
|
+
# or for writing back to state (parameter updates).
|
|
653
|
+
#
|
|
654
|
+
# dw, db = g.grad(loss, wrt: [w, b])
|
|
655
|
+
# g.write_into(w_ref, value: g.read(w_ref) - (lr * dw))
|
|
656
|
+
#
|
|
657
|
+
# Walking **ids in descending order** is enough for the reverse pass:
|
|
658
|
+
# inputs are always recorded before their consumers, so ascending id
|
|
659
|
+
# order is itself a topological order.
|
|
660
|
+
#
|
|
661
|
+
# High-level ops (:nn) differentiate only as decompositions, so graphs to
|
|
662
|
+
# differentiate are recorded with `Graph.build(differentiable: true)`.
|
|
663
|
+
def grad(loss, wrt:)
|
|
664
|
+
loss = check_tensor!(loss, "grad")
|
|
665
|
+
unless loss.length == 1
|
|
666
|
+
raise Error, "grad requires a scalar loss (shape [1]), got #{loss.shape.inspect}; " \
|
|
667
|
+
"reduce it first (for example loss.mean)"
|
|
668
|
+
end
|
|
669
|
+
targets = Array(wrt).map { |t| grad_target!(t) }
|
|
670
|
+
if (target = targets.find { |value| value.dtype == :bfloat16 })
|
|
671
|
+
raise Error, "bfloat16 training is not supported yet (wrt target #{target.shape.inspect}); " \
|
|
672
|
+
"keep trainable parameters and optimizer state in :float32"
|
|
673
|
+
end
|
|
674
|
+
|
|
675
|
+
cotangents = Hash.new { |h, k| h[k] = [] }
|
|
676
|
+
cotangents[loss.node.id] << fill(loss.shape, 1.0, dtype: loss.dtype)
|
|
677
|
+
backward(loss.node, cotangents)
|
|
678
|
+
|
|
679
|
+
targets.map do |target|
|
|
680
|
+
parts = cotangents[target.node.id]
|
|
681
|
+
parts.empty? ? fill(target.shape, 0.0, dtype: target.dtype) : sum_cotangents(parts)
|
|
682
|
+
end
|
|
683
|
+
end
|
|
684
|
+
|
|
685
|
+
# --- Outputs and compile ---
|
|
686
|
+
|
|
687
|
+
# Marks a tensor as a plan output.
|
|
688
|
+
#
|
|
689
|
+
# @param tensor [Tensor]
|
|
690
|
+
# @return [Tensor]
|
|
691
|
+
def output(tensor)
|
|
692
|
+
tensor = check_tensor!(tensor, "output")
|
|
693
|
+
# Returning a state view as is would let later writes change it, so
|
|
694
|
+
# outputs are always materialized.
|
|
695
|
+
tensor.node.attrs[:materialize] = true if tensor.node.op == :state_read
|
|
696
|
+
@outputs << tensor.node
|
|
697
|
+
tensor
|
|
698
|
+
end
|
|
699
|
+
|
|
700
|
+
# Compiles this recorded graph into an executable plan.
|
|
701
|
+
#
|
|
702
|
+
# @param vram [Integer, nil] optional device-memory budget in bytes
|
|
703
|
+
# @param spill_dir [String, nil] directory used for planned weight offload
|
|
704
|
+
# @return [Plan]
|
|
705
|
+
#
|
|
706
|
+
# vram: adds the residency pass that fits weights (:const) into the budget
|
|
707
|
+
# (#17 Phase 4 / docs/graph-offload.md). Without it everything stays
|
|
708
|
+
# resident as before and the pass does not exist.
|
|
709
|
+
#
|
|
710
|
+
# plan = g.compile(vram: 8 * 1024**3)
|
|
711
|
+
# plan.residency # tells whether it fits before running
|
|
712
|
+
def compile(vram: nil, spill_dir: nil, backing: :file, host_budget: nil) =
|
|
713
|
+
Plan.new(self, vram: vram, spill_dir: spill_dir, backing: backing, host_budget: host_budget)
|
|
714
|
+
|
|
715
|
+
def input_nodes = @inputs
|
|
716
|
+
|
|
717
|
+
# --- Internal ---
|
|
718
|
+
|
|
719
|
+
def record(op, attrs: {}, inputs: [], dtype:, shape:, cse: nil)
|
|
720
|
+
# Ops missing from the table are not recorded. Running with undeclared
|
|
721
|
+
# behavior (may it be freed, does it run synchronously) would misschedule
|
|
722
|
+
# rather than raise, so fail at record time.
|
|
723
|
+
Op.fetch(op)
|
|
724
|
+
key = cse && [op, inputs.map(&:id), cse]
|
|
725
|
+
found = key && @cse[key]
|
|
726
|
+
return found if found
|
|
727
|
+
|
|
728
|
+
node = Node.new(@nodes.length, op, inputs, attrs, dtype, shape)
|
|
729
|
+
@nodes << node
|
|
730
|
+
@cse[key] = node if key
|
|
731
|
+
node
|
|
732
|
+
end
|
|
733
|
+
|
|
734
|
+
private
|
|
735
|
+
|
|
736
|
+
# The body of backward. Visits only nodes reachable from loss in
|
|
737
|
+
# descending id order, sums the cotangents accumulated at each node and
|
|
738
|
+
# has the vjp table produce the input-side cotangents.
|
|
739
|
+
#
|
|
740
|
+
# Consumers always have larger ids than producers, so visiting in
|
|
741
|
+
# descending order guarantees a node's cotangents are complete.
|
|
742
|
+
def backward(root, cotangents)
|
|
743
|
+
reachable = {}
|
|
744
|
+
stack = [root]
|
|
745
|
+
while (node = stack.pop)
|
|
746
|
+
next if reachable[node.id]
|
|
747
|
+
|
|
748
|
+
reachable[node.id] = true
|
|
749
|
+
node.inputs.each { |input| stack << input }
|
|
750
|
+
end
|
|
751
|
+
|
|
752
|
+
# A node with a copy passes the copy to its vjp, which also swaps rows
|
|
753
|
+
# that use their own output (exp / tanh / softmax). Only cotangents
|
|
754
|
+
# accumulate under **the original node's** input ids, since the walk is
|
|
755
|
+
# over the original graph.
|
|
756
|
+
recomputed = {}
|
|
757
|
+
rebuilt = {}
|
|
758
|
+
# Adding copies grows @nodes, so walk a snapshot of the current list.
|
|
759
|
+
@nodes.dup.reverse_each do |node|
|
|
760
|
+
next unless reachable[node.id]
|
|
761
|
+
|
|
762
|
+
parts = cotangents[node.id]
|
|
763
|
+
next if parts.empty?
|
|
764
|
+
|
|
765
|
+
dy = sum_cotangents(parts)
|
|
766
|
+
cotangents[node.id] = [dy]
|
|
767
|
+
next if node.inputs.empty?
|
|
768
|
+
|
|
769
|
+
# Rebuild the region where it is first needed (= its exit, walking in
|
|
770
|
+
# reverse). Building them all up front lets the scheduler hoist them
|
|
771
|
+
# once their dependencies are ready, and the peak does not drop.
|
|
772
|
+
rebuild_region(node, reachable, recomputed, rebuilt)
|
|
773
|
+
target = recomputed[node.id] || node
|
|
774
|
+
inputs = target.inputs.map { |input| Tensor.new(self, input) }
|
|
775
|
+
Grad.apply(self, target, dy, inputs).each_with_index do |part, i|
|
|
776
|
+
cotangents[node.inputs[i].id] << part if part
|
|
777
|
+
end
|
|
778
|
+
end
|
|
779
|
+
end
|
|
780
|
+
|
|
781
|
+
# If node is in a checkpoint region not yet rebuilt, duplicate the whole
|
|
782
|
+
# region. Inputs inside the same region go to the copies; those outside
|
|
783
|
+
# (the boundary) connect as is. It bypasses record to avoid CSE — the same
|
|
784
|
+
# expression folds to the same node, so going through it would erase the copies.
|
|
785
|
+
def rebuild_region(node, reachable, map, rebuilt)
|
|
786
|
+
index = @checkpoints.index { |region| node.id >= region.from && node.id < region.to }
|
|
787
|
+
return if index.nil? || rebuilt[index]
|
|
788
|
+
|
|
789
|
+
rebuilt[index] = true
|
|
790
|
+
region = @checkpoints[index]
|
|
791
|
+
@nodes[region.from...region.to].each do |original|
|
|
792
|
+
next unless reachable[original.id]
|
|
793
|
+
next if NEVER_RECOMPUTE.include?(original.op)
|
|
794
|
+
|
|
795
|
+
inputs = original.inputs.map { |input| map[input.id] || input }
|
|
796
|
+
clone = Node.new(@nodes.length, original.op, inputs, original.attrs,
|
|
797
|
+
original.dtype, original.shape)
|
|
798
|
+
@nodes << clone
|
|
799
|
+
map[original.id] = clone
|
|
800
|
+
end
|
|
801
|
+
end
|
|
802
|
+
|
|
803
|
+
# A node with several consumers receives several cotangents. They are
|
|
804
|
+
# summed with ordinary binary ops (fusion folds them) — accumulating into
|
|
805
|
+
# a buffer would create a mutation the scheduler was never told about.
|
|
806
|
+
def sum_cotangents(parts) = parts.length == 1 ? parts.first : parts.inject(:+)
|
|
807
|
+
|
|
808
|
+
def grad_target!(target)
|
|
809
|
+
return target if target.is_a?(Tensor)
|
|
810
|
+
|
|
811
|
+
if target.is_a?(StateRef)
|
|
812
|
+
raise Error, "grad wrt: takes the tensor the forward pass used; for a state, " \
|
|
813
|
+
"pass the value you read (g.read(ref)), not the reference itself"
|
|
814
|
+
end
|
|
815
|
+
|
|
816
|
+
raise TypeError, "grad wrt: requires a Tilework::Graph::Tensor, got #{target.class}"
|
|
817
|
+
end
|
|
818
|
+
|
|
819
|
+
# Operands of different shapes get a broadcast node before recording. From
|
|
820
|
+
# then on shapes match, so **only recording** knows about broadcasting.
|
|
821
|
+
def binary_node(op, a, b, dtype:, kind:)
|
|
822
|
+
shape = common_shape!(a, b)
|
|
823
|
+
a = broadcast(a, shape)
|
|
824
|
+
b = broadcast(b, shape)
|
|
825
|
+
node = record(:binary, inputs: [a.node, b.node], attrs: { op: op, kind: kind },
|
|
826
|
+
dtype: dtype, shape: shape, cse: [:binary, kind, op, dtype])
|
|
827
|
+
Tensor.new(self, node)
|
|
828
|
+
end
|
|
829
|
+
|
|
830
|
+
def reduce_node(op, kind, tensor, axis, keepdims, dtype)
|
|
831
|
+
axis = normalize_axis!(axis, tensor.shape) unless axis.nil?
|
|
832
|
+
shape = axis.nil? ? [1] : reduced_shape(tensor.shape, axis, keepdims)
|
|
833
|
+
node = record(op, inputs: [tensor.node], attrs: { kind: kind, axis: axis, keepdims: keepdims },
|
|
834
|
+
dtype: dtype, shape: shape, cse: [op, kind, dtype, axis, keepdims])
|
|
835
|
+
Tensor.new(self, node)
|
|
836
|
+
end
|
|
837
|
+
|
|
838
|
+
def normalize_axis!(axis, shape)
|
|
839
|
+
rank = shape.length
|
|
840
|
+
raise TypeError, "axis must be an Integer, got #{axis.class}" unless axis.is_a?(Integer)
|
|
841
|
+
|
|
842
|
+
resolved = axis.negative? ? rank + axis : axis
|
|
843
|
+
raise ArgumentError, "axis #{axis} is out of range for #{shape.inspect}" unless resolved.between?(0, rank - 1)
|
|
844
|
+
|
|
845
|
+
resolved
|
|
846
|
+
end
|
|
847
|
+
|
|
848
|
+
# where's output type: the binary-op rule applied to the two selectable
|
|
849
|
+
# sides (the condition's int32 does not affect the result type — it only selects).
|
|
850
|
+
def where_dtype(a, b)
|
|
851
|
+
left = a.is_a?(Numeric) ? a : check_tensor!(a, "where a").dtype
|
|
852
|
+
right = b.is_a?(Numeric) ? b : check_tensor!(b, "where b").dtype
|
|
853
|
+
# Both may be scalars (building a mask); the shape comes from the condition.
|
|
854
|
+
if left.is_a?(Numeric) && right.is_a?(Numeric)
|
|
855
|
+
return left.is_a?(Integer) && right.is_a?(Integer) ? :int32 : :float32
|
|
856
|
+
end
|
|
857
|
+
|
|
858
|
+
left.is_a?(Numeric) ? result_dtype(right, left) : result_dtype(left, right)
|
|
859
|
+
end
|
|
860
|
+
|
|
861
|
+
# The common shape of the three operands; scalars have no shape and are skipped.
|
|
862
|
+
def where_shape!(cond, a, b)
|
|
863
|
+
operands = [a, b].reject { |value| value.is_a?(Numeric) }
|
|
864
|
+
.map { |value| check_tensor!(value, "where operand") }
|
|
865
|
+
operands.inject(cond.shape) do |acc, tensor|
|
|
866
|
+
shape = broadcast_shape(acc, tensor.shape)
|
|
867
|
+
unless shape
|
|
868
|
+
raise ArgumentError, "cannot broadcast #{acc.inspect} and #{tensor.shape.inspect} together in where"
|
|
869
|
+
end
|
|
870
|
+
|
|
871
|
+
shape
|
|
872
|
+
end
|
|
873
|
+
end
|
|
874
|
+
|
|
875
|
+
# Folded axes disappear (kept as 1 with keepdims). If none remain the
|
|
876
|
+
# shape is [1] — the same as a whole-buffer reduction, so 0-dim Tensors
|
|
877
|
+
# never need handling downstream.
|
|
878
|
+
def reduced_shape(shape, axis, keepdims)
|
|
879
|
+
dims = shape.dup
|
|
880
|
+
keepdims ? dims[axis] = 1 : dims.delete_at(axis)
|
|
881
|
+
dims.empty? ? [1] : dims
|
|
882
|
+
end
|
|
883
|
+
|
|
884
|
+
def scalar_node(op, a, scalar, dtype:, kind:, reversed:)
|
|
885
|
+
node = record(:binary, inputs: [a.node],
|
|
886
|
+
attrs: { op: op, kind: kind, scalar: scalar, reversed: reversed },
|
|
887
|
+
dtype: dtype, shape: a.shape,
|
|
888
|
+
cse: [:binary, kind, op, dtype, scalar, scalar.class.name, reversed])
|
|
889
|
+
Tensor.new(self, node)
|
|
890
|
+
end
|
|
891
|
+
|
|
892
|
+
# The result shape of a binary op: NumPy right alignment, dims matching or
|
|
893
|
+
# one being 1. **Eager (Ops) does not broadcast**, so this is expressive
|
|
894
|
+
# power only the graph has (like batched matmul and concat).
|
|
895
|
+
def common_shape!(a, b)
|
|
896
|
+
shape = broadcast_shape(a.shape, b.shape)
|
|
897
|
+
return shape if shape
|
|
898
|
+
|
|
899
|
+
raise ArgumentError, "tensor shape mismatch: #{a.shape.inspect} vs #{b.shape.inspect} " \
|
|
900
|
+
"(shapes must be equal or broadcastable)"
|
|
901
|
+
end
|
|
902
|
+
|
|
903
|
+
# nil if they do not line up. Right-align, then stretch the side that is 1.
|
|
904
|
+
def broadcast_shape(left, right)
|
|
905
|
+
rank = [left.length, right.length].max
|
|
906
|
+
a = Array.new(rank - left.length, 1) + left
|
|
907
|
+
b = Array.new(rank - right.length, 1) + right
|
|
908
|
+
dims = a.zip(b).map do |x, y|
|
|
909
|
+
return nil unless x == y || x == 1 || y == 1
|
|
910
|
+
|
|
911
|
+
[x, y].max
|
|
912
|
+
end
|
|
913
|
+
dims
|
|
914
|
+
end
|
|
915
|
+
|
|
916
|
+
# Infer a single -1, then compare element counts (as Buffer#reshape does).
|
|
917
|
+
def resolve_reshape(tensor, dims)
|
|
918
|
+
unless dims.count(-1) <= 1 && dims.all? { |d| d.is_a?(Integer) && (d.positive? || d == -1) }
|
|
919
|
+
raise ArgumentError, "reshape dims must be positive Integers with at most one -1, got #{dims.inspect}"
|
|
920
|
+
end
|
|
921
|
+
|
|
922
|
+
if (hole = dims.index(-1))
|
|
923
|
+
known = dims.reject { |d| d == -1 }.inject(1, :*)
|
|
924
|
+
unless known.positive? && (tensor.length % known).zero?
|
|
925
|
+
raise ArgumentError, "cannot reshape #{tensor.shape.inspect} to #{dims.inspect}"
|
|
926
|
+
end
|
|
927
|
+
|
|
928
|
+
dims = dims.dup
|
|
929
|
+
dims[hole] = tensor.length / known
|
|
930
|
+
end
|
|
931
|
+
unless dims.inject(1, :*) == tensor.length
|
|
932
|
+
raise ArgumentError, "cannot reshape #{tensor.shape.inspect} (#{tensor.length} elements) " \
|
|
933
|
+
"to #{dims.inspect}"
|
|
934
|
+
end
|
|
935
|
+
|
|
936
|
+
normalize_shape(dims)
|
|
937
|
+
end
|
|
938
|
+
|
|
939
|
+
# Accepts the same forms as Buffer#slice (an Integer for one, a Range for a range).
|
|
940
|
+
def slice_range!(range, limit)
|
|
941
|
+
case range
|
|
942
|
+
when Integer then [normalize_slice_index!(range, limit), 1]
|
|
943
|
+
when Range
|
|
944
|
+
first = normalize_slice_index!(range.begin || 0, limit)
|
|
945
|
+
last = range.end.nil? ? limit - 1 : normalize_slice_index!(range.end, limit)
|
|
946
|
+
last -= 1 if range.exclude_end?
|
|
947
|
+
raise ArgumentError, "slice #{range.inspect} is empty" if last < first
|
|
948
|
+
|
|
949
|
+
[first, last - first + 1]
|
|
950
|
+
else raise TypeError, "slice takes an Integer or a Range, got #{range.class}"
|
|
951
|
+
end
|
|
952
|
+
end
|
|
953
|
+
|
|
954
|
+
def normalize_slice_index!(index, limit)
|
|
955
|
+
resolved = index.negative? ? limit + index : index
|
|
956
|
+
raise ArgumentError, "slice index #{index} is out of range for #{limit} rows" unless resolved.between?(0, limit - 1)
|
|
957
|
+
|
|
958
|
+
resolved
|
|
959
|
+
end
|
|
960
|
+
|
|
961
|
+
def broadcastable?(from, to)
|
|
962
|
+
return false if from.length > to.length
|
|
963
|
+
|
|
964
|
+
broadcast_shape(from, to) == to
|
|
965
|
+
end
|
|
966
|
+
|
|
967
|
+
def matmul_shape(a, b)
|
|
968
|
+
case [a.shape.length, b.shape.length]
|
|
969
|
+
when [2, 2]
|
|
970
|
+
check_inner_dim!(a, b, a.shape[1], b.shape[0])
|
|
971
|
+
[:plain, [a.shape[0], b.shape[1]]]
|
|
972
|
+
when [3, 3]
|
|
973
|
+
unless a.shape[0] == b.shape[0]
|
|
974
|
+
raise ArgumentError, "batched matmul needs the same batch on both sides, " \
|
|
975
|
+
"got #{a.shape.inspect} x #{b.shape.inspect}"
|
|
976
|
+
end
|
|
977
|
+
|
|
978
|
+
check_inner_dim!(a, b, a.shape[2], b.shape[1])
|
|
979
|
+
[:batched, [a.shape[0], a.shape[1], b.shape[2]]]
|
|
980
|
+
else
|
|
981
|
+
raise ArgumentError, "matmul takes [m, k] x [k, n] or [b, m, k] x [b, k, n], " \
|
|
982
|
+
"got #{a.shape.inspect} x #{b.shape.inspect}"
|
|
983
|
+
end
|
|
984
|
+
end
|
|
985
|
+
|
|
986
|
+
def check_inner_dim!(a, b, left, right)
|
|
987
|
+
return if left == right
|
|
988
|
+
|
|
989
|
+
raise ArgumentError, "matmul shape mismatch: #{a.shape.inspect} x #{b.shape.inspect}"
|
|
990
|
+
end
|
|
991
|
+
|
|
992
|
+
# The same rule as Ops.promoted_dtype (int32 only when both are int32).
|
|
993
|
+
# float16 is never a compute type, so it does not appear here.
|
|
994
|
+
def promote(dtype, other)
|
|
995
|
+
other_int = other.is_a?(Symbol) ? other == :int32 : other.is_a?(Integer)
|
|
996
|
+
dtype == :int32 && other_int ? :int32 : :float32
|
|
997
|
+
end
|
|
998
|
+
|
|
999
|
+
# The same rule as Ops.result_dtypes' output type. Only float16 with
|
|
1000
|
+
# float16 (or with a scalar) stays float16 — so intermediate operations do
|
|
1001
|
+
# not throw away the bandwidth half precision saves.
|
|
1002
|
+
def result_dtype(dtype, other)
|
|
1003
|
+
same_storage = other.is_a?(Symbol) ? other == dtype : other.is_a?(Numeric)
|
|
1004
|
+
return dtype if Ops::LOW_PRECISION_DTYPES.include?(dtype) && same_storage
|
|
1005
|
+
|
|
1006
|
+
promote(dtype, other)
|
|
1007
|
+
end
|
|
1008
|
+
|
|
1009
|
+
def to_region(ref, at)
|
|
1010
|
+
case at
|
|
1011
|
+
when nil then ref.whole
|
|
1012
|
+
when Region then validate_region!(ref, at)
|
|
1013
|
+
else ref.rows(at)
|
|
1014
|
+
end
|
|
1015
|
+
end
|
|
1016
|
+
|
|
1017
|
+
def validate_region!(ref, region)
|
|
1018
|
+
return region if region.total == ref.state.buffer.length
|
|
1019
|
+
|
|
1020
|
+
raise Error, "region #{region.inspect} was built for a #{region.total}-element buffer, " \
|
|
1021
|
+
"but #{ref.state.name} has #{ref.state.buffer.length}"
|
|
1022
|
+
end
|
|
1023
|
+
|
|
1024
|
+
def check_ref!(ref)
|
|
1025
|
+
raise TypeError, "expected a Tilework::Graph::StateRef, got #{ref.class}" unless ref.is_a?(StateRef)
|
|
1026
|
+
raise Error, "state #{ref.state.name} belongs to another graph" unless ref.graph.equal?(self)
|
|
1027
|
+
|
|
1028
|
+
ref
|
|
1029
|
+
end
|
|
1030
|
+
|
|
1031
|
+
def random_node(kind, shape, dtype, attrs)
|
|
1032
|
+
unless dtype == :float32
|
|
1033
|
+
raise Error, "random generation only supports :float32 (got #{dtype.inspect})"
|
|
1034
|
+
end
|
|
1035
|
+
|
|
1036
|
+
index = @random_draws
|
|
1037
|
+
@random_draws += 1
|
|
1038
|
+
node = record(:random, attrs: attrs.merge(kind: kind, index: index),
|
|
1039
|
+
dtype: dtype, shape: normalize_shape(shape))
|
|
1040
|
+
Tensor.new(self, node)
|
|
1041
|
+
end
|
|
1042
|
+
|
|
1043
|
+
# Tensors as is; Buffers become const nodes.
|
|
1044
|
+
def lift(value, what)
|
|
1045
|
+
return const(value) if value.is_a?(Buffer)
|
|
1046
|
+
|
|
1047
|
+
check_tensor!(value, what)
|
|
1048
|
+
end
|
|
1049
|
+
|
|
1050
|
+
def check_tensor!(value, what)
|
|
1051
|
+
return value if value.is_a?(Tensor) && value.graph.equal?(self)
|
|
1052
|
+
raise TypeError, "#{what} requires a Tilework::Graph::Tensor, got #{value.class}" unless value.is_a?(Tensor)
|
|
1053
|
+
|
|
1054
|
+
raise Error, "#{what} got a tensor from another graph"
|
|
1055
|
+
end
|
|
1056
|
+
|
|
1057
|
+
def check_dtype!(dtype)
|
|
1058
|
+
return if DTYPES.include?(dtype)
|
|
1059
|
+
|
|
1060
|
+
raise Error, "the graph layer supports #{DTYPES.join(' / ')} for now, got #{dtype.inspect}"
|
|
1061
|
+
end
|
|
1062
|
+
|
|
1063
|
+
def normalize_shape(shape)
|
|
1064
|
+
dims = shape.is_a?(Array) ? shape : [shape]
|
|
1065
|
+
unless dims.any? && dims.all? { |d| d.is_a?(Integer) && d.positive? }
|
|
1066
|
+
raise ArgumentError, "shape must be a positive Integer or an Array of them, got #{shape.inspect}"
|
|
1067
|
+
end
|
|
1068
|
+
|
|
1069
|
+
dims
|
|
1070
|
+
end
|
|
1071
|
+
end
|
|
1072
|
+
end
|
|
1073
|
+
end
|