tilework 0.18.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE +21 -0
- data/lib/tilework/backend/cpu/dsl_codegen.rb +509 -0
- data/lib/tilework/backend/cpu/ops_codegen.rb +451 -0
- data/lib/tilework/backend/cpu.rb +156 -0
- data/lib/tilework/buffer.rb +753 -0
- data/lib/tilework/capabilities.rb +183 -0
- data/lib/tilework/compat/cuda.h +66 -0
- data/lib/tilework/compat/hip.h +35 -0
- data/lib/tilework/compat/metal.h +45 -0
- data/lib/tilework/conformance.rb +2296 -0
- data/lib/tilework/core_ext.rb +21 -0
- data/lib/tilework/dialect.rb +67 -0
- data/lib/tilework/dsl.rb +722 -0
- data/lib/tilework/dtype/bfloat.rb +31 -0
- data/lib/tilework/dtype/cast.rb +88 -0
- data/lib/tilework/dtype/half.rb +98 -0
- data/lib/tilework/graph/builder.rb +1073 -0
- data/lib/tilework/graph/fusion.rb +520 -0
- data/lib/tilework/graph/grad.rb +443 -0
- data/lib/tilework/graph/high_level.rb +266 -0
- data/lib/tilework/graph/kernels.rb +403 -0
- data/lib/tilework/graph/node.rb +27 -0
- data/lib/tilework/graph/op.rb +290 -0
- data/lib/tilework/graph/optim.rb +183 -0
- data/lib/tilework/graph/plan.rb +1086 -0
- data/lib/tilework/graph/pool.rb +178 -0
- data/lib/tilework/graph/region.rb +210 -0
- data/lib/tilework/graph/residency.rb +449 -0
- data/lib/tilework/graph/state.rb +64 -0
- data/lib/tilework/graph/tensor.rb +124 -0
- data/lib/tilework/graph.rb +213 -0
- data/lib/tilework/kernel.rb +163 -0
- data/lib/tilework/math/conv.rb +270 -0
- data/lib/tilework/math/fft.rb +172 -0
- data/lib/tilework/math/nn.rb +1103 -0
- data/lib/tilework/math/philox.rb +194 -0
- data/lib/tilework/math/random.rb +232 -0
- data/lib/tilework/math/tensor_core.rb +269 -0
- data/lib/tilework/ops.rb +682 -0
- data/lib/tilework/runtime/offload.rb +498 -0
- data/lib/tilework/runtime/submission.rb +163 -0
- data/lib/tilework/runtime/telemetry.rb +112 -0
- data/lib/tilework/scope.rb +33 -0
- data/lib/tilework/version.rb +5 -0
- data/lib/tilework.rb +345 -0
- metadata +92 -0
|
@@ -0,0 +1,290 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Tilework
|
|
4
|
+
module Graph
|
|
5
|
+
# The contract for how an IR op behaves. REGISTRY is the single source of
|
|
6
|
+
# the op set and of each op's properties as the scheduler sees them.
|
|
7
|
+
#
|
|
8
|
+
# It is a table to stop things falling through the cracks. These
|
|
9
|
+
# properties used to be scattered across four constants in plan.rb
|
|
10
|
+
# (NON_OWNED / SYNCHRONOUS_OPS / FOREIGN_OPS / PLACEABLE), a case in
|
|
11
|
+
# execute, FUSABLE in fusion.rb and node.rb's labels, so adding an op meant
|
|
12
|
+
# touching six places. Forgetting one raised nothing; the scheduler just
|
|
13
|
+
# silently did the wrong thing (freeing a buffer it must not, not waiting
|
|
14
|
+
# where it must). Collected here, an undeclared op fails in Op.fetch — the
|
|
15
|
+
# same idea as Capabilities::FIELDS.
|
|
16
|
+
#
|
|
17
|
+
# op = Graph::Op.fetch(node.op)
|
|
18
|
+
# op.owned? # does it own its buffer (= may it be freed)?
|
|
19
|
+
# op.synchronous?(node) # does it wait in place instead of queueing?
|
|
20
|
+
#
|
|
21
|
+
# Properties that depend on node attributes or the backend's capabilities
|
|
22
|
+
# (batched matmul, delegated GEMM / RNG, ...) are declared as Procs;
|
|
23
|
+
# anything expressible as a boolean stays a boolean.
|
|
24
|
+
class Op
|
|
25
|
+
# NN functions delegated to a vendor library when gemm_library is set.
|
|
26
|
+
# They need waits on both sides for the same reason as matmul (decided
|
|
27
|
+
# without backend names).
|
|
28
|
+
FOREIGN_NN = %i[multi_head_attention].freeze
|
|
29
|
+
|
|
30
|
+
REGISTRY = {}
|
|
31
|
+
|
|
32
|
+
class << self
|
|
33
|
+
# One row of the table.
|
|
34
|
+
#
|
|
35
|
+
# run the Plan method to execute; all take (node, inputs, bindings)
|
|
36
|
+
# launches whether it launches a kernel (or copy). A node with
|
|
37
|
+
# false leaves the queue as it was, so after a synchronous
|
|
38
|
+
# node it is "still empty" — used to decide whether
|
|
39
|
+
# automatic offload's eviction can ride on that wait
|
|
40
|
+
# (#17 Phase 4c)
|
|
41
|
+
# owned whether it owns its buffer. Buffers of nodes with false
|
|
42
|
+
# (inputs, constants, state) belong outside the graph, so
|
|
43
|
+
# liveness leaves them alone
|
|
44
|
+
# placeable whether, as a write_into value, it may write straight
|
|
45
|
+
# into the state's view. Only ops that allocate their
|
|
46
|
+
# output once and fill it (#19 §3)
|
|
47
|
+
# fusable whether elementwise fusion can fold it into one kernel
|
|
48
|
+
# index_map whether it only changes how its input is read (fusion
|
|
49
|
+
# folds it into the index)
|
|
50
|
+
# synchronous whether it runs in place instead of queueing and moving on
|
|
51
|
+
# foreign whether it may run outside our queue (synchronize on both sides)
|
|
52
|
+
# requires the capability needed to run this node (nil if none).
|
|
53
|
+
# Plan#requires collects them to report what is missing
|
|
54
|
+
# before running
|
|
55
|
+
# writes_state whether it advances a mutable state's version
|
|
56
|
+
# label the line for explain / inspect (defaults to the op name)
|
|
57
|
+
def define(name, run:, owned: true, placeable: false, fusable: false, index_map: false,
|
|
58
|
+
synchronous: false, foreign: false, writes_state: false, requires: nil,
|
|
59
|
+
launches: true, label: nil)
|
|
60
|
+
raise Error, "graph op #{name.inspect} is already defined" if REGISTRY.key?(name)
|
|
61
|
+
|
|
62
|
+
REGISTRY[name] = new(name: name, run: run, owned: owned, placeable: placeable,
|
|
63
|
+
fusable: fusable, index_map: index_map, synchronous: synchronous,
|
|
64
|
+
foreign: foreign, writes_state: writes_state, requires: requires,
|
|
65
|
+
launches: launches, label: label)
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
def fetch(name)
|
|
69
|
+
REGISTRY.fetch(name) do
|
|
70
|
+
raise Error, "unknown graph op #{name.inspect} (known: #{names.map(&:inspect).join(', ')})"
|
|
71
|
+
end
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
def defined?(name) = REGISTRY.key?(name)
|
|
75
|
+
def names = REGISTRY.keys
|
|
76
|
+
|
|
77
|
+
# Whether a delegated GEMM leaves our queue, and so has to wait on both
|
|
78
|
+
# sides. matmul and :nn make the same decision, so it lives in one
|
|
79
|
+
# place. A bridge whose vendor GEMM is enqueued on the same queue as
|
|
80
|
+
# its kernels, in submission order, declares `ordered_gemm?`; ordering
|
|
81
|
+
# then covers it exactly as it covers a kernel and no fence is needed.
|
|
82
|
+
def foreign_gemm?
|
|
83
|
+
return false if Tilework.capabilities.gemm_library.nil?
|
|
84
|
+
|
|
85
|
+
bridge = Tilework.bridge
|
|
86
|
+
!(bridge.respond_to?(:ordered_gemm?) && bridge.ordered_gemm?)
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
# Whether a capability is actually usable; nil and false both mean no.
|
|
90
|
+
def satisfied?(capability)
|
|
91
|
+
value = Tilework.capabilities.public_send(capability)
|
|
92
|
+
!value.nil? && value != false
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
# The axis shown on a reduction (none for a whole-buffer reduction).
|
|
96
|
+
def axis_suffix(node)
|
|
97
|
+
axis = node.attrs[:axis]
|
|
98
|
+
return nil if axis.nil?
|
|
99
|
+
|
|
100
|
+
"axis #{axis}#{node.attrs[:keepdims] ? ' keepdims' : ''}"
|
|
101
|
+
end
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
attr_reader :name, :run
|
|
105
|
+
|
|
106
|
+
def initialize(name:, run:, owned:, placeable:, fusable:, index_map:, synchronous:, foreign:,
|
|
107
|
+
writes_state:, requires:, launches:, label:)
|
|
108
|
+
@name = name
|
|
109
|
+
@run = run
|
|
110
|
+
@owned = owned
|
|
111
|
+
@placeable = predicate(placeable)
|
|
112
|
+
@fusable = fusable
|
|
113
|
+
@index_map = index_map
|
|
114
|
+
@synchronous = predicate(synchronous)
|
|
115
|
+
@foreign = predicate(foreign)
|
|
116
|
+
@writes_state = writes_state
|
|
117
|
+
@requires = requires
|
|
118
|
+
@launches = launches
|
|
119
|
+
@label = label
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
# The capability needed to run this node (nil if none).
|
|
123
|
+
attr_reader :requires
|
|
124
|
+
|
|
125
|
+
def owned? = @owned
|
|
126
|
+
def fusable? = @fusable
|
|
127
|
+
|
|
128
|
+
# Whether it launches anything; false means the node leaves the queue as it was.
|
|
129
|
+
def launches? = @launches
|
|
130
|
+
|
|
131
|
+
# An op that creates no value and only changes how its source buffer is
|
|
132
|
+
# read (fusion folds it into the index).
|
|
133
|
+
def index_map? = @index_map
|
|
134
|
+
def writes_state? = @writes_state
|
|
135
|
+
def placeable?(node) = @placeable.call(node)
|
|
136
|
+
def synchronous?(node) = @synchronous.call(node)
|
|
137
|
+
def foreign?(node) = @foreign.call(node)
|
|
138
|
+
def label_for(node) = @label ? @label.call(node) : name.to_s
|
|
139
|
+
|
|
140
|
+
private
|
|
141
|
+
|
|
142
|
+
# Boolean properties are wrapped in Procs too, so callers need no case split.
|
|
143
|
+
def predicate(value) = value.respond_to?(:call) ? value : ->(_node) { value }
|
|
144
|
+
end
|
|
145
|
+
|
|
146
|
+
# --- The table -----------------------------------------------------------
|
|
147
|
+
#
|
|
148
|
+
# Structural nodes. They create no value (they point at existing buffers),
|
|
149
|
+
# so owned: false.
|
|
150
|
+
|
|
151
|
+
Op.define :input, run: :run_input, owned: false, launches: false,
|
|
152
|
+
label: ->(node) { "input #{node.attrs[:name].inspect}" }
|
|
153
|
+
|
|
154
|
+
Op.define :const, run: :run_const, owned: false, launches: false, label: ->(_node) { "const" }
|
|
155
|
+
|
|
156
|
+
Op.define :state_init, run: :run_state_init, owned: false, launches: false
|
|
157
|
+
|
|
158
|
+
# A read is a view by default, so it may or may not own memory; when
|
|
159
|
+
# materialized it may be freed, hence owned: true. The release side rejects views.
|
|
160
|
+
Op.define :state_read, run: :run_state_read,
|
|
161
|
+
label: ->(node) { "read #{node.state.name} v#{node.attrs[:version]} #{node.region.inspect}" }
|
|
162
|
+
|
|
163
|
+
Op.define :state_write, run: :run_state_write, owned: false, writes_state: true,
|
|
164
|
+
label: lambda { |node|
|
|
165
|
+
"write #{node.state.name} -> v#{node.attrs[:version]} #{node.region.inspect}"
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
Op.define :state_merge, run: :run_state_merge, owned: false, writes_state: true, launches: false,
|
|
169
|
+
label: ->(node) { "merge #{node.state.name} -> v#{node.attrs[:version]}" }
|
|
170
|
+
|
|
171
|
+
# Compute nodes.
|
|
172
|
+
|
|
173
|
+
Op.define :binary, run: :run_binary, placeable: true, fusable: true,
|
|
174
|
+
label: lambda { |node|
|
|
175
|
+
"#{node.attrs[:op]}#{node.attrs.key?(:scalar) ? " #{node.attrs[:scalar]}" : ''}"
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
Op.define :unary, run: :run_unary, placeable: true, fusable: true,
|
|
179
|
+
label: ->(node) { node.attrs[:fn].to_s }
|
|
180
|
+
|
|
181
|
+
# A whole-buffer reduction (axis: nil) folds as a tree, and the execution
|
|
182
|
+
# context takes over freeing the partials between passes (#86), so it keeps
|
|
183
|
+
# queueing and moves on. An axis reduction is one kernel with no
|
|
184
|
+
# intermediates (and its output can be placed).
|
|
185
|
+
Op.define :reduce, run: :run_reduce,
|
|
186
|
+
placeable: ->(node) { !node.attrs[:axis].nil? },
|
|
187
|
+
label: ->(node) { [node.attrs[:kind], Op.axis_suffix(node)].compact.join(" ") }
|
|
188
|
+
|
|
189
|
+
Op.define :arg_reduce, run: :run_arg_reduce,
|
|
190
|
+
placeable: ->(node) { !node.attrs[:axis].nil? },
|
|
191
|
+
label: ->(node) { ["arg#{node.attrs[:kind]}", Op.axis_suffix(node)].compact.join(" ") }
|
|
192
|
+
|
|
193
|
+
# Change the type / select by condition / write indices. Each is closed per
|
|
194
|
+
# element, so each can fuse (:iota has no inputs, so it is never absorbed
|
|
195
|
+
# as an operand, though it can be the root that owns the output).
|
|
196
|
+
Op.define :cast, run: :run_cast, placeable: true, fusable: true,
|
|
197
|
+
label: ->(node) { "cast #{node.dtype}" }
|
|
198
|
+
|
|
199
|
+
Op.define :where, run: :run_where, placeable: true, fusable: true
|
|
200
|
+
|
|
201
|
+
Op.define :iota, run: :run_iota, placeable: true,
|
|
202
|
+
label: ->(node) { "iota axis #{node.attrs[:axis]}" }
|
|
203
|
+
|
|
204
|
+
# A node that only changes the shape. Execution returns a view of its input:
|
|
205
|
+
# no kernel runs and no new buffer is owned (= owned: false). Being an
|
|
206
|
+
# alias, the source buffer's lifetime must extend to this node's consumers
|
|
207
|
+
# (Plan#alias_root).
|
|
208
|
+
Op.define :reshape, run: :run_reshape, owned: false, launches: false,
|
|
209
|
+
label: ->(node) { "reshape #{node.shape.inspect}" }
|
|
210
|
+
|
|
211
|
+
# A node that only remaps indices. It can fuse — but unlike other ops it
|
|
212
|
+
# folds **into the index**, not into the expression (the consumer's kernel
|
|
213
|
+
# reads the source buffer with remapped indices). Only when not fused does
|
|
214
|
+
# one materializing kernel run. Broadcast (stretching) and permute
|
|
215
|
+
# (swapping axes) are the same operation with different stride tables.
|
|
216
|
+
Op.define :broadcast, run: :run_remap, placeable: true, fusable: true, index_map: true
|
|
217
|
+
|
|
218
|
+
Op.define :permute, run: :run_remap, placeable: true, fusable: true, index_map: true,
|
|
219
|
+
label: ->(node) { "permute #{node.attrs[:axes].inspect}" }
|
|
220
|
+
|
|
221
|
+
# A slice of the outermost dimension. Returns a view, like reshape.
|
|
222
|
+
Op.define :slice, run: :run_slice, owned: false, launches: false,
|
|
223
|
+
label: ->(node) { "slice #{node.shape.inspect}" }
|
|
224
|
+
|
|
225
|
+
# Table lookup and border fill. Both allocate their output once and fill it.
|
|
226
|
+
Op.define :gather, run: :run_gather, placeable: true,
|
|
227
|
+
label: ->(node) { "gather axis #{node.attrs[:axis]}" }
|
|
228
|
+
|
|
229
|
+
# The inverse of table lookup. It allocates its output once and fills it
|
|
230
|
+
# (each thread finishes its own share), so it is placeable. No atomics, so
|
|
231
|
+
# it stays inside our queue.
|
|
232
|
+
Op.define :scatter_add, run: :run_scatter_add, placeable: true,
|
|
233
|
+
label: ->(node) { "scatter_add axis #{node.attrs[:axis]}" }
|
|
234
|
+
|
|
235
|
+
Op.define :pad, run: :run_pad, placeable: true,
|
|
236
|
+
label: ->(node) { "pad #{node.attrs[:before].inspect}" }
|
|
237
|
+
|
|
238
|
+
# matmul is delegated to cuBLAS / MPS when gemm_library is set. Where that
|
|
239
|
+
# leaves our execution path (Op.foreign_gemm?) it waits on both sides;
|
|
240
|
+
# a bridge that declares `ordered_gemm?` stays asynchronous. Freeing the
|
|
241
|
+
# per-batch temporaries of an undelegated batch is handled by #86's
|
|
242
|
+
# deferred free.
|
|
243
|
+
Op.define :matmul, run: :run_matmul,
|
|
244
|
+
placeable: ->(node) { node.attrs[:mode] != :batched },
|
|
245
|
+
synchronous: ->(_node) { Op.foreign_gemm? },
|
|
246
|
+
foreign: ->(_node) { Op.foreign_gemm? }
|
|
247
|
+
|
|
248
|
+
Op.define :concat, run: :run_concat, placeable: true
|
|
249
|
+
|
|
250
|
+
# A delegated RNG (cuRAND / MPS) leaves our execution path, and some paths
|
|
251
|
+
# fall back to writing from the host.
|
|
252
|
+
Op.define :random, run: :run_random, synchronous: true,
|
|
253
|
+
foreign: ->(_node) { !Tilework.capabilities.rng.nil? }
|
|
254
|
+
|
|
255
|
+
# NN's fused kernels hand freeing their intermediates (scores / probs) to the
|
|
256
|
+
# execution context (#86), so they wait only when delegated.
|
|
257
|
+
Op.define :nn, run: :run_nn,
|
|
258
|
+
synchronous: ->(node) { Op::FOREIGN_NN.include?(node.attrs[:fn]) && Op.foreign_gemm? },
|
|
259
|
+
foreign: ->(node) { Op::FOREIGN_NN.include?(node.attrs[:fn]) && Op.foreign_gemm? },
|
|
260
|
+
label: ->(node) { ["nn", node.attrs[:fn], *node.attrs[:args]].join(" ") }
|
|
261
|
+
|
|
262
|
+
# 2D convolution. Delegated to cuDNN / MPSGraph, so it waits on both sides.
|
|
263
|
+
# On a backend without convolution it is decomposed into primitives at
|
|
264
|
+
# record time and never becomes this node (graph/high_level.rb's :conv2d).
|
|
265
|
+
Op.define :conv2d, run: :run_conv2d, synchronous: true, foreign: true, requires: :conv,
|
|
266
|
+
label: ->(node) {
|
|
267
|
+
"conv2d pad #{node.attrs[:pad].inspect} stride #{node.attrs[:stride].inspect} " \
|
|
268
|
+
"dilation #{node.attrs[:dilation].inspect} groups #{node.attrs[:groups].inspect}"
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
# Convolution backward (#22 Step 2). Runs on the delegate like the forward
|
|
272
|
+
# pass, so it waits on both sides.
|
|
273
|
+
%i[data filter].each do |kind|
|
|
274
|
+
Op.define :"conv2d_backward_#{kind}", run: :"run_conv2d_backward_#{kind}",
|
|
275
|
+
synchronous: true, foreign: true, requires: :conv_backward,
|
|
276
|
+
label: ->(node) {
|
|
277
|
+
"conv2d backward #{kind} pad #{node.attrs[:pad].inspect} " \
|
|
278
|
+
"dilation #{node.attrs[:dilation].inspect} groups #{node.attrs[:groups].inspect}"
|
|
279
|
+
}
|
|
280
|
+
end
|
|
281
|
+
|
|
282
|
+
# A user block. What it does is unknown, so by default it waits on both
|
|
283
|
+
# sides. `g.opaque(..., fenced: false)` is the caller's declaration that
|
|
284
|
+
# the block stays inside our queue, and only then is the fence removed.
|
|
285
|
+
Op.define :opaque, run: :run_opaque,
|
|
286
|
+
synchronous: ->(node) { node.attrs[:fenced] },
|
|
287
|
+
foreign: ->(node) { node.attrs[:fenced] },
|
|
288
|
+
label: ->(node) { "opaque #{node.attrs[:name]}" }
|
|
289
|
+
end
|
|
290
|
+
end
|
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Tilework
|
|
4
|
+
module Graph
|
|
5
|
+
# Base class for graph-recorded parameter optimizers.
|
|
6
|
+
#
|
|
7
|
+
# An optimizer is **not a new mechanism** — it is a helper on top of
|
|
8
|
+
# mutable state (#19) and gradients (graph/grad.rb) that does three things:
|
|
9
|
+
#
|
|
10
|
+
# 1. declare the parameters as g.state and return the read Tensors (attach)
|
|
11
|
+
# 2. take the gradients of the loss (g.grad)
|
|
12
|
+
# 3. write back the moments and parameters (g.write_into)
|
|
13
|
+
#
|
|
14
|
+
# It all lands in one plan, so **a step is one plan.run**. The update is
|
|
15
|
+
# elementwise, so fusion folds it to one kernel per parameter.
|
|
16
|
+
#
|
|
17
|
+
# opt = Tilework::Graph::Adam.new({ w1: w1, b1: b1 }, lr: 1e-3)
|
|
18
|
+
#
|
|
19
|
+
# plan = Tilework::Graph.build(differentiable: true) do |g|
|
|
20
|
+
# x = g.input(:x, [n, d])
|
|
21
|
+
# p = opt.attach(g) # { w1: Tensor, b1: Tensor }
|
|
22
|
+
# loss = Tilework::NN.cross_entropy(x.matmul(p[:w1]) + p[:b1], labels)
|
|
23
|
+
# opt.step(loss) # records gradients and the update
|
|
24
|
+
# g.output(loss)
|
|
25
|
+
# end.compile
|
|
26
|
+
#
|
|
27
|
+
# 50.times { plan.run(x: xs) } # the weight Buffers update in place
|
|
28
|
+
#
|
|
29
|
+
# **Buffers are allocated outside Graph.build** (in Adam.new), keeping the
|
|
30
|
+
# graph layer's promise that recording touches no device.
|
|
31
|
+
class Optimizer
|
|
32
|
+
attr_reader :params, :grads
|
|
33
|
+
|
|
34
|
+
# Creates an optimizer for mutable parameter buffers.
|
|
35
|
+
#
|
|
36
|
+
# @param params [Hash{Symbol, String => Buffer}] named parameter buffers
|
|
37
|
+
# @return [Optimizer]
|
|
38
|
+
def initialize(params)
|
|
39
|
+
raise ArgumentError, "an optimizer needs at least one parameter" if params.empty?
|
|
40
|
+
|
|
41
|
+
@params = params.transform_keys(&:to_sym)
|
|
42
|
+
@refs = {}
|
|
43
|
+
@grads = {}
|
|
44
|
+
@slots = {} # the optimizer's own mutable state (moments etc.)
|
|
45
|
+
allocate_slots!
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
# Attaches parameter buffers as graph state and returns their tensors.
|
|
49
|
+
#
|
|
50
|
+
# @param builder [Builder]
|
|
51
|
+
# @return [Hash{Symbol => Tensor}]
|
|
52
|
+
#
|
|
53
|
+
# **The forward pass must use these Tensors.** Binding the parameters
|
|
54
|
+
# separately with g.param still yields gradients, but the update would be
|
|
55
|
+
# written somewhere else and have no effect.
|
|
56
|
+
def attach(builder)
|
|
57
|
+
@builder = builder
|
|
58
|
+
@refs = @params.to_h { |name, buffer| [name, builder.state(buffer, name: name)] }
|
|
59
|
+
@slot_refs = @slots.to_h { |key, buffer| [key, builder.state(buffer, name: key)] }
|
|
60
|
+
@reads = @refs.transform_values { |ref| builder.read(ref) }
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
# Records gradient calculation and one parameter update step.
|
|
64
|
+
#
|
|
65
|
+
# @param loss [Tensor] scalar loss tensor
|
|
66
|
+
# @return [Hash{Symbol => Tensor}] gradients by parameter name
|
|
67
|
+
#
|
|
68
|
+
# The gradients are returned so they can be inspected (for clipping or logging).
|
|
69
|
+
def step(loss)
|
|
70
|
+
raise Error, "call attach(g) before step(loss)" if @builder.nil?
|
|
71
|
+
|
|
72
|
+
names = @reads.keys
|
|
73
|
+
gradients = @builder.grad(loss, wrt: names.map { |name| @reads[name] })
|
|
74
|
+
@grads = names.zip(gradients).to_h
|
|
75
|
+
names.each { |name| update(name, @reads[name], @grads[name]) }
|
|
76
|
+
@grads
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
# Releases optimizer-owned state buffers such as momentum slots.
|
|
80
|
+
#
|
|
81
|
+
# @return [Hash]
|
|
82
|
+
def free!
|
|
83
|
+
@slots.each_value(&:free)
|
|
84
|
+
@slots = {}
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
private
|
|
88
|
+
|
|
89
|
+
# One zero-filled buffer per name (Buffer.new zero-fills on allocation).
|
|
90
|
+
def zeros_for(name, suffix)
|
|
91
|
+
buffer = @params.fetch(name)
|
|
92
|
+
@slots[:"#{name}_#{suffix}"] = Buffer.new(buffer.dtype, buffer.length, shape: buffer.shape)
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
def slot(name, suffix) = @slot_refs.fetch(:"#{name}_#{suffix}")
|
|
96
|
+
|
|
97
|
+
def allocate_slots!
|
|
98
|
+
# No extra state by default (SGD).
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
def update(_name, _param, _grad)
|
|
102
|
+
raise NotImplementedError, "#{self.class} must implement #update"
|
|
103
|
+
end
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
# Stochastic gradient descent with `p -= lr * gradient` updates.
|
|
107
|
+
#
|
|
108
|
+
# No momentum, deliberately; adding it would take the same shape as Adam
|
|
109
|
+
# (one state + one update line) when it is needed.
|
|
110
|
+
class SGD < Optimizer
|
|
111
|
+
# @param params [Hash{Symbol, String => Buffer}]
|
|
112
|
+
# @param lr [Float] learning rate
|
|
113
|
+
def initialize(params, lr: 1e-2)
|
|
114
|
+
@lr = lr.to_f
|
|
115
|
+
super(params)
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
private
|
|
119
|
+
|
|
120
|
+
def update(name, param, grad)
|
|
121
|
+
@builder.write_into(@refs.fetch(name), value: param - (@lr * grad))
|
|
122
|
+
end
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
# Adam optimizer with bias correction.
|
|
126
|
+
#
|
|
127
|
+
# m = β1·m + (1-β1)·g, v = β2·v + (1-β2)·g²
|
|
128
|
+
# p -= lr · (m / (1-β1^t)) / (sqrt(v / (1-β2^t)) + eps)
|
|
129
|
+
#
|
|
130
|
+
# The bias-correction t is **state inside the graph**. As a plan input,
|
|
131
|
+
# every caller would have to pass a step count only Adam needs. Since t is
|
|
132
|
+
# a run-time value, β^t is written `exp(t · ln β)` (pow is not a primitive
|
|
133
|
+
# and this is the only place that wants it).
|
|
134
|
+
class Adam < Optimizer
|
|
135
|
+
# @param params [Hash{Symbol, String => Buffer}]
|
|
136
|
+
# @param lr [Float] learning rate
|
|
137
|
+
# @param beta1 [Float] first-moment decay
|
|
138
|
+
# @param beta2 [Float] second-moment decay
|
|
139
|
+
# @param eps [Float] denominator stability term
|
|
140
|
+
def initialize(params, lr: 1e-3, beta1: 0.9, beta2: 0.999, eps: 1e-8)
|
|
141
|
+
@lr = lr.to_f
|
|
142
|
+
@beta1 = beta1.to_f
|
|
143
|
+
@beta2 = beta2.to_f
|
|
144
|
+
@eps = eps.to_f
|
|
145
|
+
super(params)
|
|
146
|
+
end
|
|
147
|
+
|
|
148
|
+
def attach(builder)
|
|
149
|
+
super
|
|
150
|
+
# t is a one-element state shared by all parameters.
|
|
151
|
+
@step_ref = @slot_refs.fetch(:__step)
|
|
152
|
+
t = builder.read(@step_ref) + 1.0
|
|
153
|
+
builder.write_into(@step_ref, value: t)
|
|
154
|
+
# 1 - β^t. t has shape [1], so it broadcasts into the expressions below.
|
|
155
|
+
@correction1 = 1.0 - (t * Math.log(@beta1)).exp
|
|
156
|
+
@correction2 = 1.0 - (t * Math.log(@beta2)).exp
|
|
157
|
+
@reads
|
|
158
|
+
end
|
|
159
|
+
|
|
160
|
+
private
|
|
161
|
+
|
|
162
|
+
def allocate_slots!
|
|
163
|
+
@params.each_key do |name|
|
|
164
|
+
zeros_for(name, :m)
|
|
165
|
+
zeros_for(name, :v)
|
|
166
|
+
end
|
|
167
|
+
@slots[:__step] = Buffer.new(:float32, 1, shape: [1])
|
|
168
|
+
end
|
|
169
|
+
|
|
170
|
+
def update(name, param, grad)
|
|
171
|
+
m_ref = slot(name, :m)
|
|
172
|
+
v_ref = slot(name, :v)
|
|
173
|
+
m = (@beta1 * @builder.read(m_ref)) + ((1.0 - @beta1) * grad)
|
|
174
|
+
v = (@beta2 * @builder.read(v_ref)) + ((1.0 - @beta2) * grad * grad)
|
|
175
|
+
@builder.write_into(m_ref, value: m)
|
|
176
|
+
@builder.write_into(v_ref, value: v)
|
|
177
|
+
|
|
178
|
+
step = (m / @correction1) / ((v / @correction2).sqrt + @eps)
|
|
179
|
+
@builder.write_into(@refs.fetch(name), value: param - (@lr * step))
|
|
180
|
+
end
|
|
181
|
+
end
|
|
182
|
+
end
|
|
183
|
+
end
|