tilework 0.18.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE +21 -0
  3. data/lib/tilework/backend/cpu/dsl_codegen.rb +509 -0
  4. data/lib/tilework/backend/cpu/ops_codegen.rb +451 -0
  5. data/lib/tilework/backend/cpu.rb +156 -0
  6. data/lib/tilework/buffer.rb +753 -0
  7. data/lib/tilework/capabilities.rb +183 -0
  8. data/lib/tilework/compat/cuda.h +66 -0
  9. data/lib/tilework/compat/hip.h +35 -0
  10. data/lib/tilework/compat/metal.h +45 -0
  11. data/lib/tilework/conformance.rb +2296 -0
  12. data/lib/tilework/core_ext.rb +21 -0
  13. data/lib/tilework/dialect.rb +67 -0
  14. data/lib/tilework/dsl.rb +722 -0
  15. data/lib/tilework/dtype/bfloat.rb +31 -0
  16. data/lib/tilework/dtype/cast.rb +88 -0
  17. data/lib/tilework/dtype/half.rb +98 -0
  18. data/lib/tilework/graph/builder.rb +1073 -0
  19. data/lib/tilework/graph/fusion.rb +520 -0
  20. data/lib/tilework/graph/grad.rb +443 -0
  21. data/lib/tilework/graph/high_level.rb +266 -0
  22. data/lib/tilework/graph/kernels.rb +403 -0
  23. data/lib/tilework/graph/node.rb +27 -0
  24. data/lib/tilework/graph/op.rb +290 -0
  25. data/lib/tilework/graph/optim.rb +183 -0
  26. data/lib/tilework/graph/plan.rb +1086 -0
  27. data/lib/tilework/graph/pool.rb +178 -0
  28. data/lib/tilework/graph/region.rb +210 -0
  29. data/lib/tilework/graph/residency.rb +449 -0
  30. data/lib/tilework/graph/state.rb +64 -0
  31. data/lib/tilework/graph/tensor.rb +124 -0
  32. data/lib/tilework/graph.rb +213 -0
  33. data/lib/tilework/kernel.rb +163 -0
  34. data/lib/tilework/math/conv.rb +270 -0
  35. data/lib/tilework/math/fft.rb +172 -0
  36. data/lib/tilework/math/nn.rb +1103 -0
  37. data/lib/tilework/math/philox.rb +194 -0
  38. data/lib/tilework/math/random.rb +232 -0
  39. data/lib/tilework/math/tensor_core.rb +269 -0
  40. data/lib/tilework/ops.rb +682 -0
  41. data/lib/tilework/runtime/offload.rb +498 -0
  42. data/lib/tilework/runtime/submission.rb +163 -0
  43. data/lib/tilework/runtime/telemetry.rb +112 -0
  44. data/lib/tilework/scope.rb +33 -0
  45. data/lib/tilework/version.rb +5 -0
  46. data/lib/tilework.rb +345 -0
  47. metadata +92 -0
@@ -0,0 +1,290 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Tilework
4
+ module Graph
5
+ # The contract for how an IR op behaves. REGISTRY is the single source of
6
+ # the op set and of each op's properties as the scheduler sees them.
7
+ #
8
+ # It is a table to stop things falling through the cracks. These
9
+ # properties used to be scattered across four constants in plan.rb
10
+ # (NON_OWNED / SYNCHRONOUS_OPS / FOREIGN_OPS / PLACEABLE), a case in
11
+ # execute, FUSABLE in fusion.rb and node.rb's labels, so adding an op meant
12
+ # touching six places. Forgetting one raised nothing; the scheduler just
13
+ # silently did the wrong thing (freeing a buffer it must not, not waiting
14
+ # where it must). Collected here, an undeclared op fails in Op.fetch — the
15
+ # same idea as Capabilities::FIELDS.
16
+ #
17
+ # op = Graph::Op.fetch(node.op)
18
+ # op.owned? # does it own its buffer (= may it be freed)?
19
+ # op.synchronous?(node) # does it wait in place instead of queueing?
20
+ #
21
+ # Properties that depend on node attributes or the backend's capabilities
22
+ # (batched matmul, delegated GEMM / RNG, ...) are declared as Procs;
23
+ # anything expressible as a boolean stays a boolean.
24
+ class Op
25
+ # NN functions delegated to a vendor library when gemm_library is set.
26
+ # They need waits on both sides for the same reason as matmul (decided
27
+ # without backend names).
28
+ FOREIGN_NN = %i[multi_head_attention].freeze
29
+
30
+ REGISTRY = {}
31
+
32
+ class << self
33
+ # One row of the table.
34
+ #
35
+ # run the Plan method to execute; all take (node, inputs, bindings)
36
+ # launches whether it launches a kernel (or copy). A node with
37
+ # false leaves the queue as it was, so after a synchronous
38
+ # node it is "still empty" — used to decide whether
39
+ # automatic offload's eviction can ride on that wait
40
+ # (#17 Phase 4c)
41
+ # owned whether it owns its buffer. Buffers of nodes with false
42
+ # (inputs, constants, state) belong outside the graph, so
43
+ # liveness leaves them alone
44
+ # placeable whether, as a write_into value, it may write straight
45
+ # into the state's view. Only ops that allocate their
46
+ # output once and fill it (#19 §3)
47
+ # fusable whether elementwise fusion can fold it into one kernel
48
+ # index_map whether it only changes how its input is read (fusion
49
+ # folds it into the index)
50
+ # synchronous whether it runs in place instead of queueing and moving on
51
+ # foreign whether it may run outside our queue (synchronize on both sides)
52
+ # requires the capability needed to run this node (nil if none).
53
+ # Plan#requires collects them to report what is missing
54
+ # before running
55
+ # writes_state whether it advances a mutable state's version
56
+ # label the line for explain / inspect (defaults to the op name)
57
+ def define(name, run:, owned: true, placeable: false, fusable: false, index_map: false,
58
+ synchronous: false, foreign: false, writes_state: false, requires: nil,
59
+ launches: true, label: nil)
60
+ raise Error, "graph op #{name.inspect} is already defined" if REGISTRY.key?(name)
61
+
62
+ REGISTRY[name] = new(name: name, run: run, owned: owned, placeable: placeable,
63
+ fusable: fusable, index_map: index_map, synchronous: synchronous,
64
+ foreign: foreign, writes_state: writes_state, requires: requires,
65
+ launches: launches, label: label)
66
+ end
67
+
68
+ def fetch(name)
69
+ REGISTRY.fetch(name) do
70
+ raise Error, "unknown graph op #{name.inspect} (known: #{names.map(&:inspect).join(', ')})"
71
+ end
72
+ end
73
+
74
+ def defined?(name) = REGISTRY.key?(name)
75
+ def names = REGISTRY.keys
76
+
77
+ # Whether a delegated GEMM leaves our queue, and so has to wait on both
78
+ # sides. matmul and :nn make the same decision, so it lives in one
79
+ # place. A bridge whose vendor GEMM is enqueued on the same queue as
80
+ # its kernels, in submission order, declares `ordered_gemm?`; ordering
81
+ # then covers it exactly as it covers a kernel and no fence is needed.
82
+ def foreign_gemm?
83
+ return false if Tilework.capabilities.gemm_library.nil?
84
+
85
+ bridge = Tilework.bridge
86
+ !(bridge.respond_to?(:ordered_gemm?) && bridge.ordered_gemm?)
87
+ end
88
+
89
+ # Whether a capability is actually usable; nil and false both mean no.
90
+ def satisfied?(capability)
91
+ value = Tilework.capabilities.public_send(capability)
92
+ !value.nil? && value != false
93
+ end
94
+
95
+ # The axis shown on a reduction (none for a whole-buffer reduction).
96
+ def axis_suffix(node)
97
+ axis = node.attrs[:axis]
98
+ return nil if axis.nil?
99
+
100
+ "axis #{axis}#{node.attrs[:keepdims] ? ' keepdims' : ''}"
101
+ end
102
+ end
103
+
104
+ attr_reader :name, :run
105
+
106
+ def initialize(name:, run:, owned:, placeable:, fusable:, index_map:, synchronous:, foreign:,
107
+ writes_state:, requires:, launches:, label:)
108
+ @name = name
109
+ @run = run
110
+ @owned = owned
111
+ @placeable = predicate(placeable)
112
+ @fusable = fusable
113
+ @index_map = index_map
114
+ @synchronous = predicate(synchronous)
115
+ @foreign = predicate(foreign)
116
+ @writes_state = writes_state
117
+ @requires = requires
118
+ @launches = launches
119
+ @label = label
120
+ end
121
+
122
+ # The capability needed to run this node (nil if none).
123
+ attr_reader :requires
124
+
125
+ def owned? = @owned
126
+ def fusable? = @fusable
127
+
128
+ # Whether it launches anything; false means the node leaves the queue as it was.
129
+ def launches? = @launches
130
+
131
+ # An op that creates no value and only changes how its source buffer is
132
+ # read (fusion folds it into the index).
133
+ def index_map? = @index_map
134
+ def writes_state? = @writes_state
135
+ def placeable?(node) = @placeable.call(node)
136
+ def synchronous?(node) = @synchronous.call(node)
137
+ def foreign?(node) = @foreign.call(node)
138
+ def label_for(node) = @label ? @label.call(node) : name.to_s
139
+
140
+ private
141
+
142
+ # Boolean properties are wrapped in Procs too, so callers need no case split.
143
+ def predicate(value) = value.respond_to?(:call) ? value : ->(_node) { value }
144
+ end
145
+
146
+ # --- The table -----------------------------------------------------------
147
+ #
148
+ # Structural nodes. They create no value (they point at existing buffers),
149
+ # so owned: false.
150
+
151
+ Op.define :input, run: :run_input, owned: false, launches: false,
152
+ label: ->(node) { "input #{node.attrs[:name].inspect}" }
153
+
154
+ Op.define :const, run: :run_const, owned: false, launches: false, label: ->(_node) { "const" }
155
+
156
+ Op.define :state_init, run: :run_state_init, owned: false, launches: false
157
+
158
+ # A read is a view by default, so it may or may not own memory; when
159
+ # materialized it may be freed, hence owned: true. The release side rejects views.
160
+ Op.define :state_read, run: :run_state_read,
161
+ label: ->(node) { "read #{node.state.name} v#{node.attrs[:version]} #{node.region.inspect}" }
162
+
163
+ Op.define :state_write, run: :run_state_write, owned: false, writes_state: true,
164
+ label: lambda { |node|
165
+ "write #{node.state.name} -> v#{node.attrs[:version]} #{node.region.inspect}"
166
+ }
167
+
168
+ Op.define :state_merge, run: :run_state_merge, owned: false, writes_state: true, launches: false,
169
+ label: ->(node) { "merge #{node.state.name} -> v#{node.attrs[:version]}" }
170
+
171
+ # Compute nodes.
172
+
173
+ Op.define :binary, run: :run_binary, placeable: true, fusable: true,
174
+ label: lambda { |node|
175
+ "#{node.attrs[:op]}#{node.attrs.key?(:scalar) ? " #{node.attrs[:scalar]}" : ''}"
176
+ }
177
+
178
+ Op.define :unary, run: :run_unary, placeable: true, fusable: true,
179
+ label: ->(node) { node.attrs[:fn].to_s }
180
+
181
+ # A whole-buffer reduction (axis: nil) folds as a tree, and the execution
182
+ # context takes over freeing the partials between passes (#86), so it keeps
183
+ # queueing and moves on. An axis reduction is one kernel with no
184
+ # intermediates (and its output can be placed).
185
+ Op.define :reduce, run: :run_reduce,
186
+ placeable: ->(node) { !node.attrs[:axis].nil? },
187
+ label: ->(node) { [node.attrs[:kind], Op.axis_suffix(node)].compact.join(" ") }
188
+
189
+ Op.define :arg_reduce, run: :run_arg_reduce,
190
+ placeable: ->(node) { !node.attrs[:axis].nil? },
191
+ label: ->(node) { ["arg#{node.attrs[:kind]}", Op.axis_suffix(node)].compact.join(" ") }
192
+
193
+ # Change the type / select by condition / write indices. Each is closed per
194
+ # element, so each can fuse (:iota has no inputs, so it is never absorbed
195
+ # as an operand, though it can be the root that owns the output).
196
+ Op.define :cast, run: :run_cast, placeable: true, fusable: true,
197
+ label: ->(node) { "cast #{node.dtype}" }
198
+
199
+ Op.define :where, run: :run_where, placeable: true, fusable: true
200
+
201
+ Op.define :iota, run: :run_iota, placeable: true,
202
+ label: ->(node) { "iota axis #{node.attrs[:axis]}" }
203
+
204
+ # A node that only changes the shape. Execution returns a view of its input:
205
+ # no kernel runs and no new buffer is owned (= owned: false). Being an
206
+ # alias, the source buffer's lifetime must extend to this node's consumers
207
+ # (Plan#alias_root).
208
+ Op.define :reshape, run: :run_reshape, owned: false, launches: false,
209
+ label: ->(node) { "reshape #{node.shape.inspect}" }
210
+
211
+ # A node that only remaps indices. It can fuse — but unlike other ops it
212
+ # folds **into the index**, not into the expression (the consumer's kernel
213
+ # reads the source buffer with remapped indices). Only when not fused does
214
+ # one materializing kernel run. Broadcast (stretching) and permute
215
+ # (swapping axes) are the same operation with different stride tables.
216
+ Op.define :broadcast, run: :run_remap, placeable: true, fusable: true, index_map: true
217
+
218
+ Op.define :permute, run: :run_remap, placeable: true, fusable: true, index_map: true,
219
+ label: ->(node) { "permute #{node.attrs[:axes].inspect}" }
220
+
221
+ # A slice of the outermost dimension. Returns a view, like reshape.
222
+ Op.define :slice, run: :run_slice, owned: false, launches: false,
223
+ label: ->(node) { "slice #{node.shape.inspect}" }
224
+
225
+ # Table lookup and border fill. Both allocate their output once and fill it.
226
+ Op.define :gather, run: :run_gather, placeable: true,
227
+ label: ->(node) { "gather axis #{node.attrs[:axis]}" }
228
+
229
+ # The inverse of table lookup. It allocates its output once and fills it
230
+ # (each thread finishes its own share), so it is placeable. No atomics, so
231
+ # it stays inside our queue.
232
+ Op.define :scatter_add, run: :run_scatter_add, placeable: true,
233
+ label: ->(node) { "scatter_add axis #{node.attrs[:axis]}" }
234
+
235
+ Op.define :pad, run: :run_pad, placeable: true,
236
+ label: ->(node) { "pad #{node.attrs[:before].inspect}" }
237
+
238
+ # matmul is delegated to cuBLAS / MPS when gemm_library is set. Where that
239
+ # leaves our execution path (Op.foreign_gemm?) it waits on both sides;
240
+ # a bridge that declares `ordered_gemm?` stays asynchronous. Freeing the
241
+ # per-batch temporaries of an undelegated batch is handled by #86's
242
+ # deferred free.
243
+ Op.define :matmul, run: :run_matmul,
244
+ placeable: ->(node) { node.attrs[:mode] != :batched },
245
+ synchronous: ->(_node) { Op.foreign_gemm? },
246
+ foreign: ->(_node) { Op.foreign_gemm? }
247
+
248
+ Op.define :concat, run: :run_concat, placeable: true
249
+
250
+ # A delegated RNG (cuRAND / MPS) leaves our execution path, and some paths
251
+ # fall back to writing from the host.
252
+ Op.define :random, run: :run_random, synchronous: true,
253
+ foreign: ->(_node) { !Tilework.capabilities.rng.nil? }
254
+
255
+ # NN's fused kernels hand freeing their intermediates (scores / probs) to the
256
+ # execution context (#86), so they wait only when delegated.
257
+ Op.define :nn, run: :run_nn,
258
+ synchronous: ->(node) { Op::FOREIGN_NN.include?(node.attrs[:fn]) && Op.foreign_gemm? },
259
+ foreign: ->(node) { Op::FOREIGN_NN.include?(node.attrs[:fn]) && Op.foreign_gemm? },
260
+ label: ->(node) { ["nn", node.attrs[:fn], *node.attrs[:args]].join(" ") }
261
+
262
+ # 2D convolution. Delegated to cuDNN / MPSGraph, so it waits on both sides.
263
+ # On a backend without convolution it is decomposed into primitives at
264
+ # record time and never becomes this node (graph/high_level.rb's :conv2d).
265
+ Op.define :conv2d, run: :run_conv2d, synchronous: true, foreign: true, requires: :conv,
266
+ label: ->(node) {
267
+ "conv2d pad #{node.attrs[:pad].inspect} stride #{node.attrs[:stride].inspect} " \
268
+ "dilation #{node.attrs[:dilation].inspect} groups #{node.attrs[:groups].inspect}"
269
+ }
270
+
271
+ # Convolution backward (#22 Step 2). Runs on the delegate like the forward
272
+ # pass, so it waits on both sides.
273
+ %i[data filter].each do |kind|
274
+ Op.define :"conv2d_backward_#{kind}", run: :"run_conv2d_backward_#{kind}",
275
+ synchronous: true, foreign: true, requires: :conv_backward,
276
+ label: ->(node) {
277
+ "conv2d backward #{kind} pad #{node.attrs[:pad].inspect} " \
278
+ "dilation #{node.attrs[:dilation].inspect} groups #{node.attrs[:groups].inspect}"
279
+ }
280
+ end
281
+
282
+ # A user block. What it does is unknown, so by default it waits on both
283
+ # sides. `g.opaque(..., fenced: false)` is the caller's declaration that
284
+ # the block stays inside our queue, and only then is the fence removed.
285
+ Op.define :opaque, run: :run_opaque,
286
+ synchronous: ->(node) { node.attrs[:fenced] },
287
+ foreign: ->(node) { node.attrs[:fenced] },
288
+ label: ->(node) { "opaque #{node.attrs[:name]}" }
289
+ end
290
+ end
@@ -0,0 +1,183 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Tilework
4
+ module Graph
5
+ # Base class for graph-recorded parameter optimizers.
6
+ #
7
+ # An optimizer is **not a new mechanism** — it is a helper on top of
8
+ # mutable state (#19) and gradients (graph/grad.rb) that does three things:
9
+ #
10
+ # 1. declare the parameters as g.state and return the read Tensors (attach)
11
+ # 2. take the gradients of the loss (g.grad)
12
+ # 3. write back the moments and parameters (g.write_into)
13
+ #
14
+ # It all lands in one plan, so **a step is one plan.run**. The update is
15
+ # elementwise, so fusion folds it to one kernel per parameter.
16
+ #
17
+ # opt = Tilework::Graph::Adam.new({ w1: w1, b1: b1 }, lr: 1e-3)
18
+ #
19
+ # plan = Tilework::Graph.build(differentiable: true) do |g|
20
+ # x = g.input(:x, [n, d])
21
+ # p = opt.attach(g) # { w1: Tensor, b1: Tensor }
22
+ # loss = Tilework::NN.cross_entropy(x.matmul(p[:w1]) + p[:b1], labels)
23
+ # opt.step(loss) # records gradients and the update
24
+ # g.output(loss)
25
+ # end.compile
26
+ #
27
+ # 50.times { plan.run(x: xs) } # the weight Buffers update in place
28
+ #
29
+ # **Buffers are allocated outside Graph.build** (in Adam.new), keeping the
30
+ # graph layer's promise that recording touches no device.
31
+ class Optimizer
32
+ attr_reader :params, :grads
33
+
34
+ # Creates an optimizer for mutable parameter buffers.
35
+ #
36
+ # @param params [Hash{Symbol, String => Buffer}] named parameter buffers
37
+ # @return [Optimizer]
38
+ def initialize(params)
39
+ raise ArgumentError, "an optimizer needs at least one parameter" if params.empty?
40
+
41
+ @params = params.transform_keys(&:to_sym)
42
+ @refs = {}
43
+ @grads = {}
44
+ @slots = {} # the optimizer's own mutable state (moments etc.)
45
+ allocate_slots!
46
+ end
47
+
48
+ # Attaches parameter buffers as graph state and returns their tensors.
49
+ #
50
+ # @param builder [Builder]
51
+ # @return [Hash{Symbol => Tensor}]
52
+ #
53
+ # **The forward pass must use these Tensors.** Binding the parameters
54
+ # separately with g.param still yields gradients, but the update would be
55
+ # written somewhere else and have no effect.
56
+ def attach(builder)
57
+ @builder = builder
58
+ @refs = @params.to_h { |name, buffer| [name, builder.state(buffer, name: name)] }
59
+ @slot_refs = @slots.to_h { |key, buffer| [key, builder.state(buffer, name: key)] }
60
+ @reads = @refs.transform_values { |ref| builder.read(ref) }
61
+ end
62
+
63
+ # Records gradient calculation and one parameter update step.
64
+ #
65
+ # @param loss [Tensor] scalar loss tensor
66
+ # @return [Hash{Symbol => Tensor}] gradients by parameter name
67
+ #
68
+ # The gradients are returned so they can be inspected (for clipping or logging).
69
+ def step(loss)
70
+ raise Error, "call attach(g) before step(loss)" if @builder.nil?
71
+
72
+ names = @reads.keys
73
+ gradients = @builder.grad(loss, wrt: names.map { |name| @reads[name] })
74
+ @grads = names.zip(gradients).to_h
75
+ names.each { |name| update(name, @reads[name], @grads[name]) }
76
+ @grads
77
+ end
78
+
79
+ # Releases optimizer-owned state buffers such as momentum slots.
80
+ #
81
+ # @return [Hash]
82
+ def free!
83
+ @slots.each_value(&:free)
84
+ @slots = {}
85
+ end
86
+
87
+ private
88
+
89
+ # One zero-filled buffer per name (Buffer.new zero-fills on allocation).
90
+ def zeros_for(name, suffix)
91
+ buffer = @params.fetch(name)
92
+ @slots[:"#{name}_#{suffix}"] = Buffer.new(buffer.dtype, buffer.length, shape: buffer.shape)
93
+ end
94
+
95
+ def slot(name, suffix) = @slot_refs.fetch(:"#{name}_#{suffix}")
96
+
97
+ def allocate_slots!
98
+ # No extra state by default (SGD).
99
+ end
100
+
101
+ def update(_name, _param, _grad)
102
+ raise NotImplementedError, "#{self.class} must implement #update"
103
+ end
104
+ end
105
+
106
+ # Stochastic gradient descent with `p -= lr * gradient` updates.
107
+ #
108
+ # No momentum, deliberately; adding it would take the same shape as Adam
109
+ # (one state + one update line) when it is needed.
110
+ class SGD < Optimizer
111
+ # @param params [Hash{Symbol, String => Buffer}]
112
+ # @param lr [Float] learning rate
113
+ def initialize(params, lr: 1e-2)
114
+ @lr = lr.to_f
115
+ super(params)
116
+ end
117
+
118
+ private
119
+
120
+ def update(name, param, grad)
121
+ @builder.write_into(@refs.fetch(name), value: param - (@lr * grad))
122
+ end
123
+ end
124
+
125
+ # Adam optimizer with bias correction.
126
+ #
127
+ # m = β1·m + (1-β1)·g, v = β2·v + (1-β2)·g²
128
+ # p -= lr · (m / (1-β1^t)) / (sqrt(v / (1-β2^t)) + eps)
129
+ #
130
+ # The bias-correction t is **state inside the graph**. As a plan input,
131
+ # every caller would have to pass a step count only Adam needs. Since t is
132
+ # a run-time value, β^t is written `exp(t · ln β)` (pow is not a primitive
133
+ # and this is the only place that wants it).
134
+ class Adam < Optimizer
135
+ # @param params [Hash{Symbol, String => Buffer}]
136
+ # @param lr [Float] learning rate
137
+ # @param beta1 [Float] first-moment decay
138
+ # @param beta2 [Float] second-moment decay
139
+ # @param eps [Float] denominator stability term
140
+ def initialize(params, lr: 1e-3, beta1: 0.9, beta2: 0.999, eps: 1e-8)
141
+ @lr = lr.to_f
142
+ @beta1 = beta1.to_f
143
+ @beta2 = beta2.to_f
144
+ @eps = eps.to_f
145
+ super(params)
146
+ end
147
+
148
+ def attach(builder)
149
+ super
150
+ # t is a one-element state shared by all parameters.
151
+ @step_ref = @slot_refs.fetch(:__step)
152
+ t = builder.read(@step_ref) + 1.0
153
+ builder.write_into(@step_ref, value: t)
154
+ # 1 - β^t. t has shape [1], so it broadcasts into the expressions below.
155
+ @correction1 = 1.0 - (t * Math.log(@beta1)).exp
156
+ @correction2 = 1.0 - (t * Math.log(@beta2)).exp
157
+ @reads
158
+ end
159
+
160
+ private
161
+
162
+ def allocate_slots!
163
+ @params.each_key do |name|
164
+ zeros_for(name, :m)
165
+ zeros_for(name, :v)
166
+ end
167
+ @slots[:__step] = Buffer.new(:float32, 1, shape: [1])
168
+ end
169
+
170
+ def update(name, param, grad)
171
+ m_ref = slot(name, :m)
172
+ v_ref = slot(name, :v)
173
+ m = (@beta1 * @builder.read(m_ref)) + ((1.0 - @beta1) * grad)
174
+ v = (@beta2 * @builder.read(v_ref)) + ((1.0 - @beta2) * grad * grad)
175
+ @builder.write_into(m_ref, value: m)
176
+ @builder.write_into(v_ref, value: v)
177
+
178
+ step = (m / @correction1) / ((v / @correction2).sqrt + @eps)
179
+ @builder.write_into(@refs.fetch(name), value: param - (@lr * step))
180
+ end
181
+ end
182
+ end
183
+ end