tilework 0.18.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE +21 -0
- data/lib/tilework/backend/cpu/dsl_codegen.rb +509 -0
- data/lib/tilework/backend/cpu/ops_codegen.rb +451 -0
- data/lib/tilework/backend/cpu.rb +156 -0
- data/lib/tilework/buffer.rb +753 -0
- data/lib/tilework/capabilities.rb +183 -0
- data/lib/tilework/compat/cuda.h +66 -0
- data/lib/tilework/compat/hip.h +35 -0
- data/lib/tilework/compat/metal.h +45 -0
- data/lib/tilework/conformance.rb +2296 -0
- data/lib/tilework/core_ext.rb +21 -0
- data/lib/tilework/dialect.rb +67 -0
- data/lib/tilework/dsl.rb +722 -0
- data/lib/tilework/dtype/bfloat.rb +31 -0
- data/lib/tilework/dtype/cast.rb +88 -0
- data/lib/tilework/dtype/half.rb +98 -0
- data/lib/tilework/graph/builder.rb +1073 -0
- data/lib/tilework/graph/fusion.rb +520 -0
- data/lib/tilework/graph/grad.rb +443 -0
- data/lib/tilework/graph/high_level.rb +266 -0
- data/lib/tilework/graph/kernels.rb +403 -0
- data/lib/tilework/graph/node.rb +27 -0
- data/lib/tilework/graph/op.rb +290 -0
- data/lib/tilework/graph/optim.rb +183 -0
- data/lib/tilework/graph/plan.rb +1086 -0
- data/lib/tilework/graph/pool.rb +178 -0
- data/lib/tilework/graph/region.rb +210 -0
- data/lib/tilework/graph/residency.rb +449 -0
- data/lib/tilework/graph/state.rb +64 -0
- data/lib/tilework/graph/tensor.rb +124 -0
- data/lib/tilework/graph.rb +213 -0
- data/lib/tilework/kernel.rb +163 -0
- data/lib/tilework/math/conv.rb +270 -0
- data/lib/tilework/math/fft.rb +172 -0
- data/lib/tilework/math/nn.rb +1103 -0
- data/lib/tilework/math/philox.rb +194 -0
- data/lib/tilework/math/random.rb +232 -0
- data/lib/tilework/math/tensor_core.rb +269 -0
- data/lib/tilework/ops.rb +682 -0
- data/lib/tilework/runtime/offload.rb +498 -0
- data/lib/tilework/runtime/submission.rb +163 -0
- data/lib/tilework/runtime/telemetry.rb +112 -0
- data/lib/tilework/scope.rb +33 -0
- data/lib/tilework/version.rb +5 -0
- data/lib/tilework.rb +345 -0
- metadata +92 -0
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Tilework
|
|
4
|
+
module Graph
|
|
5
|
+
# A pool of intermediate buffers. Buffers coming back from liveness are kept
|
|
6
|
+
# per `[dtype, shape]` and handed out again.
|
|
7
|
+
#
|
|
8
|
+
# Eager does a `Buffer.new` per op, so a five-step elementwise chain
|
|
9
|
+
# allocates and drops four intermediates. The graph knows the execution
|
|
10
|
+
# order and each value's last use, so **rotating slots of the same shape**
|
|
11
|
+
# is enough. The pool belongs to the plan, so reruns do not reallocate
|
|
12
|
+
# either (which pays off in decode loops rerunning one plan).
|
|
13
|
+
#
|
|
14
|
+
# Only exact shape matches are reused — the same length with a different
|
|
15
|
+
# shape is a different slot, because reshape makes views and returning one
|
|
16
|
+
# to the pool would lose track of the owning buffer.
|
|
17
|
+
#
|
|
18
|
+
# It is safe with asynchronous launches because **reuse is not freeing**.
|
|
19
|
+
# The next kernel to use a slot is queued after the last kernel that read
|
|
20
|
+
# it, so same-queue ordering holds. Calling `Buffer#free` here would release
|
|
21
|
+
# memory a running kernel uses (undefined behavior on CUDA).
|
|
22
|
+
class Pool
|
|
23
|
+
attr_reader :allocated, :reused
|
|
24
|
+
|
|
25
|
+
def initialize
|
|
26
|
+
@slots = Hash.new { |hash, key| hash[key] = [] }
|
|
27
|
+
@allocated = 0
|
|
28
|
+
@reused = 0
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
def allocate(dtype, length, shape)
|
|
32
|
+
slot = @slots[[dtype, shape]]
|
|
33
|
+
while (buffer = slot.pop)
|
|
34
|
+
# It may have been freed from outside the plan, e.g. by a Scope (a plan
|
|
35
|
+
# run inside Tilework.run). Drop it rather than hold on.
|
|
36
|
+
next if buffer.freed?
|
|
37
|
+
|
|
38
|
+
@reused += 1
|
|
39
|
+
buffer.instance_variable_set(:@graph_pool_released, false)
|
|
40
|
+
return buffer
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
@allocated += 1
|
|
44
|
+
Buffer.new(dtype, length, shape: shape).tap do |buffer|
|
|
45
|
+
# Marks the buffer so Plan#release returns only what this pool lent.
|
|
46
|
+
# A separate set of object_ids would either keep plan outputs that left
|
|
47
|
+
# alive or misread reused ids, so the owner is recorded on the buffer.
|
|
48
|
+
buffer.instance_variable_set(:@graph_pool_owner, self)
|
|
49
|
+
end
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
def owns?(buffer) = buffer.instance_variable_get(:@graph_pool_owner).equal?(self)
|
|
53
|
+
|
|
54
|
+
# Adopts a buffer allocated outside the pool (a temporary an op made with
|
|
55
|
+
# `Buffer.new`, like `nn.rb`'s scores), so what arrives through deferred
|
|
56
|
+
# frees is reused on the next run.
|
|
57
|
+
def adopt(buffer)
|
|
58
|
+
buffer.instance_variable_set(:@graph_pool_owner, self)
|
|
59
|
+
buffer
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
# Returning is **idempotent**. `nn.rb`'s error path frees twice with
|
|
63
|
+
# `free unless freed?`, and `freed?` stays false while a free is deferred,
|
|
64
|
+
# so a naive push would lend the same buffer to two nodes as two slots.
|
|
65
|
+
def release(buffer)
|
|
66
|
+
return buffer if buffer.instance_variable_get(:@graph_pool_released)
|
|
67
|
+
|
|
68
|
+
buffer.instance_variable_set(:@graph_pool_released, true)
|
|
69
|
+
@slots[[buffer.dtype, buffer.shape]].push(buffer)
|
|
70
|
+
buffer
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
def pooled = @slots.each_value.sum(&:length)
|
|
74
|
+
|
|
75
|
+
def stats = { allocated: @allocated, reused: @reused, pooled: pooled }
|
|
76
|
+
|
|
77
|
+
# Releases the device memory the pool holds. Running kernels may still read
|
|
78
|
+
# the slots, so wait first.
|
|
79
|
+
def free_all!
|
|
80
|
+
Tilework.synchronize
|
|
81
|
+
@slots.each_value { |buffers| buffers.each(&:free) }
|
|
82
|
+
@slots.clear
|
|
83
|
+
nil
|
|
84
|
+
end
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
# The context plugged into Ops while running (see the Ops::CONTEXT_KEY
|
|
88
|
+
# comment). Routes output allocation to the pool and makes launches async.
|
|
89
|
+
class Context
|
|
90
|
+
def initialize(pool: nil, async: true)
|
|
91
|
+
@pool = pool
|
|
92
|
+
@async = async
|
|
93
|
+
@outer = nil
|
|
94
|
+
@deferred = []
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
attr_reader :pool
|
|
98
|
+
|
|
99
|
+
# When there is an enclosing `Tilework.submit` boundary, ownership can be
|
|
100
|
+
# handed to it (#86 stage 2). The plan keeps its own pool and passes only
|
|
101
|
+
# buffers from outside the pool to the Job.
|
|
102
|
+
attr_accessor :outer
|
|
103
|
+
|
|
104
|
+
# Lets ops that free buffers mid-call (`nn.rb`'s scores, `Ops.reduce`'s
|
|
105
|
+
# partials) run **asynchronously**: memory a running kernel reads is never
|
|
106
|
+
# released, so there is no reason left to run them synchronously (#86).
|
|
107
|
+
#
|
|
108
|
+
# With a pool the buffer is returned to it — Pool's safety argument ("the
|
|
109
|
+
# next writer of a slot is queued after its last reader") applies as is.
|
|
110
|
+
# Views own nothing, so they are not taken and are freed on the spot (the
|
|
111
|
+
# same judgment as Plan#release).
|
|
112
|
+
def defer_free(buffer)
|
|
113
|
+
return false if buffer.view?
|
|
114
|
+
|
|
115
|
+
if @pool
|
|
116
|
+
@pool.adopt(buffer) unless @pool.owns?(buffer)
|
|
117
|
+
return @pool.release(buffer)
|
|
118
|
+
end
|
|
119
|
+
return @outer.defer_free(buffer) if @outer
|
|
120
|
+
|
|
121
|
+
@deferred << buffer
|
|
122
|
+
end
|
|
123
|
+
|
|
124
|
+
# What piled up with neither a pool nor an outer boundary
|
|
125
|
+
# (`Graph.buffer_pool = false`). Running kernels may read it, so wait first.
|
|
126
|
+
def drain_deferred!
|
|
127
|
+
return if @deferred.empty?
|
|
128
|
+
|
|
129
|
+
pending = @deferred
|
|
130
|
+
@deferred = []
|
|
131
|
+
Tilework.synchronize
|
|
132
|
+
pending.each(&:free)
|
|
133
|
+
nil
|
|
134
|
+
end
|
|
135
|
+
|
|
136
|
+
def async? = @async
|
|
137
|
+
|
|
138
|
+
# Makes the next allocated output **this buffer** instead of a new one, so
|
|
139
|
+
# the node producing a `write_into` value gets the state's view as its
|
|
140
|
+
# output from the start (#19 §3) — removing one copy.
|
|
141
|
+
#
|
|
142
|
+
# It applies once, and only if the dtype and length match. Otherwise a
|
|
143
|
+
# normal allocation happens and `consumed?` stays false, so the caller
|
|
144
|
+
# knows it could not place and falls back to the copy.
|
|
145
|
+
def place(buffer)
|
|
146
|
+
@placement = buffer
|
|
147
|
+
@placement_consumed = false
|
|
148
|
+
yield
|
|
149
|
+
ensure
|
|
150
|
+
@placement = nil
|
|
151
|
+
end
|
|
152
|
+
|
|
153
|
+
def placement_consumed? = @placement_consumed
|
|
154
|
+
|
|
155
|
+
def allocate(dtype, length, shape)
|
|
156
|
+
if @placement && @placement.dtype == dtype && @placement.length == length
|
|
157
|
+
placed = @placement
|
|
158
|
+
@placement = nil
|
|
159
|
+
@placement_consumed = true
|
|
160
|
+
return placed
|
|
161
|
+
end
|
|
162
|
+
return Buffer.new(dtype, length, shape: shape) unless @pool
|
|
163
|
+
|
|
164
|
+
@pool.allocate(dtype, length, shape)
|
|
165
|
+
end
|
|
166
|
+
|
|
167
|
+
# Nodes that may run outside our queue, like delegated libraries and user
|
|
168
|
+
# blocks, run synchronously.
|
|
169
|
+
def synchronous
|
|
170
|
+
previous = @async
|
|
171
|
+
@async = false
|
|
172
|
+
yield
|
|
173
|
+
ensure
|
|
174
|
+
@async = previous
|
|
175
|
+
end
|
|
176
|
+
end
|
|
177
|
+
end
|
|
178
|
+
end
|
|
@@ -0,0 +1,210 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Tilework
|
|
4
|
+
module Graph
|
|
5
|
+
# The target region of a write to / read from mutable state.
|
|
6
|
+
#
|
|
7
|
+
# Represented as **a strided block over the flat element space**
|
|
8
|
+
# `(offset, block, stride, count)`: block elements from offset, repeated
|
|
9
|
+
# count times every stride. With count == 1 it is just the contiguous
|
|
10
|
+
# range `[offset, offset + block)`.
|
|
11
|
+
#
|
|
12
|
+
# channels c0...c1 of [n, c, h, w] → offset = c0*h*w, block = (c1-c0)*h*w,
|
|
13
|
+
# stride = c*h*w, count = n
|
|
14
|
+
#
|
|
15
|
+
# The reason for this form is overlap testing. Deciding that "writes to
|
|
16
|
+
# disjoint regions need not be serialized" needs an exact test, which is
|
|
17
|
+
# undecidable for arbitrary index expressions. Strided blocks express every
|
|
18
|
+
# shape actually needed (outermost-dimension slices and channel ranges) and
|
|
19
|
+
# the test is O(1).
|
|
20
|
+
#
|
|
21
|
+
# **Shapes it cannot decide answer "may overlap"** (#disjoint?). Erring on
|
|
22
|
+
# the safe side loses only parallelism, never correctness.
|
|
23
|
+
#
|
|
24
|
+
# offset can be a Param as well as an Integer. An offset **decided at run
|
|
25
|
+
# time**, like a KV cache's write position, is resolved from the plan's
|
|
26
|
+
# bindings (`plan.run(pos: 3)`). static? is then false and the overlap test
|
|
27
|
+
# always takes the safe side.
|
|
28
|
+
class Region
|
|
29
|
+
# An offset decided at run time; its value is `bias + bindings[name] * scale`.
|
|
30
|
+
Param = Struct.new(:name, :scale, :bias) do
|
|
31
|
+
def resolve(bindings)
|
|
32
|
+
value = bindings.fetch(name) do
|
|
33
|
+
raise Error, "plan.run is missing the runtime index #{name.inspect} " \
|
|
34
|
+
"(used as a write/read position)"
|
|
35
|
+
end
|
|
36
|
+
unless value.is_a?(Integer) && !value.negative?
|
|
37
|
+
raise ArgumentError, "runtime index #{name.inspect} must be a non-negative Integer, got #{value.inspect}"
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
bias + (value * scale)
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def key = [:param, name, scale, bias]
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
attr_reader :offset, :block, :stride, :count, :total, :shape
|
|
47
|
+
|
|
48
|
+
# total is the target buffer's element count (for range checks). shape is
|
|
49
|
+
# the logical shape of the tensor read from this region.
|
|
50
|
+
def initialize(offset:, block:, count: 1, stride: nil, total:, shape: nil)
|
|
51
|
+
@offset = offset
|
|
52
|
+
@block = block
|
|
53
|
+
@count = count
|
|
54
|
+
@stride = stride || block
|
|
55
|
+
@total = total
|
|
56
|
+
@shape = shape || [block * count]
|
|
57
|
+
validate!
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
# The whole buffer.
|
|
61
|
+
def self.whole(shape)
|
|
62
|
+
total = shape.inject(1, :*)
|
|
63
|
+
new(offset: 0, block: total, total: total, shape: shape)
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
# A slice along the outermost dimension (Buffer#slice semantics); always
|
|
67
|
+
# contiguous. range takes an Integer / Range or a run-time position (Index).
|
|
68
|
+
def self.rows(shape, range)
|
|
69
|
+
raise Error, "rows requires a shaped region (shape: #{shape.inspect})" if shape.length < 2
|
|
70
|
+
|
|
71
|
+
row = shape.drop(1).inject(1, :*)
|
|
72
|
+
total = shape.inject(1, :*)
|
|
73
|
+
first, count = normalize(range, shape[0])
|
|
74
|
+
offset = first.is_a?(Param) ? Param.new(first.name, first.scale * row, first.bias * row) : first * row
|
|
75
|
+
new(offset: offset, block: count * row, total: total, shape: [count, *shape.drop(1)])
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
# A slice along any axis: the same contiguous region as rows for
|
|
79
|
+
# axis == 0, a strided block for axis > 0.
|
|
80
|
+
def self.axis(shape, axis, range)
|
|
81
|
+
unless axis.is_a?(Integer) && (0...shape.length).cover?(axis)
|
|
82
|
+
raise ArgumentError, "axis #{axis.inspect} is out of range for shape #{shape.inspect}"
|
|
83
|
+
end
|
|
84
|
+
return rows(shape, range) if axis.zero?
|
|
85
|
+
|
|
86
|
+
inner = shape.drop(axis + 1).inject(1, :*)
|
|
87
|
+
outer = shape.take(axis).inject(1, :*)
|
|
88
|
+
first, count = normalize(range, shape[axis])
|
|
89
|
+
if first.is_a?(Param)
|
|
90
|
+
raise Error, "a runtime index is only supported on the outermost axis (got axis #{axis})"
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
out_shape = shape.dup
|
|
94
|
+
out_shape[axis] = count
|
|
95
|
+
new(offset: first * inner, block: count * inner, count: outer,
|
|
96
|
+
stride: shape[axis] * inner, total: shape.inject(1, :*), shape: out_shape)
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
# Equivalent to Buffer#normalize_range (private there, so the rule is repeated here).
|
|
100
|
+
def self.normalize(range, limit)
|
|
101
|
+
case range
|
|
102
|
+
when Index
|
|
103
|
+
[Param.new(range.name, 1, 0), range.count]
|
|
104
|
+
when Param
|
|
105
|
+
[range, 1]
|
|
106
|
+
when Integer
|
|
107
|
+
idx = range.negative? ? limit + range : range
|
|
108
|
+
raise ArgumentError, "index #{range} is out of bounds (#{limit})" unless (0...limit).cover?(idx)
|
|
109
|
+
|
|
110
|
+
[idx, 1]
|
|
111
|
+
when Range
|
|
112
|
+
first = range.begin || 0
|
|
113
|
+
first += limit if first.negative?
|
|
114
|
+
last = range.end || limit
|
|
115
|
+
last += limit if last.negative?
|
|
116
|
+
last -= 1 if range.exclude_end? && range.end
|
|
117
|
+
count = last - first + 1
|
|
118
|
+
raise ArgumentError, "range #{range} is out of bounds (#{limit})" unless first >= 0 && count.positive? && first + count <= limit
|
|
119
|
+
|
|
120
|
+
[first, count]
|
|
121
|
+
else
|
|
122
|
+
raise ArgumentError, "expected an Integer, a Range, or a runtime index, got #{range.class}"
|
|
123
|
+
end
|
|
124
|
+
end
|
|
125
|
+
private_class_method :normalize
|
|
126
|
+
|
|
127
|
+
def length = block * count
|
|
128
|
+
def contiguous? = count == 1
|
|
129
|
+
def static? = offset.is_a?(Integer)
|
|
130
|
+
|
|
131
|
+
# Applies run-time bindings and returns a Region whose offset is a fixed Integer.
|
|
132
|
+
def resolve(bindings)
|
|
133
|
+
return self if static?
|
|
134
|
+
|
|
135
|
+
concrete = Region.new(offset: offset.resolve(bindings), block: block, count: count,
|
|
136
|
+
stride: stride, total: total, shape: shape)
|
|
137
|
+
concrete.bounds_check!
|
|
138
|
+
concrete
|
|
139
|
+
end
|
|
140
|
+
|
|
141
|
+
def bounds_check!
|
|
142
|
+
last = offset + ((count - 1) * stride) + block
|
|
143
|
+
return if last <= total
|
|
144
|
+
|
|
145
|
+
raise Error, "region #{inspect} runs past the end of a #{total}-element buffer"
|
|
146
|
+
end
|
|
147
|
+
|
|
148
|
+
# Yields [offset in the target buffer, offset in the value, length] per
|
|
149
|
+
# block. The value (written from / read into) is treated as the selected
|
|
150
|
+
# region packed contiguously.
|
|
151
|
+
def each_block
|
|
152
|
+
return enum_for(:each_block) unless block_given?
|
|
153
|
+
|
|
154
|
+
count.times { |i| yield(offset + (i * stride), i * block, self.block) }
|
|
155
|
+
end
|
|
156
|
+
|
|
157
|
+
# **true only when the regions certainly do not overlap**; shapes it
|
|
158
|
+
# cannot decide answer false.
|
|
159
|
+
#
|
|
160
|
+
# - either has a run-time offset → false (conservative)
|
|
161
|
+
# - both contiguous → exact interval test
|
|
162
|
+
# - equal strides → exact test on the phase within the stride
|
|
163
|
+
# - different strides → true only when the overall spans are apart
|
|
164
|
+
def disjoint?(other)
|
|
165
|
+
return false unless static? && other.static?
|
|
166
|
+
return spans_disjoint?(other) if contiguous? && other.contiguous?
|
|
167
|
+
return spans_disjoint?(other) unless stride == other.stride
|
|
168
|
+
return true if spans_disjoint?(other)
|
|
169
|
+
|
|
170
|
+
# On the same lattice, disjoint if the phases within the stride do not overlap.
|
|
171
|
+
phase = (other.offset - offset) % stride
|
|
172
|
+
!(phase < self.block || stride - phase < other.block)
|
|
173
|
+
end
|
|
174
|
+
|
|
175
|
+
def spans_disjoint?(other)
|
|
176
|
+
span_end <= other.offset || other.span_end <= offset
|
|
177
|
+
end
|
|
178
|
+
|
|
179
|
+
def span_end = offset + ((count - 1) * stride) + block
|
|
180
|
+
|
|
181
|
+
# For the CSE key. A run-time offset compares by Param identity, not value.
|
|
182
|
+
def key
|
|
183
|
+
[static? ? offset : offset.key, block, stride, count, total, shape]
|
|
184
|
+
end
|
|
185
|
+
|
|
186
|
+
def inspect
|
|
187
|
+
head = static? ? offset.to_s : "#{offset.name}*#{offset.scale}+#{offset.bias}"
|
|
188
|
+
contiguous? ? "#<Region #{head}...+#{block}>" : "#<Region #{head}...+#{block} x#{count} stride #{stride}>"
|
|
189
|
+
end
|
|
190
|
+
|
|
191
|
+
private
|
|
192
|
+
|
|
193
|
+
def validate!
|
|
194
|
+
raise ArgumentError, "block must be a positive Integer" unless @block.is_a?(Integer) && @block.positive?
|
|
195
|
+
raise ArgumentError, "count must be a positive Integer" unless @count.is_a?(Integer) && @count.positive?
|
|
196
|
+
raise ArgumentError, "stride must be at least block" if @stride < @block
|
|
197
|
+
unless @shape.inject(1, :*) == length
|
|
198
|
+
raise ArgumentError, "region shape #{@shape.inspect} does not match #{length} elements"
|
|
199
|
+
end
|
|
200
|
+
|
|
201
|
+
bounds_check! if static?
|
|
202
|
+
end
|
|
203
|
+
end
|
|
204
|
+
|
|
205
|
+
# A position decided at run time, resolved from `plan.run(pos: 3)` bindings.
|
|
206
|
+
#
|
|
207
|
+
# g.write_into(kv0, at: g.index(:pos), value: k_new)
|
|
208
|
+
Index = Struct.new(:name, :count)
|
|
209
|
+
end
|
|
210
|
+
end
|