tilework 0.18.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE +21 -0
  3. data/lib/tilework/backend/cpu/dsl_codegen.rb +509 -0
  4. data/lib/tilework/backend/cpu/ops_codegen.rb +451 -0
  5. data/lib/tilework/backend/cpu.rb +156 -0
  6. data/lib/tilework/buffer.rb +753 -0
  7. data/lib/tilework/capabilities.rb +183 -0
  8. data/lib/tilework/compat/cuda.h +66 -0
  9. data/lib/tilework/compat/hip.h +35 -0
  10. data/lib/tilework/compat/metal.h +45 -0
  11. data/lib/tilework/conformance.rb +2296 -0
  12. data/lib/tilework/core_ext.rb +21 -0
  13. data/lib/tilework/dialect.rb +67 -0
  14. data/lib/tilework/dsl.rb +722 -0
  15. data/lib/tilework/dtype/bfloat.rb +31 -0
  16. data/lib/tilework/dtype/cast.rb +88 -0
  17. data/lib/tilework/dtype/half.rb +98 -0
  18. data/lib/tilework/graph/builder.rb +1073 -0
  19. data/lib/tilework/graph/fusion.rb +520 -0
  20. data/lib/tilework/graph/grad.rb +443 -0
  21. data/lib/tilework/graph/high_level.rb +266 -0
  22. data/lib/tilework/graph/kernels.rb +403 -0
  23. data/lib/tilework/graph/node.rb +27 -0
  24. data/lib/tilework/graph/op.rb +290 -0
  25. data/lib/tilework/graph/optim.rb +183 -0
  26. data/lib/tilework/graph/plan.rb +1086 -0
  27. data/lib/tilework/graph/pool.rb +178 -0
  28. data/lib/tilework/graph/region.rb +210 -0
  29. data/lib/tilework/graph/residency.rb +449 -0
  30. data/lib/tilework/graph/state.rb +64 -0
  31. data/lib/tilework/graph/tensor.rb +124 -0
  32. data/lib/tilework/graph.rb +213 -0
  33. data/lib/tilework/kernel.rb +163 -0
  34. data/lib/tilework/math/conv.rb +270 -0
  35. data/lib/tilework/math/fft.rb +172 -0
  36. data/lib/tilework/math/nn.rb +1103 -0
  37. data/lib/tilework/math/philox.rb +194 -0
  38. data/lib/tilework/math/random.rb +232 -0
  39. data/lib/tilework/math/tensor_core.rb +269 -0
  40. data/lib/tilework/ops.rb +682 -0
  41. data/lib/tilework/runtime/offload.rb +498 -0
  42. data/lib/tilework/runtime/submission.rb +163 -0
  43. data/lib/tilework/runtime/telemetry.rb +112 -0
  44. data/lib/tilework/scope.rb +33 -0
  45. data/lib/tilework/version.rb +5 -0
  46. data/lib/tilework.rb +345 -0
  47. metadata +92 -0
@@ -0,0 +1,178 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Tilework
4
+ module Graph
5
+ # A pool of intermediate buffers. Buffers coming back from liveness are kept
6
+ # per `[dtype, shape]` and handed out again.
7
+ #
8
+ # Eager does a `Buffer.new` per op, so a five-step elementwise chain
9
+ # allocates and drops four intermediates. The graph knows the execution
10
+ # order and each value's last use, so **rotating slots of the same shape**
11
+ # is enough. The pool belongs to the plan, so reruns do not reallocate
12
+ # either (which pays off in decode loops rerunning one plan).
13
+ #
14
+ # Only exact shape matches are reused — the same length with a different
15
+ # shape is a different slot, because reshape makes views and returning one
16
+ # to the pool would lose track of the owning buffer.
17
+ #
18
+ # It is safe with asynchronous launches because **reuse is not freeing**.
19
+ # The next kernel to use a slot is queued after the last kernel that read
20
+ # it, so same-queue ordering holds. Calling `Buffer#free` here would release
21
+ # memory a running kernel uses (undefined behavior on CUDA).
22
+ class Pool
23
+ attr_reader :allocated, :reused
24
+
25
+ def initialize
26
+ @slots = Hash.new { |hash, key| hash[key] = [] }
27
+ @allocated = 0
28
+ @reused = 0
29
+ end
30
+
31
+ def allocate(dtype, length, shape)
32
+ slot = @slots[[dtype, shape]]
33
+ while (buffer = slot.pop)
34
+ # It may have been freed from outside the plan, e.g. by a Scope (a plan
35
+ # run inside Tilework.run). Drop it rather than hold on.
36
+ next if buffer.freed?
37
+
38
+ @reused += 1
39
+ buffer.instance_variable_set(:@graph_pool_released, false)
40
+ return buffer
41
+ end
42
+
43
+ @allocated += 1
44
+ Buffer.new(dtype, length, shape: shape).tap do |buffer|
45
+ # Marks the buffer so Plan#release returns only what this pool lent.
46
+ # A separate set of object_ids would either keep plan outputs that left
47
+ # alive or misread reused ids, so the owner is recorded on the buffer.
48
+ buffer.instance_variable_set(:@graph_pool_owner, self)
49
+ end
50
+ end
51
+
52
+ def owns?(buffer) = buffer.instance_variable_get(:@graph_pool_owner).equal?(self)
53
+
54
+ # Adopts a buffer allocated outside the pool (a temporary an op made with
55
+ # `Buffer.new`, like `nn.rb`'s scores), so what arrives through deferred
56
+ # frees is reused on the next run.
57
+ def adopt(buffer)
58
+ buffer.instance_variable_set(:@graph_pool_owner, self)
59
+ buffer
60
+ end
61
+
62
+ # Returning is **idempotent**. `nn.rb`'s error path frees twice with
63
+ # `free unless freed?`, and `freed?` stays false while a free is deferred,
64
+ # so a naive push would lend the same buffer to two nodes as two slots.
65
+ def release(buffer)
66
+ return buffer if buffer.instance_variable_get(:@graph_pool_released)
67
+
68
+ buffer.instance_variable_set(:@graph_pool_released, true)
69
+ @slots[[buffer.dtype, buffer.shape]].push(buffer)
70
+ buffer
71
+ end
72
+
73
+ def pooled = @slots.each_value.sum(&:length)
74
+
75
+ def stats = { allocated: @allocated, reused: @reused, pooled: pooled }
76
+
77
+ # Releases the device memory the pool holds. Running kernels may still read
78
+ # the slots, so wait first.
79
+ def free_all!
80
+ Tilework.synchronize
81
+ @slots.each_value { |buffers| buffers.each(&:free) }
82
+ @slots.clear
83
+ nil
84
+ end
85
+ end
86
+
87
+ # The context plugged into Ops while running (see the Ops::CONTEXT_KEY
88
+ # comment). Routes output allocation to the pool and makes launches async.
89
+ class Context
90
+ def initialize(pool: nil, async: true)
91
+ @pool = pool
92
+ @async = async
93
+ @outer = nil
94
+ @deferred = []
95
+ end
96
+
97
+ attr_reader :pool
98
+
99
+ # When there is an enclosing `Tilework.submit` boundary, ownership can be
100
+ # handed to it (#86 stage 2). The plan keeps its own pool and passes only
101
+ # buffers from outside the pool to the Job.
102
+ attr_accessor :outer
103
+
104
+ # Lets ops that free buffers mid-call (`nn.rb`'s scores, `Ops.reduce`'s
105
+ # partials) run **asynchronously**: memory a running kernel reads is never
106
+ # released, so there is no reason left to run them synchronously (#86).
107
+ #
108
+ # With a pool the buffer is returned to it — Pool's safety argument ("the
109
+ # next writer of a slot is queued after its last reader") applies as is.
110
+ # Views own nothing, so they are not taken and are freed on the spot (the
111
+ # same judgment as Plan#release).
112
+ def defer_free(buffer)
113
+ return false if buffer.view?
114
+
115
+ if @pool
116
+ @pool.adopt(buffer) unless @pool.owns?(buffer)
117
+ return @pool.release(buffer)
118
+ end
119
+ return @outer.defer_free(buffer) if @outer
120
+
121
+ @deferred << buffer
122
+ end
123
+
124
+ # What piled up with neither a pool nor an outer boundary
125
+ # (`Graph.buffer_pool = false`). Running kernels may read it, so wait first.
126
+ def drain_deferred!
127
+ return if @deferred.empty?
128
+
129
+ pending = @deferred
130
+ @deferred = []
131
+ Tilework.synchronize
132
+ pending.each(&:free)
133
+ nil
134
+ end
135
+
136
+ def async? = @async
137
+
138
+ # Makes the next allocated output **this buffer** instead of a new one, so
139
+ # the node producing a `write_into` value gets the state's view as its
140
+ # output from the start (#19 §3) — removing one copy.
141
+ #
142
+ # It applies once, and only if the dtype and length match. Otherwise a
143
+ # normal allocation happens and `consumed?` stays false, so the caller
144
+ # knows it could not place and falls back to the copy.
145
+ def place(buffer)
146
+ @placement = buffer
147
+ @placement_consumed = false
148
+ yield
149
+ ensure
150
+ @placement = nil
151
+ end
152
+
153
+ def placement_consumed? = @placement_consumed
154
+
155
+ def allocate(dtype, length, shape)
156
+ if @placement && @placement.dtype == dtype && @placement.length == length
157
+ placed = @placement
158
+ @placement = nil
159
+ @placement_consumed = true
160
+ return placed
161
+ end
162
+ return Buffer.new(dtype, length, shape: shape) unless @pool
163
+
164
+ @pool.allocate(dtype, length, shape)
165
+ end
166
+
167
+ # Nodes that may run outside our queue, like delegated libraries and user
168
+ # blocks, run synchronously.
169
+ def synchronous
170
+ previous = @async
171
+ @async = false
172
+ yield
173
+ ensure
174
+ @async = previous
175
+ end
176
+ end
177
+ end
178
+ end
@@ -0,0 +1,210 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Tilework
4
+ module Graph
5
+ # The target region of a write to / read from mutable state.
6
+ #
7
+ # Represented as **a strided block over the flat element space**
8
+ # `(offset, block, stride, count)`: block elements from offset, repeated
9
+ # count times every stride. With count == 1 it is just the contiguous
10
+ # range `[offset, offset + block)`.
11
+ #
12
+ # channels c0...c1 of [n, c, h, w] → offset = c0*h*w, block = (c1-c0)*h*w,
13
+ # stride = c*h*w, count = n
14
+ #
15
+ # The reason for this form is overlap testing. Deciding that "writes to
16
+ # disjoint regions need not be serialized" needs an exact test, which is
17
+ # undecidable for arbitrary index expressions. Strided blocks express every
18
+ # shape actually needed (outermost-dimension slices and channel ranges) and
19
+ # the test is O(1).
20
+ #
21
+ # **Shapes it cannot decide answer "may overlap"** (#disjoint?). Erring on
22
+ # the safe side loses only parallelism, never correctness.
23
+ #
24
+ # offset can be a Param as well as an Integer. An offset **decided at run
25
+ # time**, like a KV cache's write position, is resolved from the plan's
26
+ # bindings (`plan.run(pos: 3)`). static? is then false and the overlap test
27
+ # always takes the safe side.
28
+ class Region
29
+ # An offset decided at run time; its value is `bias + bindings[name] * scale`.
30
+ Param = Struct.new(:name, :scale, :bias) do
31
+ def resolve(bindings)
32
+ value = bindings.fetch(name) do
33
+ raise Error, "plan.run is missing the runtime index #{name.inspect} " \
34
+ "(used as a write/read position)"
35
+ end
36
+ unless value.is_a?(Integer) && !value.negative?
37
+ raise ArgumentError, "runtime index #{name.inspect} must be a non-negative Integer, got #{value.inspect}"
38
+ end
39
+
40
+ bias + (value * scale)
41
+ end
42
+
43
+ def key = [:param, name, scale, bias]
44
+ end
45
+
46
+ attr_reader :offset, :block, :stride, :count, :total, :shape
47
+
48
+ # total is the target buffer's element count (for range checks). shape is
49
+ # the logical shape of the tensor read from this region.
50
+ def initialize(offset:, block:, count: 1, stride: nil, total:, shape: nil)
51
+ @offset = offset
52
+ @block = block
53
+ @count = count
54
+ @stride = stride || block
55
+ @total = total
56
+ @shape = shape || [block * count]
57
+ validate!
58
+ end
59
+
60
+ # The whole buffer.
61
+ def self.whole(shape)
62
+ total = shape.inject(1, :*)
63
+ new(offset: 0, block: total, total: total, shape: shape)
64
+ end
65
+
66
+ # A slice along the outermost dimension (Buffer#slice semantics); always
67
+ # contiguous. range takes an Integer / Range or a run-time position (Index).
68
+ def self.rows(shape, range)
69
+ raise Error, "rows requires a shaped region (shape: #{shape.inspect})" if shape.length < 2
70
+
71
+ row = shape.drop(1).inject(1, :*)
72
+ total = shape.inject(1, :*)
73
+ first, count = normalize(range, shape[0])
74
+ offset = first.is_a?(Param) ? Param.new(first.name, first.scale * row, first.bias * row) : first * row
75
+ new(offset: offset, block: count * row, total: total, shape: [count, *shape.drop(1)])
76
+ end
77
+
78
+ # A slice along any axis: the same contiguous region as rows for
79
+ # axis == 0, a strided block for axis > 0.
80
+ def self.axis(shape, axis, range)
81
+ unless axis.is_a?(Integer) && (0...shape.length).cover?(axis)
82
+ raise ArgumentError, "axis #{axis.inspect} is out of range for shape #{shape.inspect}"
83
+ end
84
+ return rows(shape, range) if axis.zero?
85
+
86
+ inner = shape.drop(axis + 1).inject(1, :*)
87
+ outer = shape.take(axis).inject(1, :*)
88
+ first, count = normalize(range, shape[axis])
89
+ if first.is_a?(Param)
90
+ raise Error, "a runtime index is only supported on the outermost axis (got axis #{axis})"
91
+ end
92
+
93
+ out_shape = shape.dup
94
+ out_shape[axis] = count
95
+ new(offset: first * inner, block: count * inner, count: outer,
96
+ stride: shape[axis] * inner, total: shape.inject(1, :*), shape: out_shape)
97
+ end
98
+
99
+ # Equivalent to Buffer#normalize_range (private there, so the rule is repeated here).
100
+ def self.normalize(range, limit)
101
+ case range
102
+ when Index
103
+ [Param.new(range.name, 1, 0), range.count]
104
+ when Param
105
+ [range, 1]
106
+ when Integer
107
+ idx = range.negative? ? limit + range : range
108
+ raise ArgumentError, "index #{range} is out of bounds (#{limit})" unless (0...limit).cover?(idx)
109
+
110
+ [idx, 1]
111
+ when Range
112
+ first = range.begin || 0
113
+ first += limit if first.negative?
114
+ last = range.end || limit
115
+ last += limit if last.negative?
116
+ last -= 1 if range.exclude_end? && range.end
117
+ count = last - first + 1
118
+ raise ArgumentError, "range #{range} is out of bounds (#{limit})" unless first >= 0 && count.positive? && first + count <= limit
119
+
120
+ [first, count]
121
+ else
122
+ raise ArgumentError, "expected an Integer, a Range, or a runtime index, got #{range.class}"
123
+ end
124
+ end
125
+ private_class_method :normalize
126
+
127
+ def length = block * count
128
+ def contiguous? = count == 1
129
+ def static? = offset.is_a?(Integer)
130
+
131
+ # Applies run-time bindings and returns a Region whose offset is a fixed Integer.
132
+ def resolve(bindings)
133
+ return self if static?
134
+
135
+ concrete = Region.new(offset: offset.resolve(bindings), block: block, count: count,
136
+ stride: stride, total: total, shape: shape)
137
+ concrete.bounds_check!
138
+ concrete
139
+ end
140
+
141
+ def bounds_check!
142
+ last = offset + ((count - 1) * stride) + block
143
+ return if last <= total
144
+
145
+ raise Error, "region #{inspect} runs past the end of a #{total}-element buffer"
146
+ end
147
+
148
+ # Yields [offset in the target buffer, offset in the value, length] per
149
+ # block. The value (written from / read into) is treated as the selected
150
+ # region packed contiguously.
151
+ def each_block
152
+ return enum_for(:each_block) unless block_given?
153
+
154
+ count.times { |i| yield(offset + (i * stride), i * block, self.block) }
155
+ end
156
+
157
+ # **true only when the regions certainly do not overlap**; shapes it
158
+ # cannot decide answer false.
159
+ #
160
+ # - either has a run-time offset → false (conservative)
161
+ # - both contiguous → exact interval test
162
+ # - equal strides → exact test on the phase within the stride
163
+ # - different strides → true only when the overall spans are apart
164
+ def disjoint?(other)
165
+ return false unless static? && other.static?
166
+ return spans_disjoint?(other) if contiguous? && other.contiguous?
167
+ return spans_disjoint?(other) unless stride == other.stride
168
+ return true if spans_disjoint?(other)
169
+
170
+ # On the same lattice, disjoint if the phases within the stride do not overlap.
171
+ phase = (other.offset - offset) % stride
172
+ !(phase < self.block || stride - phase < other.block)
173
+ end
174
+
175
+ def spans_disjoint?(other)
176
+ span_end <= other.offset || other.span_end <= offset
177
+ end
178
+
179
+ def span_end = offset + ((count - 1) * stride) + block
180
+
181
+ # For the CSE key. A run-time offset compares by Param identity, not value.
182
+ def key
183
+ [static? ? offset : offset.key, block, stride, count, total, shape]
184
+ end
185
+
186
+ def inspect
187
+ head = static? ? offset.to_s : "#{offset.name}*#{offset.scale}+#{offset.bias}"
188
+ contiguous? ? "#<Region #{head}...+#{block}>" : "#<Region #{head}...+#{block} x#{count} stride #{stride}>"
189
+ end
190
+
191
+ private
192
+
193
+ def validate!
194
+ raise ArgumentError, "block must be a positive Integer" unless @block.is_a?(Integer) && @block.positive?
195
+ raise ArgumentError, "count must be a positive Integer" unless @count.is_a?(Integer) && @count.positive?
196
+ raise ArgumentError, "stride must be at least block" if @stride < @block
197
+ unless @shape.inject(1, :*) == length
198
+ raise ArgumentError, "region shape #{@shape.inspect} does not match #{length} elements"
199
+ end
200
+
201
+ bounds_check! if static?
202
+ end
203
+ end
204
+
205
+ # A position decided at run time, resolved from `plan.run(pos: 3)` bindings.
206
+ #
207
+ # g.write_into(kv0, at: g.index(:pos), value: k_new)
208
+ Index = Struct.new(:name, :count)
209
+ end
210
+ end