tilework 0.18.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE +21 -0
- data/lib/tilework/backend/cpu/dsl_codegen.rb +509 -0
- data/lib/tilework/backend/cpu/ops_codegen.rb +451 -0
- data/lib/tilework/backend/cpu.rb +156 -0
- data/lib/tilework/buffer.rb +753 -0
- data/lib/tilework/capabilities.rb +183 -0
- data/lib/tilework/compat/cuda.h +66 -0
- data/lib/tilework/compat/hip.h +35 -0
- data/lib/tilework/compat/metal.h +45 -0
- data/lib/tilework/conformance.rb +2296 -0
- data/lib/tilework/core_ext.rb +21 -0
- data/lib/tilework/dialect.rb +67 -0
- data/lib/tilework/dsl.rb +722 -0
- data/lib/tilework/dtype/bfloat.rb +31 -0
- data/lib/tilework/dtype/cast.rb +88 -0
- data/lib/tilework/dtype/half.rb +98 -0
- data/lib/tilework/graph/builder.rb +1073 -0
- data/lib/tilework/graph/fusion.rb +520 -0
- data/lib/tilework/graph/grad.rb +443 -0
- data/lib/tilework/graph/high_level.rb +266 -0
- data/lib/tilework/graph/kernels.rb +403 -0
- data/lib/tilework/graph/node.rb +27 -0
- data/lib/tilework/graph/op.rb +290 -0
- data/lib/tilework/graph/optim.rb +183 -0
- data/lib/tilework/graph/plan.rb +1086 -0
- data/lib/tilework/graph/pool.rb +178 -0
- data/lib/tilework/graph/region.rb +210 -0
- data/lib/tilework/graph/residency.rb +449 -0
- data/lib/tilework/graph/state.rb +64 -0
- data/lib/tilework/graph/tensor.rb +124 -0
- data/lib/tilework/graph.rb +213 -0
- data/lib/tilework/kernel.rb +163 -0
- data/lib/tilework/math/conv.rb +270 -0
- data/lib/tilework/math/fft.rb +172 -0
- data/lib/tilework/math/nn.rb +1103 -0
- data/lib/tilework/math/philox.rb +194 -0
- data/lib/tilework/math/random.rb +232 -0
- data/lib/tilework/math/tensor_core.rb +269 -0
- data/lib/tilework/ops.rb +682 -0
- data/lib/tilework/runtime/offload.rb +498 -0
- data/lib/tilework/runtime/submission.rb +163 -0
- data/lib/tilework/runtime/telemetry.rb +112 -0
- data/lib/tilework/scope.rb +33 -0
- data/lib/tilework/version.rb +5 -0
- data/lib/tilework.rb +345 -0
- metadata +92 -0
|
@@ -0,0 +1,213 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "graph/region"
|
|
4
|
+
require_relative "graph/op"
|
|
5
|
+
require_relative "graph/node"
|
|
6
|
+
require_relative "graph/state"
|
|
7
|
+
require_relative "graph/tensor"
|
|
8
|
+
require_relative "graph/pool"
|
|
9
|
+
require_relative "graph/residency"
|
|
10
|
+
require_relative "graph/kernels"
|
|
11
|
+
require_relative "graph/high_level"
|
|
12
|
+
require_relative "graph/grad"
|
|
13
|
+
require_relative "graph/fusion"
|
|
14
|
+
require_relative "graph/builder"
|
|
15
|
+
require_relative "graph/plan"
|
|
16
|
+
require_relative "graph/optim"
|
|
17
|
+
|
|
18
|
+
module Tilework
|
|
19
|
+
# Opt-in deferred graph execution for tensors and plans.
|
|
20
|
+
#
|
|
21
|
+
# plan = Tilework::Graph.build do |g|
|
|
22
|
+
# x = g.input(:x, [4, 8])
|
|
23
|
+
# w = g.param(weights)
|
|
24
|
+
# g.output(x.matmul(w) * 2.0)
|
|
25
|
+
# end.compile
|
|
26
|
+
#
|
|
27
|
+
# y = plan.run(x: input_buffer)
|
|
28
|
+
#
|
|
29
|
+
# A layer added **on top of** the eager layer (Buffer / Ops); nothing below
|
|
30
|
+
# changes. The graph is opt-in, so being able to write eager code is never
|
|
31
|
+
# traded away.
|
|
32
|
+
#
|
|
33
|
+
# ## Mutable state
|
|
34
|
+
#
|
|
35
|
+
# A graph is a DAG of values, so things that live and change across plan
|
|
36
|
+
# runs, like a KV cache, do not fit naturally. Here a write is **an operation
|
|
37
|
+
# that returns a version**, turning the invisible memory dependency into a
|
|
38
|
+
# value edge:
|
|
39
|
+
#
|
|
40
|
+
# kv0 = g.state(k_cache)
|
|
41
|
+
# kv1 = g.write_into(kv0, at: g.index(:pos), value: k_new)
|
|
42
|
+
# k_all = g.read(kv1, 0..pos) # depends on version 1 = a real edge
|
|
43
|
+
#
|
|
44
|
+
# Unlike SSA, a version points at the same memory rather than a new value,
|
|
45
|
+
# so version edges express only RAW. Plan#compile adds WAR (writing while a
|
|
46
|
+
# reader is still reading) as ordering edges.
|
|
47
|
+
module Graph
|
|
48
|
+
# Read semantics.
|
|
49
|
+
# :view — zero copy (default); no kernel
|
|
50
|
+
# :copy — materialize. The graph becomes a pure DAG, so comparing it with
|
|
51
|
+
# :view's result verifies the state machinery itself (slow)
|
|
52
|
+
STRICT_MODES = %i[view copy].freeze
|
|
53
|
+
|
|
54
|
+
class << self
|
|
55
|
+
# Records a graph without allocating device memory.
|
|
56
|
+
#
|
|
57
|
+
# @param differentiable [Boolean] record decompositions required for reverse-mode gradients
|
|
58
|
+
# @yieldparam builder [Graph::Builder]
|
|
59
|
+
# @return [Graph::Builder] the recorded graph; call `#compile` to create a plan
|
|
60
|
+
#
|
|
61
|
+
# With differentiable: true, high-level ops (NN.softmax etc.) are recorded
|
|
62
|
+
# as their decompositions. Gradients follow from primitive vjps alone, so
|
|
63
|
+
# graphs that use g.grad are recorded this way (docs/autograd.md).
|
|
64
|
+
def build(differentiable: false)
|
|
65
|
+
builder = Builder.new(differentiable: differentiable)
|
|
66
|
+
yield builder
|
|
67
|
+
builder
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
# Whether any value is being recorded into a graph. Each `NN` function
|
|
71
|
+
# branches on this in one line so it can be called eager or in a graph.
|
|
72
|
+
def symbolic?(*values) = values.any? { |value| value.is_a?(Tensor) }
|
|
73
|
+
|
|
74
|
+
# Records a **built-in the graph does not decompose**, like `NN`, as a node.
|
|
75
|
+
#
|
|
76
|
+
# return Graph.record_nn(:softmax, [x], shape: x.shape, dtype: x.dtype) if Graph.symbolic?(x)
|
|
77
|
+
#
|
|
78
|
+
# Unlike `g.opaque`, the graph knows what this is by name: CSE applies, it
|
|
79
|
+
# runs as the same-named `NN` function, and fusion can treat it as a
|
|
80
|
+
# partner (opaque stays opaque to the end).
|
|
81
|
+
#
|
|
82
|
+
# tensors: are keyword arguments taking Tensors (gamma / beta / bias ...);
|
|
83
|
+
# nils are dropped. args: are non-Tensor positional arguments and opts:
|
|
84
|
+
# non-Tensor keyword arguments, both carried as node attributes.
|
|
85
|
+
def record_nn(fn, inputs, shape:, dtype:, tensors: {}, args: [], opts: {})
|
|
86
|
+
builder = builder_for(inputs, tensors, fn)
|
|
87
|
+
builder.nn(fn, inputs, shape: shape, dtype: dtype, tensors: tensors, args: args, opts: opts)
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
# Records a convolution (called in one line from Conv.forward). Unlike :nn
|
|
91
|
+
# the callee is Tilework::Conv, hence its own op.
|
|
92
|
+
def record_conv2d(input, weight, shape:, pad:, stride:, dilation: [1, 1], groups: 1)
|
|
93
|
+
builder = builder_for([input, weight], {}, :conv2d)
|
|
94
|
+
builder.conv2d(input, weight, shape: shape, pad: pad, stride: stride, dilation: dilation, groups: groups)
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
# Queue launches without waiting (default). false waits per node like
|
|
98
|
+
# eager — for measuring the effect and for bisecting ordering suspicions.
|
|
99
|
+
def async = @async.nil? ? true : @async
|
|
100
|
+
attr_writer :async
|
|
101
|
+
|
|
102
|
+
# Reuse intermediates from the pool (default). false allocates per op and
|
|
103
|
+
# frees at the end. Like async, for measuring and bisecting.
|
|
104
|
+
def buffer_pool = @buffer_pool.nil? ? true : @buffer_pool
|
|
105
|
+
attr_writer :buffer_pool
|
|
106
|
+
|
|
107
|
+
# Fold chained elementwise ops into one kernel (#17 Phase 2). Can be
|
|
108
|
+
# turned off like async / buffer_pool for measuring and bisecting;
|
|
109
|
+
# conformance pins that fusion does not change values.
|
|
110
|
+
def fusion = @fusion.nil? ? true : @fusion
|
|
111
|
+
attr_writer :fusion
|
|
112
|
+
|
|
113
|
+
# Fold expressions containing row-wise reductions into one kernel
|
|
114
|
+
# (docs/graph-ir.md Phase C). One thread owns one row, so with few rows
|
|
115
|
+
# per-element parallelism can be faster — switchable until measured on hardware.
|
|
116
|
+
def row_fusion = @row_fusion.nil? ? true : @row_fusion
|
|
117
|
+
attr_writer :row_fusion
|
|
118
|
+
|
|
119
|
+
# Hand the node producing a write_into value the state's view as its
|
|
120
|
+
# output (#19 §3). Off, the value is produced and then copied — like the
|
|
121
|
+
# other switches, for measuring and bisecting.
|
|
122
|
+
def placement = @placement.nil? ? true : @placement
|
|
123
|
+
attr_writer :placement
|
|
124
|
+
|
|
125
|
+
# Record high-level ops (:nn nodes) as their primitive decompositions.
|
|
126
|
+
# Default false — execution stays on the hand-written fused kernels and
|
|
127
|
+
# the decomposition serves as the definition of meaning (Tier 1 in
|
|
128
|
+
# docs/graph-ir.md). true lets the two be compared, which is what the
|
|
129
|
+
# conformance oracle uses.
|
|
130
|
+
#
|
|
131
|
+
# **It takes effect at record time**, so switch it before Graph.build.
|
|
132
|
+
def lower_high_level = @lower_high_level.nil? ? false : @lower_high_level
|
|
133
|
+
attr_writer :lower_high_level
|
|
134
|
+
|
|
135
|
+
# Page weights in and out of VRAM to fit a budget (#17 Phase 4). On by
|
|
136
|
+
# default, but **nothing happens without compile(vram:)** — this layer
|
|
137
|
+
# never starts paging without being given a budget. Turning it off is,
|
|
138
|
+
# like the other switches, for measuring and bisecting.
|
|
139
|
+
def offload = @offload.nil? ? true : @offload
|
|
140
|
+
attr_writer :offload
|
|
141
|
+
|
|
142
|
+
# Ride eviction fences on nodes that already synchronize (#17 Phase 4c).
|
|
143
|
+
# Off, every eviction adds a Tilework.synchronize — a switch for
|
|
144
|
+
# measuring how much riding along actually saves.
|
|
145
|
+
def offload_fence_reuse = @offload_fence_reuse.nil? ? true : @offload_fence_reuse
|
|
146
|
+
attr_writer :offload_fence_reuse
|
|
147
|
+
|
|
148
|
+
# Fetch asynchronously ahead of the demand point (#17 Phase 4b), closing
|
|
149
|
+
# the gap measured on Jetson between one synchronous transfer at a time
|
|
150
|
+
# (0.56 GB/s) and async depth 2–4 (1.85–2.24 GB/s). Without an async
|
|
151
|
+
# transfer ABI (Offload.async? false) it never starts, so this switch
|
|
152
|
+
# matters only for measuring and bisecting on the same machine.
|
|
153
|
+
def offload_prefetch = @offload_prefetch.nil? ? true : @offload_prefetch
|
|
154
|
+
attr_writer :offload_prefetch
|
|
155
|
+
|
|
156
|
+
def strict_state = @strict_state || :view
|
|
157
|
+
|
|
158
|
+
# Selects zero-copy or copied state reads for subsequently compiled plans.
|
|
159
|
+
#
|
|
160
|
+
# @param mode [:view, :copy]
|
|
161
|
+
# @return [:view, :copy]
|
|
162
|
+
# @raise [ArgumentError] for an unsupported mode
|
|
163
|
+
def strict_state=(mode)
|
|
164
|
+
mode = mode.to_sym
|
|
165
|
+
unless STRICT_MODES.include?(mode)
|
|
166
|
+
raise ArgumentError, "unknown strict_state #{mode.inspect} (expected #{STRICT_MODES.join(' or ')})"
|
|
167
|
+
end
|
|
168
|
+
|
|
169
|
+
@strict_state = mode
|
|
170
|
+
end
|
|
171
|
+
|
|
172
|
+
# An on-device copy, used for state writes that cannot be placed and for
|
|
173
|
+
# materializing reads. When the producer can write straight into the
|
|
174
|
+
# state's view (Graph.placement), no copy runs.
|
|
175
|
+
def copy(src, dst)
|
|
176
|
+
unless src.dtype == dst.dtype && src.length == dst.length
|
|
177
|
+
raise Error, "cannot copy #{src.length} #{src.dtype} elements into " \
|
|
178
|
+
"#{dst.length} #{dst.dtype} elements"
|
|
179
|
+
end
|
|
180
|
+
|
|
181
|
+
n = src.length
|
|
182
|
+
args = [flat(src, n), flat(dst, n), n]
|
|
183
|
+
grid = (n / Ops::BLOCK.to_f).ceil
|
|
184
|
+
# The bounds guard is in the kernel body, so launch with explicit
|
|
185
|
+
# grid/block rather than the auto-guarded .() (only this form can be async).
|
|
186
|
+
if Ops.context&.async?
|
|
187
|
+
copy_kernel.launch_async(grid: grid, block: Ops::BLOCK, args: args)
|
|
188
|
+
else
|
|
189
|
+
copy_kernel.launch(grid: grid, block: Ops::BLOCK, args: args)
|
|
190
|
+
end
|
|
191
|
+
dst
|
|
192
|
+
end
|
|
193
|
+
|
|
194
|
+
private
|
|
195
|
+
|
|
196
|
+
def builder_for(inputs, tensors, fn)
|
|
197
|
+
found = (inputs + tensors.values).find { |value| value.is_a?(Tensor) }
|
|
198
|
+
raise Error, "#{fn} got no Tilework::Graph::Tensor to record against" if found.nil?
|
|
199
|
+
|
|
200
|
+
found.graph
|
|
201
|
+
end
|
|
202
|
+
|
|
203
|
+
def flat(buffer, n) = buffer.shape.length > 1 ? buffer.reshape(n) : buffer
|
|
204
|
+
|
|
205
|
+
def copy_kernel
|
|
206
|
+
@copy_kernel ||= Tilework.kernel("graph_copy") do |a, b, n|
|
|
207
|
+
i = global_id_x
|
|
208
|
+
b[i] = a[i] if i < n
|
|
209
|
+
end
|
|
210
|
+
end
|
|
211
|
+
end
|
|
212
|
+
end
|
|
213
|
+
end
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "digest"
|
|
4
|
+
|
|
5
|
+
module Tilework
|
|
6
|
+
# A compiled kernel pipeline.
|
|
7
|
+
#
|
|
8
|
+
# `grid` is the number of thread groups and `block` is the thread-group size,
|
|
9
|
+
# matching CUDA's `<<<grid, block>>>` execution model.
|
|
10
|
+
class Kernel
|
|
11
|
+
@cache = {}
|
|
12
|
+
@cache_mutex = Mutex.new
|
|
13
|
+
|
|
14
|
+
class << self
|
|
15
|
+
# Compiles source and reuses an existing pipeline for identical source and name.
|
|
16
|
+
#
|
|
17
|
+
# @api private
|
|
18
|
+
# @param src [String]
|
|
19
|
+
# @param name [String, Symbol]
|
|
20
|
+
# @param raw [Boolean]
|
|
21
|
+
# @return [Kernel]
|
|
22
|
+
def compile(src, name:, raw: false)
|
|
23
|
+
processed = raw ? src : Dialect.expand(src)
|
|
24
|
+
bridge = Tilework.bridge
|
|
25
|
+
key = [bridge.object_id, cache_key(processed, name)]
|
|
26
|
+
pipeline = @cache_mutex.synchronize do
|
|
27
|
+
cache_hit = @cache.key?(key)
|
|
28
|
+
Telemetry.instrument(:compile, label: name, cache_hit: cache_hit) do
|
|
29
|
+
@cache[key] ||= create_pipeline(bridge, processed, name)
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
new(pipeline, name, bridge, processed)
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def cache_key(processed_src, name)
|
|
36
|
+
Digest::SHA256.hexdigest("#{processed_src}\0#{name}")
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
# @return [Integer] number of cached compiled pipelines
|
|
40
|
+
def cache_size
|
|
41
|
+
@cache_mutex.synchronize { @cache.size }
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
# Clears the in-process pipeline cache.
|
|
45
|
+
#
|
|
46
|
+
# @return [Hash] the cleared cache
|
|
47
|
+
def clear_cache!
|
|
48
|
+
@cache_mutex.synchronize { @cache.clear }
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
private
|
|
52
|
+
|
|
53
|
+
def create_pipeline(bridge, src, name)
|
|
54
|
+
bridge.pipeline_create(src, name)
|
|
55
|
+
rescue RuntimeError => e
|
|
56
|
+
raise CompileError, "failed to compile kernel '#{name}':\n#{e.message}"
|
|
57
|
+
end
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
attr_reader :name
|
|
61
|
+
|
|
62
|
+
def initialize(pipeline, name, bridge, processed_source)
|
|
63
|
+
@pipeline = pipeline
|
|
64
|
+
@name = name
|
|
65
|
+
@bridge = bridge
|
|
66
|
+
@processed_source = processed_source
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
# Launches the kernel and waits for its completion.
|
|
70
|
+
#
|
|
71
|
+
# @param grid [Integer, Array<Integer>] one to three positive grid dimensions
|
|
72
|
+
# @param block [Integer, Array<Integer>] one to three positive thread-group dimensions
|
|
73
|
+
# @param args [Array<Buffer, Numeric>] kernel arguments
|
|
74
|
+
# @return [nil]
|
|
75
|
+
# @raise [LaunchError] if arguments or launch dimensions are invalid
|
|
76
|
+
def launch(grid:, block:, args: [])
|
|
77
|
+
dispatch(grid, block, args, wait: true)
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
# Launches the kernel without waiting for completion.
|
|
81
|
+
#
|
|
82
|
+
# @param grid [Integer, Array<Integer>]
|
|
83
|
+
# @param block [Integer, Array<Integer>]
|
|
84
|
+
# @param args [Array<Buffer, Numeric>]
|
|
85
|
+
# @return [nil]
|
|
86
|
+
# @note Read a buffer or call {Tilework.synchronize} before consuming results on the host.
|
|
87
|
+
def launch_async(grid:, block:, args: [])
|
|
88
|
+
dispatch(grid, block, args, wait: false)
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
# @return [Integer] backend limit for threads in one thread group
|
|
92
|
+
def max_threads_per_block
|
|
93
|
+
@bridge.pipeline_max_threads(@pipeline)
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
private
|
|
97
|
+
|
|
98
|
+
def dispatch(grid, block, args, wait:)
|
|
99
|
+
recompile_for_current_bridge! unless Tilework.bridge.equal?(@bridge)
|
|
100
|
+
|
|
101
|
+
grid = normalize_dim(grid, "grid")
|
|
102
|
+
block = normalize_dim(block, "block")
|
|
103
|
+
validate_block!(block)
|
|
104
|
+
native_args = convert_args(args)
|
|
105
|
+
bytes = args.sum { |a| a.is_a?(Buffer) ? a.bytesize : 0 }
|
|
106
|
+
threads = grid.zip(block).reduce(1) { |acc, (g, b)| acc * g * b }
|
|
107
|
+
|
|
108
|
+
begin
|
|
109
|
+
Telemetry.instrument(:dispatch, label: @name, bytes: bytes, threads: threads) do
|
|
110
|
+
@bridge.dispatch(@pipeline, native_args, grid, block, wait)
|
|
111
|
+
end
|
|
112
|
+
rescue RuntimeError => e
|
|
113
|
+
raise LaunchError, "kernel '#{@name}' failed: #{e.message}"
|
|
114
|
+
end
|
|
115
|
+
nil
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
# DSL and built-in kernels are memoized above the pipeline cache. A backend
|
|
119
|
+
# change therefore cannot merely reject the old object: refresh its native
|
|
120
|
+
# pipeline lazily so those long-lived Ruby kernel objects stay portable.
|
|
121
|
+
def recompile_for_current_bridge!
|
|
122
|
+
refreshed = self.class.compile(@processed_source, name: @name, raw: true)
|
|
123
|
+
@pipeline = refreshed.instance_variable_get(:@pipeline)
|
|
124
|
+
@bridge = refreshed.instance_variable_get(:@bridge)
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
def normalize_dim(dim, label)
|
|
128
|
+
dims = dim.is_a?(Integer) ? [dim, 1, 1] : Array(dim)
|
|
129
|
+
dims += [1] * (3 - dims.length) if dims.length < 3
|
|
130
|
+
unless dims.length == 3 && dims.all? { |d| d.is_a?(Integer) && d.positive? }
|
|
131
|
+
raise LaunchError, "#{label} must be a positive Integer or an Array of up to 3 positive Integers, got #{dim.inspect}"
|
|
132
|
+
end
|
|
133
|
+
dims
|
|
134
|
+
end
|
|
135
|
+
|
|
136
|
+
def validate_block!(block)
|
|
137
|
+
total = block.reduce(:*)
|
|
138
|
+
max = max_threads_per_block
|
|
139
|
+
return if total <= max
|
|
140
|
+
|
|
141
|
+
raise LaunchError,
|
|
142
|
+
"block size #{block.inspect} (#{total} threads) exceeds the pipeline limit of #{max} threads per threadgroup"
|
|
143
|
+
end
|
|
144
|
+
|
|
145
|
+
# A Buffer becomes setBuffer; a Ruby number becomes a setBytes byte string
|
|
146
|
+
# (fixed Float -> float32 / Integer -> int32 for now).
|
|
147
|
+
def convert_args(args)
|
|
148
|
+
if args.length > MAX_KERNEL_ARGS
|
|
149
|
+
raise LaunchError, "too many kernel arguments: #{args.length} (Metal limit is #{MAX_KERNEL_ARGS})"
|
|
150
|
+
end
|
|
151
|
+
|
|
152
|
+
args.map.with_index do |arg, i|
|
|
153
|
+
case arg
|
|
154
|
+
when Buffer then arg.__handle__
|
|
155
|
+
when Float then [arg].pack("f")
|
|
156
|
+
when Integer then [arg].pack("l")
|
|
157
|
+
else
|
|
158
|
+
raise LaunchError, "unsupported kernel argument at index #{i}: #{arg.class} (expected Tilework::Buffer, Float, or Integer)"
|
|
159
|
+
end
|
|
160
|
+
end
|
|
161
|
+
end
|
|
162
|
+
end
|
|
163
|
+
end
|
|
@@ -0,0 +1,270 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Tilework
|
|
4
|
+
# Two-dimensional NCHW convolution provided by an available backend library.
|
|
5
|
+
#
|
|
6
|
+
# Tensors are **NCHW**:
|
|
7
|
+
# input [N, C_in, H, W] / weight [C_out, C_in, kh, kw] / output [N, C_out, H', W']
|
|
8
|
+
#
|
|
9
|
+
# **Do not hand-write im2col + GEMM.** Measured, im2col takes 74–84% of the
|
|
10
|
+
# time: GEMM alone reaches 2581 GF/s but the whole convolution drops to
|
|
11
|
+
# 613 GF/s, because im2col writes out kh*kw times the data and is memory-
|
|
12
|
+
# bound. cuDNN's implicit GEMM has no expansion at all and reaches
|
|
13
|
+
# fp16 4747 GF/s (7.7x) on the same shape.
|
|
14
|
+
module Conv
|
|
15
|
+
FP32 = 0
|
|
16
|
+
FP16 = 1
|
|
17
|
+
|
|
18
|
+
module_function
|
|
19
|
+
|
|
20
|
+
# @return [Boolean] whether the active backend exposes forward convolution
|
|
21
|
+
def available?
|
|
22
|
+
return false unless Tilework.backend
|
|
23
|
+
return false unless Tilework.bridge.respond_to?(:conv2d_available?)
|
|
24
|
+
|
|
25
|
+
Tilework.bridge.conv2d_available?
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
# @return [Symbol, nil] backend convolution library identifier
|
|
29
|
+
def library
|
|
30
|
+
return nil unless available?
|
|
31
|
+
return Tilework.bridge.conv_library_name if Tilework.bridge.respond_to?(:conv_library_name)
|
|
32
|
+
|
|
33
|
+
:cudnn
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
# @return [Boolean] whether the active backend exposes convolution backward passes
|
|
37
|
+
#
|
|
38
|
+
# Backward is a separate optional ABI from forward; a bridge may have only
|
|
39
|
+
# forward, so it is queried independently.
|
|
40
|
+
def backward_available?
|
|
41
|
+
return false unless available?
|
|
42
|
+
return false unless Tilework.bridge.respond_to?(:conv2d_backward_available?)
|
|
43
|
+
|
|
44
|
+
Tilework.bridge.conv2d_backward_available?
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
# Calculates one output spatial dimension.
|
|
48
|
+
#
|
|
49
|
+
# @param size [Integer] input size
|
|
50
|
+
# @param kernel [Integer] kernel size
|
|
51
|
+
# @param pad [Integer] padding on each side
|
|
52
|
+
# @param stride [Integer]
|
|
53
|
+
# @param dilation [Integer]
|
|
54
|
+
# @return [Integer]
|
|
55
|
+
def output_size(size, kernel, pad, stride, dilation = 1)
|
|
56
|
+
effective_kernel = ((kernel - 1) * dilation) + 1
|
|
57
|
+
((size + (2 * pad) - effective_kernel) / stride) + 1
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
# Performs NCHW convolution.
|
|
61
|
+
#
|
|
62
|
+
# @param input [Buffer, Graph::Tensor] `[N, C_in, H, W]`
|
|
63
|
+
# @param weight [Buffer, Graph::Tensor] `[C_out, C_in / groups, kh, kw]`
|
|
64
|
+
# @param pad [Integer, Array<Integer>]
|
|
65
|
+
# @param stride [Integer, Array<Integer>]
|
|
66
|
+
# @param dilation [Integer, Array<Integer>]
|
|
67
|
+
# @param groups [Integer]
|
|
68
|
+
# @return [Buffer, Graph::Tensor]
|
|
69
|
+
#
|
|
70
|
+
# padding / stride / dilation take a scalar (same for both axes) or [h, w].
|
|
71
|
+
# `groups` is the group count (`groups == cin` for depthwise).
|
|
72
|
+
#
|
|
73
|
+
# x = Tilework.randn([1, 3, 256, 256]) # NCHW
|
|
74
|
+
# w = Tilework.randn([64, 3, 3, 3]) # C_out, C_in, kh, kw
|
|
75
|
+
# y = Tilework::Conv.forward(x, w, pad: 1) # => [1, 64, 256, 256]
|
|
76
|
+
#
|
|
77
|
+
# Input and weight dtypes must match. float16 takes the tensor-core path,
|
|
78
|
+
# but **accumulation is always FP32**. With `groups > 1` the weight is
|
|
79
|
+
# `[C_out, C_in / groups, kh, kw]`.
|
|
80
|
+
def forward(input, weight, pad: 0, stride: 1, dilation: 1, groups: 1)
|
|
81
|
+
n, cin, h, w = check_input!(input)
|
|
82
|
+
cout, wcin, kh, kw = check_weight!(weight, cin)
|
|
83
|
+
pad_h, pad_w = pair(pad)
|
|
84
|
+
stride_h, stride_w = pair(stride)
|
|
85
|
+
dilation_h, dilation_w = pair(dilation)
|
|
86
|
+
|
|
87
|
+
check_groups!(cin, cout, wcin, groups)
|
|
88
|
+
|
|
89
|
+
oh = output_size(h, kh, pad_h, stride_h, dilation_h)
|
|
90
|
+
ow = output_size(w, kw, pad_w, stride_w, dilation_w)
|
|
91
|
+
unless oh.positive? && ow.positive?
|
|
92
|
+
raise ArgumentError, "kernel #{kh}x#{kw} with pad #{pad_h},#{pad_w} is larger than the input #{h}x#{w}"
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
dtype = check_dtypes!(input, weight)
|
|
96
|
+
# Record into the graph. On a backend without convolution it is
|
|
97
|
+
# decomposed into primitives (im2col + matmul) at record time, so it
|
|
98
|
+
# works on :cpu too (graph/high_level.rb's :conv2d).
|
|
99
|
+
if Graph.symbolic?(input, weight)
|
|
100
|
+
return Graph.record_conv2d(input, weight, shape: [n, cout, oh, ow],
|
|
101
|
+
pad: [pad_h, pad_w], stride: [stride_h, stride_w],
|
|
102
|
+
dilation: [dilation_h, dilation_w], groups: groups)
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
require_available!
|
|
106
|
+
unless Tilework.capabilities.conv_dtypes.include?(input.dtype)
|
|
107
|
+
raise Error, "#{Tilework.capabilities.conv} convolution does not support #{input.dtype.inspect} " \
|
|
108
|
+
"on the #{Tilework.backend.inspect} backend; use Graph for the portable decomposition"
|
|
109
|
+
end
|
|
110
|
+
# Take the output from the pool. A direct Buffer.new would never reuse the
|
|
111
|
+
# slots Plan#release returns to the pool, so they would pile up every run.
|
|
112
|
+
out = Ops.allocate_output(input.dtype, n * cout * oh * ow, [n, cout, oh, ow])
|
|
113
|
+
Tilework.bridge.conv2d(input.__handle__, weight.__handle__, out.__handle__,
|
|
114
|
+
[n, cin, h, w, cout, kh, kw, pad_h, pad_w, stride_h, stride_w,
|
|
115
|
+
dilation_h, dilation_w, groups], dtype)
|
|
116
|
+
out
|
|
117
|
+
rescue StandardError
|
|
118
|
+
out&.free
|
|
119
|
+
raise
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
# Calculates input and filter gradients for NCHW convolution.
|
|
123
|
+
#
|
|
124
|
+
# @param input [Buffer]
|
|
125
|
+
# @param weight [Buffer]
|
|
126
|
+
# @param grad_output [Buffer]
|
|
127
|
+
# @param pad [Integer, Array<Integer>]
|
|
128
|
+
# @param stride [Integer, Array<Integer>]
|
|
129
|
+
# @param dilation [Integer, Array<Integer>]
|
|
130
|
+
# @param groups [Integer]
|
|
131
|
+
# @return [Array<Buffer>] input gradient and filter gradient
|
|
132
|
+
#
|
|
133
|
+
# Takes the forward arguments plus the output gradient `grad_output` and
|
|
134
|
+
# returns **the input gradient and the weight gradient**.
|
|
135
|
+
#
|
|
136
|
+
# y = Tilework::Conv.forward(x, w, pad: 1)
|
|
137
|
+
# dx, dw = Tilework::Conv.backward(x, w, dy, pad: 1)
|
|
138
|
+
#
|
|
139
|
+
# Descriptors are shared with the forward plan, so **a forward pass of the
|
|
140
|
+
# same shape must have run once first** — which a training loop always does.
|
|
141
|
+
def backward(input, weight, grad_output, pad: 0, stride: 1, dilation: 1, groups: 1)
|
|
142
|
+
data = backward_data(weight, grad_output, input, pad: pad, stride: stride, dilation: dilation, groups: groups)
|
|
143
|
+
filter = backward_filter(input, grad_output, weight, pad: pad, stride: stride, dilation: dilation, groups: groups)
|
|
144
|
+
[data, filter]
|
|
145
|
+
rescue StandardError
|
|
146
|
+
data&.free
|
|
147
|
+
filter&.free
|
|
148
|
+
raise
|
|
149
|
+
end
|
|
150
|
+
|
|
151
|
+
# Input gradient dx. `like` is a Buffer shaped like the input (used only for its shape).
|
|
152
|
+
def backward_data(weight, grad_output, like, pad: 0, stride: 1, dilation: 1, groups: 1)
|
|
153
|
+
dims, dtype = backward_setup!(like, weight, grad_output, pad, stride, dilation, groups)
|
|
154
|
+
out = Ops.allocate_output(like.dtype, like.length, like.shape)
|
|
155
|
+
Tilework.bridge.conv2d_backward(weight.__handle__, grad_output.__handle__, out.__handle__,
|
|
156
|
+
dims, dtype, 0)
|
|
157
|
+
out
|
|
158
|
+
rescue StandardError
|
|
159
|
+
out&.free
|
|
160
|
+
raise
|
|
161
|
+
end
|
|
162
|
+
|
|
163
|
+
# Weight gradient dw. `like` is a Buffer shaped like the weight.
|
|
164
|
+
def backward_filter(input, grad_output, like, pad: 0, stride: 1, dilation: 1, groups: 1)
|
|
165
|
+
dims, dtype = backward_setup!(input, like, grad_output, pad, stride, dilation, groups)
|
|
166
|
+
out = Ops.allocate_output(like.dtype, like.length, like.shape)
|
|
167
|
+
Tilework.bridge.conv2d_backward(input.__handle__, grad_output.__handle__, out.__handle__,
|
|
168
|
+
dims, dtype, 1)
|
|
169
|
+
out
|
|
170
|
+
rescue StandardError
|
|
171
|
+
out&.free
|
|
172
|
+
raise
|
|
173
|
+
end
|
|
174
|
+
|
|
175
|
+
# --- Internal ---
|
|
176
|
+
|
|
177
|
+
# Validates backward arguments and builds dims. Besides the forward checks,
|
|
178
|
+
# grad_output must have the forward output's shape (otherwise cuDNN quietly
|
|
179
|
+
# reads somewhere else).
|
|
180
|
+
def backward_setup!(input, weight, grad_output, pad, stride, dilation = 1, groups = 1)
|
|
181
|
+
unless backward_available?
|
|
182
|
+
raise Error, "convolution backward passes are not available on the " \
|
|
183
|
+
"#{Tilework.backend.inspect} backend (cuDNN is used on :cuda)"
|
|
184
|
+
end
|
|
185
|
+
if [input, weight, grad_output].any? { |buffer| buffer.dtype == :bfloat16 }
|
|
186
|
+
raise Error, "bfloat16 convolution backward is not supported; keep training buffers in :float32"
|
|
187
|
+
end
|
|
188
|
+
|
|
189
|
+
n, cin, h, w = check_input!(input)
|
|
190
|
+
cout, wcin, kh, kw = check_weight!(weight, cin)
|
|
191
|
+
pad_h, pad_w = pair(pad)
|
|
192
|
+
stride_h, stride_w = pair(stride)
|
|
193
|
+
dilation_h, dilation_w = pair(dilation)
|
|
194
|
+
|
|
195
|
+
check_groups!(cin, cout, wcin, groups)
|
|
196
|
+
|
|
197
|
+
oh = output_size(h, kh, pad_h, stride_h, dilation_h)
|
|
198
|
+
ow = output_size(w, kw, pad_w, stride_w, dilation_w)
|
|
199
|
+
expected = [n, cout, oh, ow]
|
|
200
|
+
unless grad_output.shape == expected
|
|
201
|
+
raise ArgumentError,
|
|
202
|
+
"grad_output shape #{grad_output.shape.inspect} does not match the forward output #{expected.inspect}"
|
|
203
|
+
end
|
|
204
|
+
|
|
205
|
+
dtype = check_dtypes!(input, weight)
|
|
206
|
+
unless grad_output.dtype == input.dtype
|
|
207
|
+
raise ArgumentError,
|
|
208
|
+
"grad_output dtype #{grad_output.dtype.inspect} does not match the input's #{input.dtype.inspect}"
|
|
209
|
+
end
|
|
210
|
+
|
|
211
|
+
[[n, cin, h, w, cout, kh, kw, pad_h, pad_w, stride_h, stride_w, dilation_h, dilation_w, groups], dtype]
|
|
212
|
+
end
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
def pair(value)
|
|
216
|
+
value.is_a?(Array) ? value : [value, value]
|
|
217
|
+
end
|
|
218
|
+
|
|
219
|
+
# Validates `groups`: cin and cout must both divide, and the weight's input
|
|
220
|
+
# channel count must equal cin/groups. With groups == 1 this is the usual
|
|
221
|
+
# "weight input channels == cin" check.
|
|
222
|
+
def check_groups!(cin, cout, wcin, groups)
|
|
223
|
+
unless groups.positive? && (cin % groups).zero? && (cout % groups).zero?
|
|
224
|
+
raise ArgumentError,
|
|
225
|
+
"groups #{groups} must evenly divide both input channels #{cin} and output channels #{cout}"
|
|
226
|
+
end
|
|
227
|
+
|
|
228
|
+
expected_wcin = cin / groups
|
|
229
|
+
return if wcin == expected_wcin
|
|
230
|
+
|
|
231
|
+
raise ArgumentError,
|
|
232
|
+
"weight input channels #{wcin} do not match input channels #{cin} / groups #{groups} (#{expected_wcin})"
|
|
233
|
+
end
|
|
234
|
+
|
|
235
|
+
def check_input!(input)
|
|
236
|
+
return input.shape if input.shape.length == 4
|
|
237
|
+
|
|
238
|
+
raise ArgumentError, "conv2d input must be 4D [N, C, H, W], got #{input.shape.inspect}"
|
|
239
|
+
end
|
|
240
|
+
|
|
241
|
+
def check_weight!(weight, _cin)
|
|
242
|
+
return weight.shape if weight.shape.length == 4
|
|
243
|
+
|
|
244
|
+
raise ArgumentError,
|
|
245
|
+
"conv2d weight must be 4D [C_out, C_in, kh, kw], got #{weight.shape.inspect}"
|
|
246
|
+
end
|
|
247
|
+
|
|
248
|
+
def check_dtypes!(input, weight)
|
|
249
|
+
unless input.dtype == weight.dtype
|
|
250
|
+
raise ArgumentError,
|
|
251
|
+
"input and weight dtypes must match (got #{input.dtype.inspect} and #{weight.dtype.inspect})"
|
|
252
|
+
end
|
|
253
|
+
|
|
254
|
+
case input.dtype
|
|
255
|
+
when :float32 then FP32
|
|
256
|
+
when :float16 then FP16
|
|
257
|
+
when :bfloat16 then 2
|
|
258
|
+
else
|
|
259
|
+
raise Error, "conv2d supports :float32, :float16, and :bfloat16 (got #{input.dtype.inspect})"
|
|
260
|
+
end
|
|
261
|
+
end
|
|
262
|
+
|
|
263
|
+
def require_available!
|
|
264
|
+
return if available?
|
|
265
|
+
|
|
266
|
+
raise Error, "no convolution library is available on the #{Tilework.backend.inspect} backend " \
|
|
267
|
+
"(cuDNN is used on :cuda)"
|
|
268
|
+
end
|
|
269
|
+
end
|
|
270
|
+
end
|