tilework 0.18.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE +21 -0
  3. data/lib/tilework/backend/cpu/dsl_codegen.rb +509 -0
  4. data/lib/tilework/backend/cpu/ops_codegen.rb +451 -0
  5. data/lib/tilework/backend/cpu.rb +156 -0
  6. data/lib/tilework/buffer.rb +753 -0
  7. data/lib/tilework/capabilities.rb +183 -0
  8. data/lib/tilework/compat/cuda.h +66 -0
  9. data/lib/tilework/compat/hip.h +35 -0
  10. data/lib/tilework/compat/metal.h +45 -0
  11. data/lib/tilework/conformance.rb +2296 -0
  12. data/lib/tilework/core_ext.rb +21 -0
  13. data/lib/tilework/dialect.rb +67 -0
  14. data/lib/tilework/dsl.rb +722 -0
  15. data/lib/tilework/dtype/bfloat.rb +31 -0
  16. data/lib/tilework/dtype/cast.rb +88 -0
  17. data/lib/tilework/dtype/half.rb +98 -0
  18. data/lib/tilework/graph/builder.rb +1073 -0
  19. data/lib/tilework/graph/fusion.rb +520 -0
  20. data/lib/tilework/graph/grad.rb +443 -0
  21. data/lib/tilework/graph/high_level.rb +266 -0
  22. data/lib/tilework/graph/kernels.rb +403 -0
  23. data/lib/tilework/graph/node.rb +27 -0
  24. data/lib/tilework/graph/op.rb +290 -0
  25. data/lib/tilework/graph/optim.rb +183 -0
  26. data/lib/tilework/graph/plan.rb +1086 -0
  27. data/lib/tilework/graph/pool.rb +178 -0
  28. data/lib/tilework/graph/region.rb +210 -0
  29. data/lib/tilework/graph/residency.rb +449 -0
  30. data/lib/tilework/graph/state.rb +64 -0
  31. data/lib/tilework/graph/tensor.rb +124 -0
  32. data/lib/tilework/graph.rb +213 -0
  33. data/lib/tilework/kernel.rb +163 -0
  34. data/lib/tilework/math/conv.rb +270 -0
  35. data/lib/tilework/math/fft.rb +172 -0
  36. data/lib/tilework/math/nn.rb +1103 -0
  37. data/lib/tilework/math/philox.rb +194 -0
  38. data/lib/tilework/math/random.rb +232 -0
  39. data/lib/tilework/math/tensor_core.rb +269 -0
  40. data/lib/tilework/ops.rb +682 -0
  41. data/lib/tilework/runtime/offload.rb +498 -0
  42. data/lib/tilework/runtime/submission.rb +163 -0
  43. data/lib/tilework/runtime/telemetry.rb +112 -0
  44. data/lib/tilework/scope.rb +33 -0
  45. data/lib/tilework/version.rb +5 -0
  46. data/lib/tilework.rb +345 -0
  47. metadata +92 -0
@@ -0,0 +1,213 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "graph/region"
4
+ require_relative "graph/op"
5
+ require_relative "graph/node"
6
+ require_relative "graph/state"
7
+ require_relative "graph/tensor"
8
+ require_relative "graph/pool"
9
+ require_relative "graph/residency"
10
+ require_relative "graph/kernels"
11
+ require_relative "graph/high_level"
12
+ require_relative "graph/grad"
13
+ require_relative "graph/fusion"
14
+ require_relative "graph/builder"
15
+ require_relative "graph/plan"
16
+ require_relative "graph/optim"
17
+
18
+ module Tilework
19
+ # Opt-in deferred graph execution for tensors and plans.
20
+ #
21
+ # plan = Tilework::Graph.build do |g|
22
+ # x = g.input(:x, [4, 8])
23
+ # w = g.param(weights)
24
+ # g.output(x.matmul(w) * 2.0)
25
+ # end.compile
26
+ #
27
+ # y = plan.run(x: input_buffer)
28
+ #
29
+ # A layer added **on top of** the eager layer (Buffer / Ops); nothing below
30
+ # changes. The graph is opt-in, so being able to write eager code is never
31
+ # traded away.
32
+ #
33
+ # ## Mutable state
34
+ #
35
+ # A graph is a DAG of values, so things that live and change across plan
36
+ # runs, like a KV cache, do not fit naturally. Here a write is **an operation
37
+ # that returns a version**, turning the invisible memory dependency into a
38
+ # value edge:
39
+ #
40
+ # kv0 = g.state(k_cache)
41
+ # kv1 = g.write_into(kv0, at: g.index(:pos), value: k_new)
42
+ # k_all = g.read(kv1, 0..pos) # depends on version 1 = a real edge
43
+ #
44
+ # Unlike SSA, a version points at the same memory rather than a new value,
45
+ # so version edges express only RAW. Plan#compile adds WAR (writing while a
46
+ # reader is still reading) as ordering edges.
47
+ module Graph
48
+ # Read semantics.
49
+ # :view — zero copy (default); no kernel
50
+ # :copy — materialize. The graph becomes a pure DAG, so comparing it with
51
+ # :view's result verifies the state machinery itself (slow)
52
+ STRICT_MODES = %i[view copy].freeze
53
+
54
+ class << self
55
+ # Records a graph without allocating device memory.
56
+ #
57
+ # @param differentiable [Boolean] record decompositions required for reverse-mode gradients
58
+ # @yieldparam builder [Graph::Builder]
59
+ # @return [Graph::Builder] the recorded graph; call `#compile` to create a plan
60
+ #
61
+ # With differentiable: true, high-level ops (NN.softmax etc.) are recorded
62
+ # as their decompositions. Gradients follow from primitive vjps alone, so
63
+ # graphs that use g.grad are recorded this way (docs/autograd.md).
64
+ def build(differentiable: false)
65
+ builder = Builder.new(differentiable: differentiable)
66
+ yield builder
67
+ builder
68
+ end
69
+
70
+ # Whether any value is being recorded into a graph. Each `NN` function
71
+ # branches on this in one line so it can be called eager or in a graph.
72
+ def symbolic?(*values) = values.any? { |value| value.is_a?(Tensor) }
73
+
74
+ # Records a **built-in the graph does not decompose**, like `NN`, as a node.
75
+ #
76
+ # return Graph.record_nn(:softmax, [x], shape: x.shape, dtype: x.dtype) if Graph.symbolic?(x)
77
+ #
78
+ # Unlike `g.opaque`, the graph knows what this is by name: CSE applies, it
79
+ # runs as the same-named `NN` function, and fusion can treat it as a
80
+ # partner (opaque stays opaque to the end).
81
+ #
82
+ # tensors: are keyword arguments taking Tensors (gamma / beta / bias ...);
83
+ # nils are dropped. args: are non-Tensor positional arguments and opts:
84
+ # non-Tensor keyword arguments, both carried as node attributes.
85
+ def record_nn(fn, inputs, shape:, dtype:, tensors: {}, args: [], opts: {})
86
+ builder = builder_for(inputs, tensors, fn)
87
+ builder.nn(fn, inputs, shape: shape, dtype: dtype, tensors: tensors, args: args, opts: opts)
88
+ end
89
+
90
+ # Records a convolution (called in one line from Conv.forward). Unlike :nn
91
+ # the callee is Tilework::Conv, hence its own op.
92
+ def record_conv2d(input, weight, shape:, pad:, stride:, dilation: [1, 1], groups: 1)
93
+ builder = builder_for([input, weight], {}, :conv2d)
94
+ builder.conv2d(input, weight, shape: shape, pad: pad, stride: stride, dilation: dilation, groups: groups)
95
+ end
96
+
97
+ # Queue launches without waiting (default). false waits per node like
98
+ # eager — for measuring the effect and for bisecting ordering suspicions.
99
+ def async = @async.nil? ? true : @async
100
+ attr_writer :async
101
+
102
+ # Reuse intermediates from the pool (default). false allocates per op and
103
+ # frees at the end. Like async, for measuring and bisecting.
104
+ def buffer_pool = @buffer_pool.nil? ? true : @buffer_pool
105
+ attr_writer :buffer_pool
106
+
107
+ # Fold chained elementwise ops into one kernel (#17 Phase 2). Can be
108
+ # turned off like async / buffer_pool for measuring and bisecting;
109
+ # conformance pins that fusion does not change values.
110
+ def fusion = @fusion.nil? ? true : @fusion
111
+ attr_writer :fusion
112
+
113
+ # Fold expressions containing row-wise reductions into one kernel
114
+ # (docs/graph-ir.md Phase C). One thread owns one row, so with few rows
115
+ # per-element parallelism can be faster — switchable until measured on hardware.
116
+ def row_fusion = @row_fusion.nil? ? true : @row_fusion
117
+ attr_writer :row_fusion
118
+
119
+ # Hand the node producing a write_into value the state's view as its
120
+ # output (#19 §3). Off, the value is produced and then copied — like the
121
+ # other switches, for measuring and bisecting.
122
+ def placement = @placement.nil? ? true : @placement
123
+ attr_writer :placement
124
+
125
+ # Record high-level ops (:nn nodes) as their primitive decompositions.
126
+ # Default false — execution stays on the hand-written fused kernels and
127
+ # the decomposition serves as the definition of meaning (Tier 1 in
128
+ # docs/graph-ir.md). true lets the two be compared, which is what the
129
+ # conformance oracle uses.
130
+ #
131
+ # **It takes effect at record time**, so switch it before Graph.build.
132
+ def lower_high_level = @lower_high_level.nil? ? false : @lower_high_level
133
+ attr_writer :lower_high_level
134
+
135
+ # Page weights in and out of VRAM to fit a budget (#17 Phase 4). On by
136
+ # default, but **nothing happens without compile(vram:)** — this layer
137
+ # never starts paging without being given a budget. Turning it off is,
138
+ # like the other switches, for measuring and bisecting.
139
+ def offload = @offload.nil? ? true : @offload
140
+ attr_writer :offload
141
+
142
+ # Ride eviction fences on nodes that already synchronize (#17 Phase 4c).
143
+ # Off, every eviction adds a Tilework.synchronize — a switch for
144
+ # measuring how much riding along actually saves.
145
+ def offload_fence_reuse = @offload_fence_reuse.nil? ? true : @offload_fence_reuse
146
+ attr_writer :offload_fence_reuse
147
+
148
+ # Fetch asynchronously ahead of the demand point (#17 Phase 4b), closing
149
+ # the gap measured on Jetson between one synchronous transfer at a time
150
+ # (0.56 GB/s) and async depth 2–4 (1.85–2.24 GB/s). Without an async
151
+ # transfer ABI (Offload.async? false) it never starts, so this switch
152
+ # matters only for measuring and bisecting on the same machine.
153
+ def offload_prefetch = @offload_prefetch.nil? ? true : @offload_prefetch
154
+ attr_writer :offload_prefetch
155
+
156
+ def strict_state = @strict_state || :view
157
+
158
+ # Selects zero-copy or copied state reads for subsequently compiled plans.
159
+ #
160
+ # @param mode [:view, :copy]
161
+ # @return [:view, :copy]
162
+ # @raise [ArgumentError] for an unsupported mode
163
+ def strict_state=(mode)
164
+ mode = mode.to_sym
165
+ unless STRICT_MODES.include?(mode)
166
+ raise ArgumentError, "unknown strict_state #{mode.inspect} (expected #{STRICT_MODES.join(' or ')})"
167
+ end
168
+
169
+ @strict_state = mode
170
+ end
171
+
172
+ # An on-device copy, used for state writes that cannot be placed and for
173
+ # materializing reads. When the producer can write straight into the
174
+ # state's view (Graph.placement), no copy runs.
175
+ def copy(src, dst)
176
+ unless src.dtype == dst.dtype && src.length == dst.length
177
+ raise Error, "cannot copy #{src.length} #{src.dtype} elements into " \
178
+ "#{dst.length} #{dst.dtype} elements"
179
+ end
180
+
181
+ n = src.length
182
+ args = [flat(src, n), flat(dst, n), n]
183
+ grid = (n / Ops::BLOCK.to_f).ceil
184
+ # The bounds guard is in the kernel body, so launch with explicit
185
+ # grid/block rather than the auto-guarded .() (only this form can be async).
186
+ if Ops.context&.async?
187
+ copy_kernel.launch_async(grid: grid, block: Ops::BLOCK, args: args)
188
+ else
189
+ copy_kernel.launch(grid: grid, block: Ops::BLOCK, args: args)
190
+ end
191
+ dst
192
+ end
193
+
194
+ private
195
+
196
+ def builder_for(inputs, tensors, fn)
197
+ found = (inputs + tensors.values).find { |value| value.is_a?(Tensor) }
198
+ raise Error, "#{fn} got no Tilework::Graph::Tensor to record against" if found.nil?
199
+
200
+ found.graph
201
+ end
202
+
203
+ def flat(buffer, n) = buffer.shape.length > 1 ? buffer.reshape(n) : buffer
204
+
205
+ def copy_kernel
206
+ @copy_kernel ||= Tilework.kernel("graph_copy") do |a, b, n|
207
+ i = global_id_x
208
+ b[i] = a[i] if i < n
209
+ end
210
+ end
211
+ end
212
+ end
213
+ end
@@ -0,0 +1,163 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "digest"
4
+
5
+ module Tilework
6
+ # A compiled kernel pipeline.
7
+ #
8
+ # `grid` is the number of thread groups and `block` is the thread-group size,
9
+ # matching CUDA's `<<<grid, block>>>` execution model.
10
+ class Kernel
11
+ @cache = {}
12
+ @cache_mutex = Mutex.new
13
+
14
+ class << self
15
+ # Compiles source and reuses an existing pipeline for identical source and name.
16
+ #
17
+ # @api private
18
+ # @param src [String]
19
+ # @param name [String, Symbol]
20
+ # @param raw [Boolean]
21
+ # @return [Kernel]
22
+ def compile(src, name:, raw: false)
23
+ processed = raw ? src : Dialect.expand(src)
24
+ bridge = Tilework.bridge
25
+ key = [bridge.object_id, cache_key(processed, name)]
26
+ pipeline = @cache_mutex.synchronize do
27
+ cache_hit = @cache.key?(key)
28
+ Telemetry.instrument(:compile, label: name, cache_hit: cache_hit) do
29
+ @cache[key] ||= create_pipeline(bridge, processed, name)
30
+ end
31
+ end
32
+ new(pipeline, name, bridge, processed)
33
+ end
34
+
35
+ def cache_key(processed_src, name)
36
+ Digest::SHA256.hexdigest("#{processed_src}\0#{name}")
37
+ end
38
+
39
+ # @return [Integer] number of cached compiled pipelines
40
+ def cache_size
41
+ @cache_mutex.synchronize { @cache.size }
42
+ end
43
+
44
+ # Clears the in-process pipeline cache.
45
+ #
46
+ # @return [Hash] the cleared cache
47
+ def clear_cache!
48
+ @cache_mutex.synchronize { @cache.clear }
49
+ end
50
+
51
+ private
52
+
53
+ def create_pipeline(bridge, src, name)
54
+ bridge.pipeline_create(src, name)
55
+ rescue RuntimeError => e
56
+ raise CompileError, "failed to compile kernel '#{name}':\n#{e.message}"
57
+ end
58
+ end
59
+
60
+ attr_reader :name
61
+
62
+ def initialize(pipeline, name, bridge, processed_source)
63
+ @pipeline = pipeline
64
+ @name = name
65
+ @bridge = bridge
66
+ @processed_source = processed_source
67
+ end
68
+
69
+ # Launches the kernel and waits for its completion.
70
+ #
71
+ # @param grid [Integer, Array<Integer>] one to three positive grid dimensions
72
+ # @param block [Integer, Array<Integer>] one to three positive thread-group dimensions
73
+ # @param args [Array<Buffer, Numeric>] kernel arguments
74
+ # @return [nil]
75
+ # @raise [LaunchError] if arguments or launch dimensions are invalid
76
+ def launch(grid:, block:, args: [])
77
+ dispatch(grid, block, args, wait: true)
78
+ end
79
+
80
+ # Launches the kernel without waiting for completion.
81
+ #
82
+ # @param grid [Integer, Array<Integer>]
83
+ # @param block [Integer, Array<Integer>]
84
+ # @param args [Array<Buffer, Numeric>]
85
+ # @return [nil]
86
+ # @note Read a buffer or call {Tilework.synchronize} before consuming results on the host.
87
+ def launch_async(grid:, block:, args: [])
88
+ dispatch(grid, block, args, wait: false)
89
+ end
90
+
91
+ # @return [Integer] backend limit for threads in one thread group
92
+ def max_threads_per_block
93
+ @bridge.pipeline_max_threads(@pipeline)
94
+ end
95
+
96
+ private
97
+
98
+ def dispatch(grid, block, args, wait:)
99
+ recompile_for_current_bridge! unless Tilework.bridge.equal?(@bridge)
100
+
101
+ grid = normalize_dim(grid, "grid")
102
+ block = normalize_dim(block, "block")
103
+ validate_block!(block)
104
+ native_args = convert_args(args)
105
+ bytes = args.sum { |a| a.is_a?(Buffer) ? a.bytesize : 0 }
106
+ threads = grid.zip(block).reduce(1) { |acc, (g, b)| acc * g * b }
107
+
108
+ begin
109
+ Telemetry.instrument(:dispatch, label: @name, bytes: bytes, threads: threads) do
110
+ @bridge.dispatch(@pipeline, native_args, grid, block, wait)
111
+ end
112
+ rescue RuntimeError => e
113
+ raise LaunchError, "kernel '#{@name}' failed: #{e.message}"
114
+ end
115
+ nil
116
+ end
117
+
118
+ # DSL and built-in kernels are memoized above the pipeline cache. A backend
119
+ # change therefore cannot merely reject the old object: refresh its native
120
+ # pipeline lazily so those long-lived Ruby kernel objects stay portable.
121
+ def recompile_for_current_bridge!
122
+ refreshed = self.class.compile(@processed_source, name: @name, raw: true)
123
+ @pipeline = refreshed.instance_variable_get(:@pipeline)
124
+ @bridge = refreshed.instance_variable_get(:@bridge)
125
+ end
126
+
127
+ def normalize_dim(dim, label)
128
+ dims = dim.is_a?(Integer) ? [dim, 1, 1] : Array(dim)
129
+ dims += [1] * (3 - dims.length) if dims.length < 3
130
+ unless dims.length == 3 && dims.all? { |d| d.is_a?(Integer) && d.positive? }
131
+ raise LaunchError, "#{label} must be a positive Integer or an Array of up to 3 positive Integers, got #{dim.inspect}"
132
+ end
133
+ dims
134
+ end
135
+
136
+ def validate_block!(block)
137
+ total = block.reduce(:*)
138
+ max = max_threads_per_block
139
+ return if total <= max
140
+
141
+ raise LaunchError,
142
+ "block size #{block.inspect} (#{total} threads) exceeds the pipeline limit of #{max} threads per threadgroup"
143
+ end
144
+
145
+ # A Buffer becomes setBuffer; a Ruby number becomes a setBytes byte string
146
+ # (fixed Float -> float32 / Integer -> int32 for now).
147
+ def convert_args(args)
148
+ if args.length > MAX_KERNEL_ARGS
149
+ raise LaunchError, "too many kernel arguments: #{args.length} (Metal limit is #{MAX_KERNEL_ARGS})"
150
+ end
151
+
152
+ args.map.with_index do |arg, i|
153
+ case arg
154
+ when Buffer then arg.__handle__
155
+ when Float then [arg].pack("f")
156
+ when Integer then [arg].pack("l")
157
+ else
158
+ raise LaunchError, "unsupported kernel argument at index #{i}: #{arg.class} (expected Tilework::Buffer, Float, or Integer)"
159
+ end
160
+ end
161
+ end
162
+ end
163
+ end
@@ -0,0 +1,270 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Tilework
4
+ # Two-dimensional NCHW convolution provided by an available backend library.
5
+ #
6
+ # Tensors are **NCHW**:
7
+ # input [N, C_in, H, W] / weight [C_out, C_in, kh, kw] / output [N, C_out, H', W']
8
+ #
9
+ # **Do not hand-write im2col + GEMM.** Measured, im2col takes 74–84% of the
10
+ # time: GEMM alone reaches 2581 GF/s but the whole convolution drops to
11
+ # 613 GF/s, because im2col writes out kh*kw times the data and is memory-
12
+ # bound. cuDNN's implicit GEMM has no expansion at all and reaches
13
+ # fp16 4747 GF/s (7.7x) on the same shape.
14
+ module Conv
15
+ FP32 = 0
16
+ FP16 = 1
17
+
18
+ module_function
19
+
20
+ # @return [Boolean] whether the active backend exposes forward convolution
21
+ def available?
22
+ return false unless Tilework.backend
23
+ return false unless Tilework.bridge.respond_to?(:conv2d_available?)
24
+
25
+ Tilework.bridge.conv2d_available?
26
+ end
27
+
28
+ # @return [Symbol, nil] backend convolution library identifier
29
+ def library
30
+ return nil unless available?
31
+ return Tilework.bridge.conv_library_name if Tilework.bridge.respond_to?(:conv_library_name)
32
+
33
+ :cudnn
34
+ end
35
+
36
+ # @return [Boolean] whether the active backend exposes convolution backward passes
37
+ #
38
+ # Backward is a separate optional ABI from forward; a bridge may have only
39
+ # forward, so it is queried independently.
40
+ def backward_available?
41
+ return false unless available?
42
+ return false unless Tilework.bridge.respond_to?(:conv2d_backward_available?)
43
+
44
+ Tilework.bridge.conv2d_backward_available?
45
+ end
46
+
47
+ # Calculates one output spatial dimension.
48
+ #
49
+ # @param size [Integer] input size
50
+ # @param kernel [Integer] kernel size
51
+ # @param pad [Integer] padding on each side
52
+ # @param stride [Integer]
53
+ # @param dilation [Integer]
54
+ # @return [Integer]
55
+ def output_size(size, kernel, pad, stride, dilation = 1)
56
+ effective_kernel = ((kernel - 1) * dilation) + 1
57
+ ((size + (2 * pad) - effective_kernel) / stride) + 1
58
+ end
59
+
60
+ # Performs NCHW convolution.
61
+ #
62
+ # @param input [Buffer, Graph::Tensor] `[N, C_in, H, W]`
63
+ # @param weight [Buffer, Graph::Tensor] `[C_out, C_in / groups, kh, kw]`
64
+ # @param pad [Integer, Array<Integer>]
65
+ # @param stride [Integer, Array<Integer>]
66
+ # @param dilation [Integer, Array<Integer>]
67
+ # @param groups [Integer]
68
+ # @return [Buffer, Graph::Tensor]
69
+ #
70
+ # padding / stride / dilation take a scalar (same for both axes) or [h, w].
71
+ # `groups` is the group count (`groups == cin` for depthwise).
72
+ #
73
+ # x = Tilework.randn([1, 3, 256, 256]) # NCHW
74
+ # w = Tilework.randn([64, 3, 3, 3]) # C_out, C_in, kh, kw
75
+ # y = Tilework::Conv.forward(x, w, pad: 1) # => [1, 64, 256, 256]
76
+ #
77
+ # Input and weight dtypes must match. float16 takes the tensor-core path,
78
+ # but **accumulation is always FP32**. With `groups > 1` the weight is
79
+ # `[C_out, C_in / groups, kh, kw]`.
80
+ def forward(input, weight, pad: 0, stride: 1, dilation: 1, groups: 1)
81
+ n, cin, h, w = check_input!(input)
82
+ cout, wcin, kh, kw = check_weight!(weight, cin)
83
+ pad_h, pad_w = pair(pad)
84
+ stride_h, stride_w = pair(stride)
85
+ dilation_h, dilation_w = pair(dilation)
86
+
87
+ check_groups!(cin, cout, wcin, groups)
88
+
89
+ oh = output_size(h, kh, pad_h, stride_h, dilation_h)
90
+ ow = output_size(w, kw, pad_w, stride_w, dilation_w)
91
+ unless oh.positive? && ow.positive?
92
+ raise ArgumentError, "kernel #{kh}x#{kw} with pad #{pad_h},#{pad_w} is larger than the input #{h}x#{w}"
93
+ end
94
+
95
+ dtype = check_dtypes!(input, weight)
96
+ # Record into the graph. On a backend without convolution it is
97
+ # decomposed into primitives (im2col + matmul) at record time, so it
98
+ # works on :cpu too (graph/high_level.rb's :conv2d).
99
+ if Graph.symbolic?(input, weight)
100
+ return Graph.record_conv2d(input, weight, shape: [n, cout, oh, ow],
101
+ pad: [pad_h, pad_w], stride: [stride_h, stride_w],
102
+ dilation: [dilation_h, dilation_w], groups: groups)
103
+ end
104
+
105
+ require_available!
106
+ unless Tilework.capabilities.conv_dtypes.include?(input.dtype)
107
+ raise Error, "#{Tilework.capabilities.conv} convolution does not support #{input.dtype.inspect} " \
108
+ "on the #{Tilework.backend.inspect} backend; use Graph for the portable decomposition"
109
+ end
110
+ # Take the output from the pool. A direct Buffer.new would never reuse the
111
+ # slots Plan#release returns to the pool, so they would pile up every run.
112
+ out = Ops.allocate_output(input.dtype, n * cout * oh * ow, [n, cout, oh, ow])
113
+ Tilework.bridge.conv2d(input.__handle__, weight.__handle__, out.__handle__,
114
+ [n, cin, h, w, cout, kh, kw, pad_h, pad_w, stride_h, stride_w,
115
+ dilation_h, dilation_w, groups], dtype)
116
+ out
117
+ rescue StandardError
118
+ out&.free
119
+ raise
120
+ end
121
+
122
+ # Calculates input and filter gradients for NCHW convolution.
123
+ #
124
+ # @param input [Buffer]
125
+ # @param weight [Buffer]
126
+ # @param grad_output [Buffer]
127
+ # @param pad [Integer, Array<Integer>]
128
+ # @param stride [Integer, Array<Integer>]
129
+ # @param dilation [Integer, Array<Integer>]
130
+ # @param groups [Integer]
131
+ # @return [Array<Buffer>] input gradient and filter gradient
132
+ #
133
+ # Takes the forward arguments plus the output gradient `grad_output` and
134
+ # returns **the input gradient and the weight gradient**.
135
+ #
136
+ # y = Tilework::Conv.forward(x, w, pad: 1)
137
+ # dx, dw = Tilework::Conv.backward(x, w, dy, pad: 1)
138
+ #
139
+ # Descriptors are shared with the forward plan, so **a forward pass of the
140
+ # same shape must have run once first** — which a training loop always does.
141
+ def backward(input, weight, grad_output, pad: 0, stride: 1, dilation: 1, groups: 1)
142
+ data = backward_data(weight, grad_output, input, pad: pad, stride: stride, dilation: dilation, groups: groups)
143
+ filter = backward_filter(input, grad_output, weight, pad: pad, stride: stride, dilation: dilation, groups: groups)
144
+ [data, filter]
145
+ rescue StandardError
146
+ data&.free
147
+ filter&.free
148
+ raise
149
+ end
150
+
151
+ # Input gradient dx. `like` is a Buffer shaped like the input (used only for its shape).
152
+ def backward_data(weight, grad_output, like, pad: 0, stride: 1, dilation: 1, groups: 1)
153
+ dims, dtype = backward_setup!(like, weight, grad_output, pad, stride, dilation, groups)
154
+ out = Ops.allocate_output(like.dtype, like.length, like.shape)
155
+ Tilework.bridge.conv2d_backward(weight.__handle__, grad_output.__handle__, out.__handle__,
156
+ dims, dtype, 0)
157
+ out
158
+ rescue StandardError
159
+ out&.free
160
+ raise
161
+ end
162
+
163
+ # Weight gradient dw. `like` is a Buffer shaped like the weight.
164
+ def backward_filter(input, grad_output, like, pad: 0, stride: 1, dilation: 1, groups: 1)
165
+ dims, dtype = backward_setup!(input, like, grad_output, pad, stride, dilation, groups)
166
+ out = Ops.allocate_output(like.dtype, like.length, like.shape)
167
+ Tilework.bridge.conv2d_backward(input.__handle__, grad_output.__handle__, out.__handle__,
168
+ dims, dtype, 1)
169
+ out
170
+ rescue StandardError
171
+ out&.free
172
+ raise
173
+ end
174
+
175
+ # --- Internal ---
176
+
177
+ # Validates backward arguments and builds dims. Besides the forward checks,
178
+ # grad_output must have the forward output's shape (otherwise cuDNN quietly
179
+ # reads somewhere else).
180
+ def backward_setup!(input, weight, grad_output, pad, stride, dilation = 1, groups = 1)
181
+ unless backward_available?
182
+ raise Error, "convolution backward passes are not available on the " \
183
+ "#{Tilework.backend.inspect} backend (cuDNN is used on :cuda)"
184
+ end
185
+ if [input, weight, grad_output].any? { |buffer| buffer.dtype == :bfloat16 }
186
+ raise Error, "bfloat16 convolution backward is not supported; keep training buffers in :float32"
187
+ end
188
+
189
+ n, cin, h, w = check_input!(input)
190
+ cout, wcin, kh, kw = check_weight!(weight, cin)
191
+ pad_h, pad_w = pair(pad)
192
+ stride_h, stride_w = pair(stride)
193
+ dilation_h, dilation_w = pair(dilation)
194
+
195
+ check_groups!(cin, cout, wcin, groups)
196
+
197
+ oh = output_size(h, kh, pad_h, stride_h, dilation_h)
198
+ ow = output_size(w, kw, pad_w, stride_w, dilation_w)
199
+ expected = [n, cout, oh, ow]
200
+ unless grad_output.shape == expected
201
+ raise ArgumentError,
202
+ "grad_output shape #{grad_output.shape.inspect} does not match the forward output #{expected.inspect}"
203
+ end
204
+
205
+ dtype = check_dtypes!(input, weight)
206
+ unless grad_output.dtype == input.dtype
207
+ raise ArgumentError,
208
+ "grad_output dtype #{grad_output.dtype.inspect} does not match the input's #{input.dtype.inspect}"
209
+ end
210
+
211
+ [[n, cin, h, w, cout, kh, kw, pad_h, pad_w, stride_h, stride_w, dilation_h, dilation_w, groups], dtype]
212
+ end
213
+
214
+
215
+ def pair(value)
216
+ value.is_a?(Array) ? value : [value, value]
217
+ end
218
+
219
+ # Validates `groups`: cin and cout must both divide, and the weight's input
220
+ # channel count must equal cin/groups. With groups == 1 this is the usual
221
+ # "weight input channels == cin" check.
222
+ def check_groups!(cin, cout, wcin, groups)
223
+ unless groups.positive? && (cin % groups).zero? && (cout % groups).zero?
224
+ raise ArgumentError,
225
+ "groups #{groups} must evenly divide both input channels #{cin} and output channels #{cout}"
226
+ end
227
+
228
+ expected_wcin = cin / groups
229
+ return if wcin == expected_wcin
230
+
231
+ raise ArgumentError,
232
+ "weight input channels #{wcin} do not match input channels #{cin} / groups #{groups} (#{expected_wcin})"
233
+ end
234
+
235
+ def check_input!(input)
236
+ return input.shape if input.shape.length == 4
237
+
238
+ raise ArgumentError, "conv2d input must be 4D [N, C, H, W], got #{input.shape.inspect}"
239
+ end
240
+
241
+ def check_weight!(weight, _cin)
242
+ return weight.shape if weight.shape.length == 4
243
+
244
+ raise ArgumentError,
245
+ "conv2d weight must be 4D [C_out, C_in, kh, kw], got #{weight.shape.inspect}"
246
+ end
247
+
248
+ def check_dtypes!(input, weight)
249
+ unless input.dtype == weight.dtype
250
+ raise ArgumentError,
251
+ "input and weight dtypes must match (got #{input.dtype.inspect} and #{weight.dtype.inspect})"
252
+ end
253
+
254
+ case input.dtype
255
+ when :float32 then FP32
256
+ when :float16 then FP16
257
+ when :bfloat16 then 2
258
+ else
259
+ raise Error, "conv2d supports :float32, :float16, and :bfloat16 (got #{input.dtype.inspect})"
260
+ end
261
+ end
262
+
263
+ def require_available!
264
+ return if available?
265
+
266
+ raise Error, "no convolution library is available on the #{Tilework.backend.inspect} backend " \
267
+ "(cuDNN is used on :cuda)"
268
+ end
269
+ end
270
+ end