tilework-metal 0.18.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,1203 @@
1
+ import Metal
2
+ import MetalPerformanceShaders
3
+ import MetalPerformanceShadersGraph
4
+ import Foundation
5
+
6
+ // The C ABI exported to the Ruby C shim (gpu_bridge.c). All policy (type
7
+ // conversion, validation, caching, exception hierarchy) lives in Ruby; this
8
+ // provides Metal mechanism only. Handles hand ownership to the C side
9
+ // (TypedData's free) via Unmanaged.passRetained.
10
+
11
+ final class BridgeContext {
12
+ static let shared = BridgeContext()
13
+ let device: MTLDevice
14
+ let queue: MTLCommandQueue
15
+ private var pending: [MTLCommandBuffer] = []
16
+ private var batchCommandBuffer: MTLCommandBuffer?
17
+ private var batchEncoder: MTLComputeCommandEncoder?
18
+ private var batchDispatches = 0
19
+ private let batchLimit = max(1, Int(ProcessInfo.processInfo.environment["TILEWORK_METAL_BATCH_SIZE"] ?? "8") ?? 8)
20
+ private var asyncError: String?
21
+ private let lock = NSLock()
22
+
23
+ private init() {
24
+ guard let device = MTLCreateSystemDefaultDevice(),
25
+ let queue = device.makeCommandQueue() else {
26
+ fatalError("tilework/metal: no Metal device available")
27
+ }
28
+ self.device = device
29
+ self.queue = queue
30
+ }
31
+
32
+ func track(_ cb: MTLCommandBuffer) {
33
+ lock.lock()
34
+ pruneFinishedLocked()
35
+ pending.append(cb)
36
+ lock.unlock()
37
+ }
38
+
39
+ // Nothing waits on an asynchronous command buffer at the call that
40
+ // submitted it, so its failure has no caller to return to. Keep the first
41
+ // one and report it at the next synchronization boundary instead of
42
+ // dropping it. Call with `lock` held.
43
+ private func recordFailureLocked(_ cb: MTLCommandBuffer) {
44
+ guard asyncError == nil, let error = cb.error else { return }
45
+ asyncError = error.localizedDescription
46
+ }
47
+
48
+ private func pruneFinishedLocked() {
49
+ pending.removeAll { cb in
50
+ if cb.status == .error { recordFailureLocked(cb) }
51
+ return cb.status == .completed || cb.status == .error
52
+ }
53
+ }
54
+
55
+ // The first failure of an asynchronous command buffer since the last call.
56
+ func takeAsyncError() -> String? {
57
+ lock.lock()
58
+ defer { lock.unlock() }
59
+ let error = asyncError
60
+ asyncError = nil
61
+ return error
62
+ }
63
+
64
+ // Async dispatches in one Tilework.submit/Graph boundary share a command
65
+ // buffer and compute encoder. setBytes copies scalar payloads at encode
66
+ // time, so callers do not need to retain the Ruby strings after dispatch.
67
+ func encoderForBatch() -> MTLComputeCommandEncoder? {
68
+ lock.lock()
69
+ defer { lock.unlock() }
70
+ if let encoder = batchEncoder { return encoder }
71
+ guard let cb = queue.makeCommandBuffer(),
72
+ let encoder = cb.makeComputeCommandEncoder() else { return nil }
73
+ batchCommandBuffer = cb
74
+ batchEncoder = encoder
75
+ return encoder
76
+ }
77
+
78
+ private func commitBatch() {
79
+ lock.lock()
80
+ guard let cb = batchCommandBuffer, let encoder = batchEncoder else {
81
+ lock.unlock()
82
+ return
83
+ }
84
+ batchCommandBuffer = nil
85
+ batchEncoder = nil
86
+ batchDispatches = 0
87
+ encoder.endEncoding()
88
+ cb.commit()
89
+ pending.append(cb)
90
+ lock.unlock()
91
+ }
92
+
93
+ // Delegated MPS / MPSGraph operations use their own command buffer.
94
+ // Commit any open custom-kernel batch first: an encoded but uncommitted
95
+ // dispatch is not on the queue yet, so a delegated command buffer
96
+ // committed now would run *before* the kernels that produce its inputs.
97
+ // No host wait is needed, because the queue executes committed command
98
+ // buffers in order. A path that touches `contents()` on the host must
99
+ // call `synchronize()` instead.
100
+ func flushBatch() {
101
+ commitBatch()
102
+ }
103
+
104
+ func didEncodeBatchDispatch() {
105
+ lock.lock()
106
+ batchDispatches += 1
107
+ let full = batchDispatches >= batchLimit
108
+ lock.unlock()
109
+ if full { commitBatch() }
110
+ }
111
+
112
+ func synchronize() {
113
+ commitBatch()
114
+ lock.lock()
115
+ let toWait = pending
116
+ pending.removeAll()
117
+ lock.unlock()
118
+ for cb in toWait { cb.waitUntilCompleted() }
119
+ lock.lock()
120
+ for cb in toWait { recordFailureLocked(cb) }
121
+ lock.unlock()
122
+ }
123
+
124
+ // A shared MTLBuffer that was dirtied through `contents()` keeps its
125
+ // system-memory pages resident after its last strong reference goes away.
126
+ // Marking it empty tells Metal that both the resource and its contents can
127
+ // be discarded. It must happen after the last async command buffer that
128
+ // could still use it; the command queue's FIFO ordering makes the newest
129
+ // tracked buffer a sufficient fence for all earlier launches.
130
+ func discard(_ buffer: MTLBuffer) {
131
+ // The buffer may still be referenced by an open async encoder. Commit
132
+ // that batch before deciding whether an ordered purge fence is needed.
133
+ commitBatch()
134
+ lock.lock()
135
+ pruneFinishedLocked()
136
+ let hasPendingWork = !pending.isEmpty
137
+ lock.unlock()
138
+
139
+ guard hasPendingWork else {
140
+ _ = buffer.setPurgeableState(.empty)
141
+ return
142
+ }
143
+
144
+ // Metal requires completion handlers to be registered before commit.
145
+ // This empty command buffer is ordered after every previously
146
+ // submitted command on `queue`, and so is a safe, non-blocking fence.
147
+ guard let fence = queue.makeCommandBuffer() else {
148
+ return
149
+ }
150
+ fence.addCompletedHandler { [buffer] _ in
151
+ autoreleasepool {
152
+ _ = buffer.setPurgeableState(.empty)
153
+ }
154
+ }
155
+ fence.commit()
156
+ track(fence)
157
+ }
158
+ }
159
+
160
+ final class BufferBox {
161
+ let buffer: MTLBuffer
162
+ init(_ buffer: MTLBuffer) { self.buffer = buffer }
163
+ }
164
+
165
+ final class PipelineBox {
166
+ let pso: MTLComputePipelineState
167
+ init(_ pso: MTLComputePipelineState) { self.pso = pso }
168
+ }
169
+
170
+ private func setError(_ out: UnsafeMutablePointer<UnsafeMutablePointer<CChar>?>?, _ message: String) {
171
+ out?.pointee = strdup(message)
172
+ }
173
+
174
+ @_cdecl("gpu_bridge_device_name")
175
+ public func gpu_bridge_device_name() -> UnsafeMutablePointer<CChar>? {
176
+ return strdup(BridgeContext.shared.device.name)
177
+ }
178
+
179
+ @_cdecl("gpu_bridge_device_available")
180
+ public func gpu_bridge_device_available() -> Int32 {
181
+ return MTLCreateSystemDefaultDevice() != nil ? 1 : 0
182
+ }
183
+
184
+ // MARK: - Buffer
185
+
186
+ @_cdecl("gpu_bridge_buffer_alloc")
187
+ public func gpu_bridge_buffer_alloc(_ bytes: Int) -> UnsafeMutableRawPointer? {
188
+ // Ruby invokes this entrypoint directly, without an AppKit/Foundation run
189
+ // loop to drain Objective-C autorelease pools. `makeBuffer` returns an
190
+ // autoreleased Metal object; BufferBox retains it for the lifetime of the
191
+ // Ruby handle, but the original autorelease must still be drained here.
192
+ // Without this pool every allocation keeps an extra MTLBuffer reference
193
+ // until process exit, so explicit Buffer#free cannot reclaim inference
194
+ // weights.
195
+ return autoreleasepool { () -> UnsafeMutableRawPointer? in
196
+ guard let buf = BridgeContext.shared.device.makeBuffer(length: max(bytes, 1),
197
+ options: .storageModeShared) else {
198
+ return nil
199
+ }
200
+ return Unmanaged.passRetained(BufferBox(buf)).toOpaque()
201
+ }
202
+ }
203
+
204
+ // With storageModeShared, buffer_write / buffer_read are **raw memcpys from
205
+ // the host** and never go on the command queue, so calling them while an
206
+ // in-flight kernel touches the same buffer is a data race. Waiting is core's
207
+ // job (Buffer#write / #to_a etc. call Tilework.synchronize) — waiting here
208
+ // every time would drag synchronization into paths made of many small
209
+ // transfers, such as offload.
210
+ @_cdecl("gpu_bridge_buffer_write")
211
+ public func gpu_bridge_buffer_write(_ handle: UnsafeMutableRawPointer,
212
+ _ src: UnsafeRawPointer,
213
+ _ bytes: Int, _ offset: Int) {
214
+ let box = Unmanaged<BufferBox>.fromOpaque(handle).takeUnretainedValue()
215
+ box.buffer.contents().advanced(by: offset).copyMemory(from: src, byteCount: bytes)
216
+ }
217
+
218
+ @_cdecl("gpu_bridge_buffer_read")
219
+ public func gpu_bridge_buffer_read(_ handle: UnsafeMutableRawPointer,
220
+ _ dst: UnsafeMutableRawPointer,
221
+ _ bytes: Int, _ offset: Int) {
222
+ let box = Unmanaged<BufferBox>.fromOpaque(handle).takeUnretainedValue()
223
+ dst.copyMemory(from: box.buffer.contents().advanced(by: offset), byteCount: bytes)
224
+ }
225
+
226
+ @_cdecl("gpu_bridge_buffer_free")
227
+ public func gpu_bridge_buffer_free(_ handle: UnsafeMutableRawPointer) {
228
+ let box = Unmanaged<BufferBox>.fromOpaque(handle).takeUnretainedValue()
229
+ BridgeContext.shared.discard(box.buffer)
230
+ Unmanaged<BufferBox>.fromOpaque(handle).release()
231
+ }
232
+
233
+ // MARK: - Pipeline
234
+
235
+ @_cdecl("gpu_bridge_pipeline_create")
236
+ public func gpu_bridge_pipeline_create(_ source: UnsafePointer<CChar>,
237
+ _ fnName: UnsafePointer<CChar>,
238
+ _ errorOut: UnsafeMutablePointer<UnsafeMutablePointer<CChar>?>?)
239
+ -> UnsafeMutableRawPointer? {
240
+ // Like buffers, pipeline creation can be called repeatedly from Ruby with
241
+ // no ambient autorelease pool. PipelineBox owns the compiled state; this
242
+ // pool only releases the temporary library/function objects.
243
+ return autoreleasepool { () -> UnsafeMutableRawPointer? in
244
+ let ctx = BridgeContext.shared
245
+ let src = String(cString: source)
246
+ let name = String(cString: fnName)
247
+ do {
248
+ let library = try ctx.device.makeLibrary(source: src, options: nil)
249
+ guard let fn = library.makeFunction(name: name) else {
250
+ setError(errorOut, "kernel function '\(name)' not found in compiled library")
251
+ return nil
252
+ }
253
+ let pso = try ctx.device.makeComputePipelineState(function: fn)
254
+ return Unmanaged.passRetained(PipelineBox(pso)).toOpaque()
255
+ } catch {
256
+ setError(errorOut, error.localizedDescription)
257
+ return nil
258
+ }
259
+ }
260
+ }
261
+
262
+ @_cdecl("gpu_bridge_pipeline_max_threads")
263
+ public func gpu_bridge_pipeline_max_threads(_ handle: UnsafeMutableRawPointer) -> Int {
264
+ let box = Unmanaged<PipelineBox>.fromOpaque(handle).takeUnretainedValue()
265
+ return box.pso.maxTotalThreadsPerThreadgroup
266
+ }
267
+
268
+ @_cdecl("gpu_bridge_pipeline_free")
269
+ public func gpu_bridge_pipeline_free(_ handle: UnsafeMutableRawPointer) {
270
+ Unmanaged<PipelineBox>.fromOpaque(handle).release()
271
+ }
272
+
273
+ // MARK: - GEMM (MPSMatrixMultiplication)
274
+ //
275
+ // The two GEMM entrypoints have *different* buffer-layout contracts (see
276
+ // bridge.cpp, the CUDA reference impl, and tensor_core.rb):
277
+ //
278
+ // - `gemm` (plain, below): a/b/c are Tilework::Buffer's native row-major
279
+ // layout. MPS is natively row-major too, so this needs no reinterpretation
280
+ // — read A/B/C directly as row-major matrices.
281
+ //
282
+ // - `gemm_strided_batched` (encodeGemm, this section): a/b/c are genuinely
283
+ // column-major (lda/ldb/ldc/trans_a/trans_b are the caller's cuBLAS-style
284
+ // column-major description; NN.multi_head_attention builds these itself).
285
+ // Since MPS is row-major, we read each column-major buffer directly as
286
+ // its own row-major transpose instead of copying/transposing — a col-
287
+ // major (r, c) buffer with leading dimension ld is bit-identical to a
288
+ // row-major (c, r) buffer with rowBytes = ld * elemSize. Combined with
289
+ // cuBLAS's own "swap A/B, swap m/n, keep trans flags" identity for
290
+ // row-major libraries, this lets the call be encoded with zero extra
291
+ // data movement:
292
+ //
293
+ // leftMatrix = rowview(B), transpose = trans_b (represents op(B)^T)
294
+ // rightMatrix = rowview(A), transpose = trans_a (represents op(A)^T)
295
+ // result = rowview(C) (represents C^T, i.e. C's
296
+ // raw bytes read row-major)
297
+ //
298
+ // See core/lib/tilework/math/tensor_core.rb for the full ABI contract.
299
+
300
+ private func mpsDataType(_ code: Int32) -> MPSDataType {
301
+ if code == 1 { return .float16 }
302
+ if code == 2 { return .bFloat16 }
303
+ return .float32
304
+ }
305
+
306
+ private func elemSize(_ code: Int32) -> Int {
307
+ return code == 0 ? 4 : 2
308
+ }
309
+
310
+ // `handle` is always the base BufferBox pointer returned by buffer_alloc —
311
+ // never offset (see gpu_bridge_dispatch's comment in gpu_bridge.c: CUDA does
312
+ // pointer arithmetic, Metal takes the offset separately). Views pass their
313
+ // byte offset in `offset`, applied here via MPSMatrix's own offset param
314
+ // rather than by shifting the opaque handle, which is not a raw device
315
+ // pointer and cannot be offset like one.
316
+ private func mpsMatrix(_ handle: UnsafeMutableRawPointer, offset: Int, rows: Int, columns: Int,
317
+ rowBytes: Int, matrices: Int, matrixBytes: Int,
318
+ dataType: MPSDataType) -> MPSMatrix {
319
+ let box = Unmanaged<BufferBox>.fromOpaque(handle).takeUnretainedValue()
320
+ let desc = MPSMatrixDescriptor(rows: rows, columns: columns, matrices: matrices,
321
+ rowBytes: rowBytes, matrixBytes: matrixBytes,
322
+ dataType: dataType)
323
+ return MPSMatrix(buffer: box.buffer, offset: offset, descriptor: desc)
324
+ }
325
+
326
+ private func encodeGemm(_ a: UnsafeMutableRawPointer, _ aOff: Int,
327
+ _ b: UnsafeMutableRawPointer, _ bOff: Int,
328
+ _ c: UnsafeMutableRawPointer, _ cOff: Int,
329
+ m: Int, k: Int, n: Int,
330
+ lda: Int, ldb: Int, ldc: Int,
331
+ strideA: Int, strideB: Int, strideC: Int,
332
+ transA: Bool, transB: Bool,
333
+ batch: Int, alpha: Double, inDtype: Int32,
334
+ errorOut: UnsafeMutablePointer<UnsafeMutablePointer<CChar>?>?) -> Int32 {
335
+ let ctx = BridgeContext.shared
336
+ ctx.flushBatch()
337
+ let inType = mpsDataType(inDtype)
338
+ let inElem = elemSize(inDtype)
339
+ let outElem = 4 // ABI contract: the output is always float32
340
+
341
+ // Raw dimensions in BLAS convention (before transposes are applied).
342
+ let aRawRows = transA ? k : m
343
+ let aRawCols = transA ? m : k
344
+ let bRawRows = transB ? n : k
345
+ let bRawCols = transB ? k : n
346
+
347
+ let leftMat = mpsMatrix(b, offset: bOff, rows: bRawCols, columns: bRawRows,
348
+ rowBytes: ldb * inElem, matrices: batch,
349
+ matrixBytes: strideB * inElem, dataType: inType)
350
+ let rightMat = mpsMatrix(a, offset: aOff, rows: aRawCols, columns: aRawRows,
351
+ rowBytes: lda * inElem, matrices: batch,
352
+ matrixBytes: strideA * inElem, dataType: inType)
353
+ let resultMat = mpsMatrix(c, offset: cOff, rows: n, columns: m,
354
+ rowBytes: ldc * outElem, matrices: batch,
355
+ matrixBytes: strideC * outElem, dataType: .float32)
356
+
357
+ let mul = MPSMatrixMultiplication(device: ctx.device,
358
+ transposeLeft: transB, transposeRight: transA,
359
+ resultRows: n, resultColumns: m, interiorColumns: k,
360
+ alpha: alpha, beta: 0.0)
361
+
362
+ guard let cb = ctx.queue.makeCommandBuffer() else {
363
+ setError(errorOut, "failed to create command buffer for gemm")
364
+ return -1
365
+ }
366
+ mul.encode(commandBuffer: cb, leftMatrix: leftMat, rightMatrix: rightMat, resultMatrix: resultMat)
367
+ cb.commit()
368
+ ctx.track(cb)
369
+ return 0
370
+ }
371
+
372
+ // MARK: - GEMM: tiled simdgroup_matrix kernel (Phase 2, fp32 only)
373
+ //
374
+ // MPS (above) covers every shape correctly, but for large aligned fp32
375
+ // matmuls a hand-tiled kernel using simdgroup_float8x8 fragments can beat it
376
+ // by reusing each global-memory read across a whole threadgroup tile instead
377
+ // of paying MPS's more general dispatch overhead. This kernel deliberately
378
+ // mirrors the existing WMMA path's own tradeoff (see TensorCore's BLOCK_M/N/K
379
+ // comments in tensor_core.rb): it only accepts shapes that tile evenly, and
380
+ // anything else falls straight back to MPS — no partial-tile bounds checks,
381
+ // no attempt to be a general-purpose kernel.
382
+ //
383
+ // One simdgroup (32 threads) computes one 32x32 output tile as a 4x4 grid of
384
+ // 8x8 fragments, staging a 32x32 slice of A and B through threadgroup memory
385
+ // per K-step so the 32x32=1024 global loads are shared across all 16
386
+ // fragment multiplies that use them, instead of re-reading per fragment.
387
+ private let simdgroupGemmTile = 32
388
+ private let simdgroupGemmFrag = 8
389
+
390
+ private let simdgroupGemmSource = """
391
+ #include <metal_stdlib>
392
+ using namespace metal;
393
+
394
+ constant int TILE = \(simdgroupGemmTile);
395
+ constant int FRAG = \(simdgroupGemmFrag);
396
+ constant int FRAGS = TILE / FRAG;
397
+
398
+ kernel void gemm_simdgroup_f32(
399
+ device const float* A [[buffer(0)]],
400
+ device const float* B [[buffer(1)]],
401
+ device float* C [[buffer(2)]],
402
+ constant int& M [[buffer(3)]],
403
+ constant int& K [[buffer(4)]],
404
+ constant int& N [[buffer(5)]],
405
+ uint2 tgid [[threadgroup_position_in_grid]],
406
+ uint lane [[thread_index_in_simdgroup]])
407
+ {
408
+ threadgroup float As[TILE][TILE];
409
+ threadgroup float Bs[TILE][TILE];
410
+
411
+ int tileRow = int(tgid.y) * TILE;
412
+ int tileCol = int(tgid.x) * TILE;
413
+
414
+ simdgroup_float8x8 acc[FRAGS][FRAGS];
415
+ for (int i = 0; i < FRAGS; i++)
416
+ for (int j = 0; j < FRAGS; j++)
417
+ acc[i][j] = simdgroup_float8x8(0);
418
+
419
+ // Caller (gpu_bridge_gemm) only takes this path when M/N/K are all
420
+ // multiples of TILE, so every tile below is fully in-bounds.
421
+ for (int k0 = 0; k0 < K; k0 += TILE) {
422
+ for (int e = int(lane); e < TILE * TILE; e += 32) {
423
+ int r = e / TILE, c = e % TILE;
424
+ As[r][c] = A[(tileRow + r) * K + (k0 + c)];
425
+ Bs[r][c] = B[(k0 + r) * N + (tileCol + c)];
426
+ }
427
+ threadgroup_barrier(mem_flags::mem_threadgroup);
428
+
429
+ for (int kk = 0; kk < TILE; kk += FRAG) {
430
+ simdgroup_float8x8 aFrag[FRAGS], bFrag[FRAGS];
431
+ for (int i = 0; i < FRAGS; i++)
432
+ simdgroup_load(aFrag[i], &As[i * FRAG][kk], TILE);
433
+ for (int j = 0; j < FRAGS; j++)
434
+ simdgroup_load(bFrag[j], &Bs[kk][j * FRAG], TILE);
435
+ for (int i = 0; i < FRAGS; i++)
436
+ for (int j = 0; j < FRAGS; j++)
437
+ simdgroup_multiply_accumulate(acc[i][j], aFrag[i], bFrag[j], acc[i][j]);
438
+ }
439
+ threadgroup_barrier(mem_flags::mem_threadgroup);
440
+ }
441
+
442
+ for (int i = 0; i < FRAGS; i++)
443
+ for (int j = 0; j < FRAGS; j++)
444
+ simdgroup_store(acc[i][j], &C[(tileRow + i * FRAG) * N + (tileCol + j * FRAG)], N);
445
+ }
446
+ """
447
+
448
+ private final class SimdgroupGemmKernel {
449
+ let pso: MTLComputePipelineState
450
+ private init(pso: MTLComputePipelineState) { self.pso = pso }
451
+
452
+ // Compiled once and cached. Where compilation itself fails (unsupported GPU
453
+ // family etc.) it is nil and callers quietly fall back to MPS — it is an
454
+ // optimization for supported GPUs only, so there is no reason to error.
455
+ static let shared: SimdgroupGemmKernel? = {
456
+ let ctx = BridgeContext.shared
457
+ guard let library = try? ctx.device.makeLibrary(source: simdgroupGemmSource, options: nil),
458
+ let fn = library.makeFunction(name: "gemm_simdgroup_f32"),
459
+ let pso = try? ctx.device.makeComputePipelineState(function: fn) else {
460
+ return nil
461
+ }
462
+ return SimdgroupGemmKernel(pso: pso)
463
+ }()
464
+ }
465
+
466
+ // Measured (M4-class, fp32, square 256/512/1024) at only 0.2–0.5x of MPS on
467
+ // every size (no double buffering or register blocking). The code is kept as
468
+ // a fallback for when MPS is unavailable, with the threshold at infinity so it
469
+ // is never chosen. Restore a measured value if multi-simdgroup tiling and
470
+ // double buffering bring it close to MPS.
471
+ private let simdgroupGemmMinElements = Int.max
472
+
473
+ private func encodeSimdgroupGemm(_ kernel: SimdgroupGemmKernel,
474
+ _ a: UnsafeMutableRawPointer, _ b: UnsafeMutableRawPointer,
475
+ _ c: UnsafeMutableRawPointer,
476
+ m: Int, k: Int, n: Int,
477
+ errorOut: UnsafeMutablePointer<UnsafeMutablePointer<CChar>?>?) -> Int32 {
478
+ let ctx = BridgeContext.shared
479
+ ctx.flushBatch()
480
+ let boxA = Unmanaged<BufferBox>.fromOpaque(a).takeUnretainedValue()
481
+ let boxB = Unmanaged<BufferBox>.fromOpaque(b).takeUnretainedValue()
482
+ let boxC = Unmanaged<BufferBox>.fromOpaque(c).takeUnretainedValue()
483
+
484
+ guard let cb = ctx.queue.makeCommandBuffer(),
485
+ let encoder = cb.makeComputeCommandEncoder() else {
486
+ setError(errorOut, "failed to create command buffer/encoder for simdgroup gemm")
487
+ return -1
488
+ }
489
+ var m32 = Int32(m), k32 = Int32(k), n32 = Int32(n)
490
+ encoder.setComputePipelineState(kernel.pso)
491
+ encoder.setBuffer(boxA.buffer, offset: 0, index: 0)
492
+ encoder.setBuffer(boxB.buffer, offset: 0, index: 1)
493
+ encoder.setBuffer(boxC.buffer, offset: 0, index: 2)
494
+ encoder.setBytes(&m32, length: 4, index: 3)
495
+ encoder.setBytes(&k32, length: 4, index: 4)
496
+ encoder.setBytes(&n32, length: 4, index: 5)
497
+ encoder.dispatchThreadgroups(MTLSize(width: n / simdgroupGemmTile, height: m / simdgroupGemmTile, depth: 1),
498
+ threadsPerThreadgroup: MTLSize(width: 32, height: 1, depth: 1))
499
+ encoder.endEncoding()
500
+ cb.commit()
501
+ ctx.track(cb)
502
+ return 0
503
+ }
504
+
505
+ // Plain GEMM: a/b/c are row-major (see the GEMM section above).
506
+ @_cdecl("gpu_bridge_gemm")
507
+ public func gpu_bridge_gemm(_ a: UnsafeMutableRawPointer, _ aOff: Int,
508
+ _ b: UnsafeMutableRawPointer, _ bOff: Int,
509
+ _ c: UnsafeMutableRawPointer, _ cOff: Int,
510
+ _ m: Int, _ k: Int, _ n: Int,
511
+ _ allowTF32: Int32, _ inDtype: Int32,
512
+ _ errorOut: UnsafeMutablePointer<UnsafeMutablePointer<CChar>?>?) -> Int32 {
513
+ return autoreleasepool { () -> Int32 in
514
+ // allowTF32 is CUDA-specific (TensorFloat-32) and meaningless on Metal, so it is ignored.
515
+ // The hand-tiled kernel does not accept views (offset != 0), so any call
516
+ // with an offset goes to MPS unconditionally.
517
+ let tileAligned = m % simdgroupGemmTile == 0 && n % simdgroupGemmTile == 0 && k % simdgroupGemmTile == 0
518
+ if inDtype == 0, tileAligned, aOff == 0, bOff == 0, cOff == 0,
519
+ m * n * k >= simdgroupGemmMinElements,
520
+ let kernel = SimdgroupGemmKernel.shared {
521
+ return encodeSimdgroupGemm(kernel, a, b, c, m: m, k: k, n: n, errorOut: errorOut)
522
+ }
523
+
524
+ let ctx = BridgeContext.shared
525
+ ctx.flushBatch()
526
+ let inType = mpsDataType(inDtype)
527
+ let inElem = elemSize(inDtype)
528
+
529
+ let matA = mpsMatrix(a, offset: aOff, rows: m, columns: k, rowBytes: k * inElem,
530
+ matrices: 1, matrixBytes: m * k * inElem, dataType: inType)
531
+ let matB = mpsMatrix(b, offset: bOff, rows: k, columns: n, rowBytes: n * inElem,
532
+ matrices: 1, matrixBytes: k * n * inElem, dataType: inType)
533
+ let matC = mpsMatrix(c, offset: cOff, rows: m, columns: n, rowBytes: n * 4,
534
+ matrices: 1, matrixBytes: m * n * 4, dataType: .float32)
535
+
536
+ let mul = MPSMatrixMultiplication(device: ctx.device,
537
+ transposeLeft: false, transposeRight: false,
538
+ resultRows: m, resultColumns: n, interiorColumns: k,
539
+ alpha: 1.0, beta: 0.0)
540
+
541
+ guard let cb = ctx.queue.makeCommandBuffer() else {
542
+ setError(errorOut, "failed to create command buffer for gemm")
543
+ return -1
544
+ }
545
+ mul.encode(commandBuffer: cb, leftMatrix: matA, rightMatrix: matB, resultMatrix: matC)
546
+ cb.commit()
547
+ ctx.track(cb)
548
+ return 0
549
+ }
550
+ }
551
+
552
+ @_cdecl("gpu_bridge_gemm_strided_batched")
553
+ public func gpu_bridge_gemm_strided_batched(_ a: UnsafeMutableRawPointer, _ aOff: Int,
554
+ _ b: UnsafeMutableRawPointer, _ bOff: Int,
555
+ _ c: UnsafeMutableRawPointer, _ cOff: Int,
556
+ _ m: Int, _ k: Int, _ n: Int,
557
+ _ lda: Int, _ ldb: Int, _ ldc: Int,
558
+ _ strideA: Int, _ strideB: Int, _ strideC: Int,
559
+ _ transA: Int32, _ transB: Int32,
560
+ _ batch: Int, _ alpha: Double,
561
+ _ allowTF32: Int32, _ inDtype: Int32,
562
+ _ errorOut: UnsafeMutablePointer<UnsafeMutablePointer<CChar>?>?) -> Int32 {
563
+ return autoreleasepool { () -> Int32 in
564
+ return encodeGemm(a, aOff, b, bOff, c, cOff, m: m, k: k, n: n,
565
+ lda: lda, ldb: ldb, ldc: ldc,
566
+ strideA: strideA, strideB: strideB, strideC: strideC,
567
+ transA: transA != 0, transB: transB != 0,
568
+ batch: batch, alpha: alpha, inDtype: inDtype,
569
+ errorOut: errorOut)
570
+ }
571
+ }
572
+
573
+ // MARK: - Random (MPSMatrixRandomPhilox, cuRAND analogue)
574
+ //
575
+ // See random.rb for kind/mean/stddev. UNIFORM = 0, NORMAL = 1.
576
+ // cuRAND's convention makes uniform **(0, 1]** (excluding 0, including 1),
577
+ // but MPS's uniformDistributionDescriptor is [minimum, maximum) — the open end
578
+ // is on the other side, so the result is flipped with 1.0 - x afterwards.
579
+
580
+ private let RANDOM_KIND_UNIFORM: Int32 = 0
581
+
582
+ @_cdecl("gpu_bridge_random")
583
+ public func gpu_bridge_random(_ handle: UnsafeMutableRawPointer, _ n: Int, _ kind: Int32,
584
+ _ seed: UInt64, _ mean: Float, _ stddev: Float,
585
+ _ errorOut: UnsafeMutablePointer<UnsafeMutablePointer<CChar>?>?) -> Int32 {
586
+ return autoreleasepool { () -> Int32 in
587
+ let ctx = BridgeContext.shared
588
+ let box = Unmanaged<BufferBox>.fromOpaque(handle).takeUnretainedValue()
589
+
590
+ let desc: MPSMatrixRandomDistributionDescriptor
591
+ if kind == RANDOM_KIND_UNIFORM {
592
+ desc = MPSMatrixRandomDistributionDescriptor.uniformDistributionDescriptor(withMinimum: 0.0, maximum: 1.0)
593
+ } else {
594
+ desc = MPSMatrixRandomDistributionDescriptor.normalDistributionDescriptor(withMean: mean, standardDeviation: stddev)
595
+ }
596
+
597
+ let rng = MPSMatrixRandomPhilox(device: ctx.device, destinationDataType: .float32,
598
+ seed: Int(bitPattern: UInt(truncatingIfNeeded: seed)),
599
+ distributionDescriptor: desc)
600
+
601
+ // MPSMatrixRandomPhilox emits four elements at a time. A count that is
602
+ // not a multiple of four **kills the process with an assert** (not an
603
+ // exception, so Ruby cannot catch it). Generate into a scratch buffer
604
+ // rounded up to a multiple of four and copy back the first n elements.
605
+ let padded = (n + 3) & ~3
606
+ let scratch: MTLBuffer?
607
+ if padded == n {
608
+ scratch = nil
609
+ } else {
610
+ guard let buf = ctx.device.makeBuffer(length: padded * 4, options: .storageModeShared) else {
611
+ setError(errorOut, "failed to allocate \(padded * 4) bytes for random")
612
+ return -1
613
+ }
614
+ scratch = buf
615
+ }
616
+ let dest = scratch ?? box.buffer
617
+
618
+ let matDesc = MPSMatrixDescriptor(rows: 1, columns: padded, rowBytes: padded * 4, dataType: .float32)
619
+ let matrix = MPSMatrix(buffer: dest, descriptor: matDesc)
620
+
621
+ guard let cb = ctx.queue.makeCommandBuffer() else {
622
+ setError(errorOut, "failed to create command buffer for random")
623
+ return -1
624
+ }
625
+ ctx.flushBatch()
626
+ rng.encode(commandBuffer: cb, destinationMatrix: matrix)
627
+ cb.commit()
628
+ cb.waitUntilCompleted()
629
+ if let error = cb.error {
630
+ setError(errorOut, error.localizedDescription)
631
+ return -1
632
+ }
633
+
634
+ if let scratch = scratch {
635
+ memcpy(box.buffer.contents(), scratch.contents(), n * 4)
636
+ }
637
+
638
+ if kind == RANDOM_KIND_UNIFORM {
639
+ let ptr = box.buffer.contents().bindMemory(to: Float.self, capacity: n)
640
+ for i in 0..<n { ptr[i] = 1.0 - ptr[i] }
641
+ }
642
+
643
+ return 0
644
+ }
645
+ }
646
+
647
+ // MARK: - FFT (MPSGraph, cuFFT analogue)
648
+ //
649
+ // See fft.rb for kind/direction. R2C = 0, C2R = 1, C2C = 2, FORWARD = 1,
650
+ // INVERSE = -1. Normalization (1/n) is fft.rb's job, so this always uses
651
+ // MPSGraphFFTScalingModeNone (no scaling).
652
+ //
653
+ // Complex numbers are interleaved float32 ([re0, im0, re1, im1, ...]) per the
654
+ // core contract. MPSDataTypeComplexFloat32 is a pair of 2x32-bit (re, im), so
655
+ // a Buffer's raw bytes can be read as complexFloat32 directly — no copy.
656
+ //
657
+ // Graph compilation is expensive, so graphs are cached per shape key (kind,
658
+ // direction, n, batch), the same idea as cuFFT's plan cache.
659
+
660
+ private struct FFTPlanKey: Hashable {
661
+ let kind: Int32
662
+ let direction: Int32
663
+ let n: Int
664
+ let batch: Int
665
+ }
666
+
667
+ private final class FFTPlan {
668
+ let graph: MPSGraph
669
+ let inputTensor: MPSGraphTensor
670
+ let outputTensor: MPSGraphTensor
671
+ let inputShape: [NSNumber]
672
+ let outputShape: [NSNumber]
673
+ let inputDataType: MPSDataType
674
+ let outputDataType: MPSDataType
675
+
676
+ init(graph: MPSGraph, inputTensor: MPSGraphTensor, outputTensor: MPSGraphTensor,
677
+ inputShape: [NSNumber], outputShape: [NSNumber],
678
+ inputDataType: MPSDataType, outputDataType: MPSDataType) {
679
+ self.graph = graph
680
+ self.inputTensor = inputTensor
681
+ self.outputTensor = outputTensor
682
+ self.inputShape = inputShape
683
+ self.outputShape = outputShape
684
+ self.inputDataType = inputDataType
685
+ self.outputDataType = outputDataType
686
+ }
687
+ }
688
+
689
+ private let FFT_KIND_R2C: Int32 = 0
690
+ private let FFT_KIND_C2R: Int32 = 1
691
+ private let FFT_DIR_INVERSE: Int32 = -1
692
+
693
+ private var fftPlanCache: [FFTPlanKey: FFTPlan] = [:]
694
+ private let fftPlanLock = NSLock()
695
+
696
+ private func fftPlan(kind: Int32, direction: Int32, n: Int, batch: Int) -> FFTPlan {
697
+ let key = FFTPlanKey(kind: kind, direction: direction, n: n, batch: batch)
698
+
699
+ fftPlanLock.lock()
700
+ defer { fftPlanLock.unlock() }
701
+ if let cached = fftPlanCache[key] { return cached }
702
+
703
+ let graph = MPSGraph()
704
+ let desc = MPSGraphFFTDescriptor()
705
+ desc.scalingMode = .none
706
+ desc.inverse = direction == FFT_DIR_INVERSE
707
+
708
+ let plan: FFTPlan
709
+ switch kind {
710
+ case FFT_KIND_R2C:
711
+ let inShape: [NSNumber] = [NSNumber(value: batch), NSNumber(value: n)]
712
+ let input = graph.placeholder(shape: inShape, dataType: .float32, name: nil)
713
+ let output = graph.realToHermiteanFFT(input, axes: [1], descriptor: desc, name: nil)
714
+ let outShape: [NSNumber] = [NSNumber(value: batch), NSNumber(value: n / 2 + 1)]
715
+ plan = FFTPlan(graph: graph, inputTensor: input, outputTensor: output,
716
+ inputShape: inShape, outputShape: outShape,
717
+ inputDataType: .float32, outputDataType: .complexFloat32)
718
+ case FFT_KIND_C2R:
719
+ let inSize = n / 2 + 1
720
+ let inShape: [NSNumber] = [NSNumber(value: batch), NSNumber(value: inSize)]
721
+ desc.roundToOddHermitean = n % 2 != 0
722
+ let input = graph.placeholder(shape: inShape, dataType: .complexFloat32, name: nil)
723
+ let output = graph.HermiteanToRealFFT(input, axes: [1], descriptor: desc, name: nil)
724
+ let outShape: [NSNumber] = [NSNumber(value: batch), NSNumber(value: n)]
725
+ plan = FFTPlan(graph: graph, inputTensor: input, outputTensor: output,
726
+ inputShape: inShape, outputShape: outShape,
727
+ inputDataType: .complexFloat32, outputDataType: .float32)
728
+ default: // C2C
729
+ let inShape: [NSNumber] = [NSNumber(value: batch), NSNumber(value: n)]
730
+ let input = graph.placeholder(shape: inShape, dataType: .complexFloat32, name: nil)
731
+ let output = graph.fastFourierTransform(input, axes: [1], descriptor: desc, name: nil)
732
+ plan = FFTPlan(graph: graph, inputTensor: input, outputTensor: output,
733
+ inputShape: inShape, outputShape: inShape,
734
+ inputDataType: .complexFloat32, outputDataType: .complexFloat32)
735
+ }
736
+
737
+ fftPlanCache[key] = plan
738
+ return plan
739
+ }
740
+
741
+ @_cdecl("gpu_bridge_fft")
742
+ public func gpu_bridge_fft(_ inHandle: UnsafeMutableRawPointer, _ outHandle: UnsafeMutableRawPointer,
743
+ _ n: Int, _ batch: Int, _ kind: Int32, _ direction: Int32,
744
+ _ errorOut: UnsafeMutablePointer<UnsafeMutablePointer<CChar>?>?) -> Int32 {
745
+ return autoreleasepool { () -> Int32 in
746
+ let ctx = BridgeContext.shared
747
+ let inBox = Unmanaged<BufferBox>.fromOpaque(inHandle).takeUnretainedValue()
748
+ let outBox = Unmanaged<BufferBox>.fromOpaque(outHandle).takeUnretainedValue()
749
+
750
+ ctx.flushBatch()
751
+ let plan = fftPlan(kind: kind, direction: direction, n: n, batch: batch)
752
+ let inputData = MPSGraphTensorData(inBox.buffer, shape: plan.inputShape, dataType: plan.inputDataType)
753
+ let outputData = MPSGraphTensorData(outBox.buffer, shape: plan.outputShape, dataType: plan.outputDataType)
754
+
755
+ plan.graph.run(with: ctx.queue, feeds: [plan.inputTensor: inputData],
756
+ targetOperations: nil,
757
+ resultsDictionary: [plan.outputTensor: outputData])
758
+
759
+ return 0
760
+ }
761
+ }
762
+
763
+ // MARK: - Conv2D (MPSGraph, cuDNN analogue)
764
+ //
765
+ // The layout is NCHW / weights OIHW per the core contract (see conv.rb). dtype
766
+ // uses the same codes as gemm (0 = float32, 1 = float16, 2 = bfloat16), so
767
+ // mpsDataType(_:) is reused. Unlike gemm, the output dtype follows the input
768
+ // (accumulation precision is left to MPSGraph's implementation).
769
+ //
770
+ // For mode see gpu_bridge_conv2d_backward's calling convention (conv.rb):
771
+ // 0 = input gradient (a=weights, b=grad_output), 1 = weight gradient
772
+ // (a=input, b=grad_output). All three are (tensorA, tensorB) -> output with
773
+ // two inputs and one output, so one ConvPlan struct represents them.
774
+
775
+ private let CONV_OP_FORWARD: Int32 = 0
776
+ private let CONV_OP_BACKWARD_DATA: Int32 = 1
777
+ private let CONV_OP_BACKWARD_FILTER: Int32 = 2
778
+
779
+ private struct ConvPlanKey: Hashable {
780
+ let op: Int32
781
+ let n: Int32, cin: Int32, h: Int32, w: Int32
782
+ let cout: Int32, kh: Int32, kw: Int32
783
+ let padH: Int32, padW: Int32, strideH: Int32, strideW: Int32
784
+ let dilationH: Int32, dilationW: Int32, groups: Int32
785
+ let dtype: Int32
786
+ }
787
+
788
+ private final class ConvPlan {
789
+ let graph: MPSGraph
790
+ let tensorA: MPSGraphTensor
791
+ let tensorB: MPSGraphTensor
792
+ let output: MPSGraphTensor
793
+ let shapeA: [NSNumber]
794
+ let shapeB: [NSNumber]
795
+ let outputShape: [NSNumber]
796
+ let dataType: MPSDataType
797
+
798
+ init(graph: MPSGraph, tensorA: MPSGraphTensor, tensorB: MPSGraphTensor, output: MPSGraphTensor,
799
+ shapeA: [NSNumber], shapeB: [NSNumber], outputShape: [NSNumber], dataType: MPSDataType) {
800
+ self.graph = graph
801
+ self.tensorA = tensorA
802
+ self.tensorB = tensorB
803
+ self.output = output
804
+ self.shapeA = shapeA
805
+ self.shapeB = shapeB
806
+ self.outputShape = outputShape
807
+ self.dataType = dataType
808
+ }
809
+ }
810
+
811
+ private var convPlanCache: [ConvPlanKey: ConvPlan] = [:]
812
+ private let convPlanLock = NSLock()
813
+
814
+ // The Ruby shim registers this as an end proc. MPSGraph plans are held by
815
+ // process-global dictionaries rather than Ruby TypedData, so release them
816
+ // explicitly after queued work has drained. Avoid touching BridgeContext when
817
+ // neither cache was ever populated; shutdown must not initialize Metal merely
818
+ // because the extension was required.
819
+ @_cdecl("gpu_bridge_shutdown")
820
+ public func gpu_bridge_shutdown() {
821
+ fftPlanLock.lock()
822
+ let hasFFTPlans = !fftPlanCache.isEmpty
823
+ fftPlanLock.unlock()
824
+ convPlanLock.lock()
825
+ let hasConvPlans = !convPlanCache.isEmpty
826
+ convPlanLock.unlock()
827
+ guard hasFFTPlans || hasConvPlans else { return }
828
+
829
+ BridgeContext.shared.synchronize()
830
+ autoreleasepool {
831
+ fftPlanLock.lock()
832
+ fftPlanCache.removeAll()
833
+ fftPlanLock.unlock()
834
+ convPlanLock.lock()
835
+ convPlanCache.removeAll()
836
+ convPlanLock.unlock()
837
+ }
838
+ }
839
+
840
+ private func convPlan(op: Int32, n: Int32, cin: Int32, h: Int32, w: Int32,
841
+ cout: Int32, kh: Int32, kw: Int32,
842
+ padH: Int32, padW: Int32, strideH: Int32, strideW: Int32,
843
+ dilationH: Int32, dilationW: Int32, groups: Int32, dtype: Int32) -> ConvPlan {
844
+ let key = ConvPlanKey(op: op, n: n, cin: cin, h: h, w: w, cout: cout, kh: kh, kw: kw,
845
+ padH: padH, padW: padW, strideH: strideH, strideW: strideW,
846
+ dilationH: dilationH, dilationW: dilationW, groups: groups, dtype: dtype)
847
+
848
+ convPlanLock.lock()
849
+ defer { convPlanLock.unlock() }
850
+ if let cached = convPlanCache[key] { return cached }
851
+
852
+ let graph = MPSGraph()
853
+ let mpsType = mpsDataType(dtype)
854
+ let effectiveKh = (Int(kh) - 1) * Int(dilationH) + 1
855
+ let effectiveKw = (Int(kw) - 1) * Int(dilationW) + 1
856
+ let oh = (Int(h) + 2 * Int(padH) - effectiveKh) / Int(strideH) + 1
857
+ let ow = (Int(w) + 2 * Int(padW) - effectiveKw) / Int(strideW) + 1
858
+
859
+ let desc = MPSGraphConvolution2DOpDescriptor(strideInX: Int(strideW), strideInY: Int(strideH),
860
+ dilationRateInX: Int(dilationW), dilationRateInY: Int(dilationH),
861
+ groups: Int(groups),
862
+ paddingLeft: Int(padW), paddingRight: Int(padW),
863
+ paddingTop: Int(padH), paddingBottom: Int(padH),
864
+ paddingStyle: .explicit,
865
+ dataLayout: .NCHW, weightsLayout: .OIHW)!
866
+
867
+ let cinPerGroup = cin / groups
868
+ let srcShape: [NSNumber] = [n, cin, h, w].map { NSNumber(value: $0) }
869
+ let weightsShape: [NSNumber] = [cout, cinPerGroup, kh, kw].map { NSNumber(value: $0) }
870
+ let outShape: [NSNumber] = [n, cout, Int32(oh), Int32(ow)].map { NSNumber(value: $0) }
871
+
872
+ let plan: ConvPlan
873
+ switch op {
874
+ case CONV_OP_BACKWARD_DATA:
875
+ let weights = graph.placeholder(shape: weightsShape, dataType: mpsType, name: nil)
876
+ let gradOutput = graph.placeholder(shape: outShape, dataType: mpsType, name: nil)
877
+ let dx = graph.convolution2DDataGradient(gradOutput, weights: weights, outputShape: srcShape,
878
+ forwardConvolutionDescriptor: desc, name: nil)
879
+ plan = ConvPlan(graph: graph, tensorA: weights, tensorB: gradOutput, output: dx,
880
+ shapeA: weightsShape, shapeB: outShape, outputShape: srcShape, dataType: mpsType)
881
+ case CONV_OP_BACKWARD_FILTER:
882
+ let src = graph.placeholder(shape: srcShape, dataType: mpsType, name: nil)
883
+ let gradOutput = graph.placeholder(shape: outShape, dataType: mpsType, name: nil)
884
+ let dw = graph.convolution2DWeightsGradient(gradOutput, source: src, outputShape: weightsShape,
885
+ forwardConvolutionDescriptor: desc, name: nil)
886
+ plan = ConvPlan(graph: graph, tensorA: src, tensorB: gradOutput, output: dw,
887
+ shapeA: srcShape, shapeB: outShape, outputShape: weightsShape, dataType: mpsType)
888
+ default: // CONV_OP_FORWARD
889
+ let src = graph.placeholder(shape: srcShape, dataType: mpsType, name: nil)
890
+ let weights = graph.placeholder(shape: weightsShape, dataType: mpsType, name: nil)
891
+ let out = graph.convolution2D(src, weights: weights, descriptor: desc, name: nil)
892
+ plan = ConvPlan(graph: graph, tensorA: src, tensorB: weights, output: out,
893
+ shapeA: srcShape, shapeB: weightsShape, outputShape: outShape, dataType: mpsType)
894
+ }
895
+
896
+ convPlanCache[key] = plan
897
+ return plan
898
+ }
899
+
900
+ private func runConvPlan(_ plan: ConvPlan, bufA: MTLBuffer, bufB: MTLBuffer, bufOut: MTLBuffer) {
901
+ let ctx = BridgeContext.shared
902
+ ctx.flushBatch()
903
+ let dataA = MPSGraphTensorData(bufA, shape: plan.shapeA, dataType: plan.dataType)
904
+ let dataB = MPSGraphTensorData(bufB, shape: plan.shapeB, dataType: plan.dataType)
905
+ let dataOut = MPSGraphTensorData(bufOut, shape: plan.outputShape, dataType: plan.dataType)
906
+
907
+ plan.graph.run(with: ctx.queue, feeds: [plan.tensorA: dataA, plan.tensorB: dataB],
908
+ targetOperations: nil,
909
+ resultsDictionary: [plan.output: dataOut])
910
+ }
911
+
912
+ private func runConvPlan(_ plan: ConvPlan, a: UnsafeMutableRawPointer, b: UnsafeMutableRawPointer,
913
+ out: UnsafeMutableRawPointer) {
914
+ let boxA = Unmanaged<BufferBox>.fromOpaque(a).takeUnretainedValue()
915
+ let boxB = Unmanaged<BufferBox>.fromOpaque(b).takeUnretainedValue()
916
+ let boxOut = Unmanaged<BufferBox>.fromOpaque(out).takeUnretainedValue()
917
+ runConvPlan(plan, bufA: boxA.buffer, bufB: boxB.buffer, bufOut: boxOut.buffer)
918
+ }
919
+
920
+ // MPSGraph's convolution **silently breaks** when output channels exceed 65536
921
+ // (rows past 65536 are never written and the first few rows differ; no error
922
+ // or warning).
923
+ //
924
+ // Up to exactly 65536 it is correct, so cout is split into chunks of that
925
+ // size. Buffers are storageModeShared, so slicing is a memcpy (graph.run is
926
+ // synchronous).
927
+ private let CONV_MAX_COUT: Int32 = 65536
928
+
929
+ private func convChunkBuffer(_ bytes: Int, _ what: String,
930
+ _ errorOut: UnsafeMutablePointer<UnsafeMutablePointer<CChar>?>?) -> MTLBuffer? {
931
+ guard let buf = BridgeContext.shared.device.makeBuffer(length: max(bytes, 1), options: .storageModeShared) else {
932
+ setError(errorOut, "failed to allocate \(bytes) bytes for the chunked conv2d \(what)")
933
+ return nil
934
+ }
935
+ return buf
936
+ }
937
+
938
+ // Slice / write back along cout of [n, cout, plane]. plane is in bytes
939
+ // (oh*ow*elemSize), not elements.
940
+ private func convCopyChannels(dst: UnsafeMutableRawPointer, dstChannels: Int, dstOffset: Int,
941
+ src: UnsafeMutableRawPointer, srcChannels: Int, srcOffset: Int,
942
+ count: Int, batch: Int, planeBytes: Int) {
943
+ for i in 0..<batch {
944
+ let d = dst + (((i * dstChannels) + dstOffset) * planeBytes)
945
+ let s = src + (((i * srcChannels) + srcOffset) * planeBytes)
946
+ memcpy(d, s, count * planeBytes)
947
+ }
948
+ }
949
+
950
+ // dst += src elementwise. Only backward_data has chunk results landing on the
951
+ // same dx (splitting by output channel makes the input gradient the sum over
952
+ // all chunks), so it must accumulate rather than write.
953
+ private func convAccumulate(dst: UnsafeMutableRawPointer, src: UnsafeMutableRawPointer,
954
+ count: Int, dtype: Int32) {
955
+ if dtype == 1 {
956
+ let d = dst.bindMemory(to: Float16.self, capacity: count)
957
+ let s = src.bindMemory(to: Float16.self, capacity: count)
958
+ for i in 0..<count { d[i] += s[i] }
959
+ } else {
960
+ let d = dst.bindMemory(to: Float.self, capacity: count)
961
+ let s = src.bindMemory(to: Float.self, capacity: count)
962
+ for i in 0..<count { d[i] += s[i] }
963
+ }
964
+ }
965
+
966
+ // Taken only when cout > CONV_MAX_COUT. Ops differ only in what they slice and
967
+ // write back; the split (cout in steps of CONV_MAX_COUT) is shared.
968
+ private func runConvChunked(op: Int32, a: UnsafeMutableRawPointer, b: UnsafeMutableRawPointer,
969
+ out: UnsafeMutableRawPointer,
970
+ n: Int32, cin: Int32, h: Int32, width: Int32,
971
+ cout: Int32, kh: Int32, kw: Int32,
972
+ padH: Int32, padW: Int32, strideH: Int32, strideW: Int32,
973
+ dilationH: Int32, dilationW: Int32, groups: Int32,
974
+ dtype: Int32,
975
+ errorOut: UnsafeMutablePointer<UnsafeMutablePointer<CChar>?>?) -> Int32 {
976
+ // Grouped convolution (groups > 1) mixes up channels unless chunk
977
+ // boundaries align with group boundaries. A grouped conv with cout > 65536
978
+ // is unrealistic, so reject it explicitly rather than break silently.
979
+ if groups != 1 {
980
+ setError(errorOut, "conv2d: groups > 1 with cout > \(CONV_MAX_COUT) is not supported " +
981
+ "(chunked cout path only handles groups == 1)")
982
+ return -1
983
+ }
984
+
985
+ let boxA = Unmanaged<BufferBox>.fromOpaque(a).takeUnretainedValue()
986
+ let boxB = Unmanaged<BufferBox>.fromOpaque(b).takeUnretainedValue()
987
+ let boxOut = Unmanaged<BufferBox>.fromOpaque(out).takeUnretainedValue()
988
+
989
+ // This path slices its operands with memcpy on the host, so queue order is
990
+ // not enough here: everything that writes them must have finished.
991
+ BridgeContext.shared.synchronize()
992
+
993
+ let es = elemSize(dtype)
994
+ let effectiveKh = (Int(kh) - 1) * Int(dilationH) + 1
995
+ let effectiveKw = (Int(kw) - 1) * Int(dilationW) + 1
996
+ let oh = (Int(h) + 2 * Int(padH) - effectiveKh) / Int(strideH) + 1
997
+ let ow = (Int(width) + 2 * Int(padW) - effectiveKw) / Int(strideW) + 1
998
+ let outPlane = oh * ow * es // bytes per output channel
999
+ let weightPlane = Int(cin) * Int(kh) * Int(kw) * es // bytes per output channel of the weight
1000
+ let batch = Int(n)
1001
+
1002
+ // backward_data accumulates, so zero the destination first.
1003
+ if op == CONV_OP_BACKWARD_DATA {
1004
+ memset(boxOut.buffer.contents(), 0, batch * Int(cin) * Int(h) * Int(width) * es)
1005
+ }
1006
+
1007
+ var done: Int32 = 0
1008
+ while done < cout {
1009
+ let chunk = min(CONV_MAX_COUT, cout - done)
1010
+ let plan = convPlan(op: op, n: n, cin: cin, h: h, w: width,
1011
+ cout: chunk, kh: kh, kw: kw, padH: padH, padW: padW,
1012
+ strideH: strideH, strideW: strideW,
1013
+ dilationH: dilationH, dilationW: dilationW, groups: 1, dtype: dtype)
1014
+
1015
+ switch op {
1016
+ case CONV_OP_BACKWARD_DATA:
1017
+ // a = weight [cout, cin, kh, kw] (contiguous), b = output gradient [n, cout, oh, ow]
1018
+ guard let wTmp = convChunkBuffer(Int(chunk) * weightPlane, "weights", errorOut),
1019
+ let gTmp = convChunkBuffer(batch * Int(chunk) * outPlane, "grad_output", errorOut),
1020
+ let dxTmp = convChunkBuffer(batch * Int(cin) * Int(h) * Int(width) * es, "grad_input", errorOut)
1021
+ else { return -1 }
1022
+ memcpy(wTmp.contents(), boxA.buffer.contents() + Int(done) * weightPlane, Int(chunk) * weightPlane)
1023
+ convCopyChannels(dst: gTmp.contents(), dstChannels: Int(chunk), dstOffset: 0,
1024
+ src: boxB.buffer.contents(), srcChannels: Int(cout), srcOffset: Int(done),
1025
+ count: Int(chunk), batch: batch, planeBytes: outPlane)
1026
+ runConvPlan(plan, bufA: wTmp, bufB: gTmp, bufOut: dxTmp)
1027
+ convAccumulate(dst: boxOut.buffer.contents(), src: dxTmp.contents(),
1028
+ count: batch * Int(cin) * Int(h) * Int(width), dtype: dtype)
1029
+
1030
+ case CONV_OP_BACKWARD_FILTER:
1031
+ // a = input (all of it), b = output gradient, output = weight gradient [cout, cin, kh, kw]
1032
+ guard let gTmp = convChunkBuffer(batch * Int(chunk) * outPlane, "grad_output", errorOut),
1033
+ let dwTmp = convChunkBuffer(Int(chunk) * weightPlane, "grad_weight", errorOut)
1034
+ else { return -1 }
1035
+ convCopyChannels(dst: gTmp.contents(), dstChannels: Int(chunk), dstOffset: 0,
1036
+ src: boxB.buffer.contents(), srcChannels: Int(cout), srcOffset: Int(done),
1037
+ count: Int(chunk), batch: batch, planeBytes: outPlane)
1038
+ runConvPlan(plan, bufA: boxA.buffer, bufB: gTmp, bufOut: dwTmp)
1039
+ memcpy(boxOut.buffer.contents() + Int(done) * weightPlane, dwTmp.contents(), Int(chunk) * weightPlane)
1040
+
1041
+ default: // CONV_OP_FORWARD
1042
+ // a = input (all of it), b = weight [cout, cin, kh, kw] (contiguous)
1043
+ guard let wTmp = convChunkBuffer(Int(chunk) * weightPlane, "weights", errorOut),
1044
+ let yTmp = convChunkBuffer(batch * Int(chunk) * outPlane, "output", errorOut)
1045
+ else { return -1 }
1046
+ memcpy(wTmp.contents(), boxB.buffer.contents() + Int(done) * weightPlane, Int(chunk) * weightPlane)
1047
+ runConvPlan(plan, bufA: boxA.buffer, bufB: wTmp, bufOut: yTmp)
1048
+ convCopyChannels(dst: boxOut.buffer.contents(), dstChannels: Int(cout), dstOffset: Int(done),
1049
+ src: yTmp.contents(), srcChannels: Int(chunk), srcOffset: 0,
1050
+ count: Int(chunk), batch: batch, planeBytes: outPlane)
1051
+ }
1052
+
1053
+ done += chunk
1054
+ }
1055
+ return 0
1056
+ }
1057
+
1058
+ @_cdecl("gpu_bridge_conv2d")
1059
+ public func gpu_bridge_conv2d(_ x: UnsafeMutableRawPointer, _ w: UnsafeMutableRawPointer, _ y: UnsafeMutableRawPointer,
1060
+ _ n: Int32, _ cin: Int32, _ h: Int32, _ width: Int32,
1061
+ _ cout: Int32, _ kh: Int32, _ kw: Int32,
1062
+ _ padH: Int32, _ padW: Int32, _ strideH: Int32, _ strideW: Int32,
1063
+ _ dilationH: Int32, _ dilationW: Int32, _ groups: Int32,
1064
+ _ dtype: Int32,
1065
+ _ errorOut: UnsafeMutablePointer<UnsafeMutablePointer<CChar>?>?) -> Int32 {
1066
+ return autoreleasepool { () -> Int32 in
1067
+ if cout > CONV_MAX_COUT {
1068
+ return runConvChunked(op: CONV_OP_FORWARD, a: x, b: w, out: y,
1069
+ n: n, cin: cin, h: h, width: width, cout: cout, kh: kh, kw: kw,
1070
+ padH: padH, padW: padW, strideH: strideH, strideW: strideW,
1071
+ dilationH: dilationH, dilationW: dilationW, groups: groups,
1072
+ dtype: dtype, errorOut: errorOut)
1073
+ }
1074
+ let plan = convPlan(op: CONV_OP_FORWARD, n: n, cin: cin, h: h, w: width,
1075
+ cout: cout, kh: kh, kw: kw, padH: padH, padW: padW,
1076
+ strideH: strideH, strideW: strideW,
1077
+ dilationH: dilationH, dilationW: dilationW, groups: groups, dtype: dtype)
1078
+ runConvPlan(plan, a: x, b: w, out: y)
1079
+ return 0
1080
+ }
1081
+ }
1082
+
1083
+ @_cdecl("gpu_bridge_conv2d_backward")
1084
+ public func gpu_bridge_conv2d_backward(_ a: UnsafeMutableRawPointer, _ b: UnsafeMutableRawPointer,
1085
+ _ out: UnsafeMutableRawPointer,
1086
+ _ n: Int32, _ cin: Int32, _ h: Int32, _ width: Int32,
1087
+ _ cout: Int32, _ kh: Int32, _ kw: Int32,
1088
+ _ padH: Int32, _ padW: Int32, _ strideH: Int32, _ strideW: Int32,
1089
+ _ dilationH: Int32, _ dilationW: Int32, _ groups: Int32,
1090
+ _ dtype: Int32, _ mode: Int32,
1091
+ _ errorOut: UnsafeMutablePointer<UnsafeMutablePointer<CChar>?>?) -> Int32 {
1092
+ return autoreleasepool { () -> Int32 in
1093
+ let op = mode == 0 ? CONV_OP_BACKWARD_DATA : CONV_OP_BACKWARD_FILTER
1094
+ if cout > CONV_MAX_COUT {
1095
+ return runConvChunked(op: op, a: a, b: b, out: out,
1096
+ n: n, cin: cin, h: h, width: width, cout: cout, kh: kh, kw: kw,
1097
+ padH: padH, padW: padW, strideH: strideH, strideW: strideW,
1098
+ dilationH: dilationH, dilationW: dilationW, groups: groups,
1099
+ dtype: dtype, errorOut: errorOut)
1100
+ }
1101
+ let plan = convPlan(op: op, n: n, cin: cin, h: h, w: width,
1102
+ cout: cout, kh: kh, kw: kw, padH: padH, padW: padW,
1103
+ strideH: strideH, strideW: strideW,
1104
+ dilationH: dilationH, dilationW: dilationW, groups: groups, dtype: dtype)
1105
+ runConvPlan(plan, a: a, b: b, out: out)
1106
+ return 0
1107
+ }
1108
+ }
1109
+
1110
+ // MARK: - Dispatch
1111
+
1112
+ // args is an array of kinds: kind=0 is an MTLBuffer handle, kind=1 raw setBytes bytes.
1113
+ @_cdecl("gpu_bridge_dispatch")
1114
+ public func gpu_bridge_dispatch(_ pipelineHandle: UnsafeMutableRawPointer,
1115
+ _ argKinds: UnsafePointer<Int32>?,
1116
+ _ argPtrs: UnsafePointer<UnsafeMutableRawPointer?>?,
1117
+ _ argLens: UnsafePointer<Int>?,
1118
+ _ argCount: Int32,
1119
+ _ gx: Int, _ gy: Int, _ gz: Int,
1120
+ _ bx: Int, _ by: Int, _ bz: Int,
1121
+ _ wait: Int32,
1122
+ _ errorOut: UnsafeMutablePointer<UnsafeMutablePointer<CChar>?>?) -> Int32 {
1123
+ return autoreleasepool { () -> Int32 in
1124
+ let ctx = BridgeContext.shared
1125
+ let pso = Unmanaged<PipelineBox>.fromOpaque(pipelineHandle).takeUnretainedValue().pso
1126
+
1127
+ let cb: MTLCommandBuffer?
1128
+ let encoder: MTLComputeCommandEncoder
1129
+ if wait == 0 {
1130
+ cb = nil
1131
+ guard let shared = ctx.encoderForBatch() else {
1132
+ setError(errorOut, "failed to create batched command buffer/encoder")
1133
+ return -1
1134
+ }
1135
+ encoder = shared
1136
+ } else {
1137
+ // Preserve queue order if an async batch is still open.
1138
+ ctx.synchronize()
1139
+ guard let commandBuffer = ctx.queue.makeCommandBuffer(),
1140
+ let synchronous = commandBuffer.makeComputeCommandEncoder() else {
1141
+ setError(errorOut, "failed to create command buffer/encoder")
1142
+ return -1
1143
+ }
1144
+ cb = commandBuffer
1145
+ encoder = synchronous
1146
+ }
1147
+ encoder.setComputePipelineState(pso)
1148
+
1149
+ for i in 0..<Int(argCount) {
1150
+ guard let ptr = argPtrs?[i] else {
1151
+ if wait != 0 { encoder.endEncoding() }
1152
+ setError(errorOut, "argument \(i) is null")
1153
+ return -1
1154
+ }
1155
+ if argKinds?[i] == 0 {
1156
+ let box = Unmanaged<BufferBox>.fromOpaque(ptr).takeUnretainedValue()
1157
+ // For buffer arguments argLens carries the view's byte offset
1158
+ // (see gpu_bridge.c); 0 for an ordinary buffer.
1159
+ encoder.setBuffer(box.buffer, offset: argLens?[i] ?? 0, index: i)
1160
+ } else {
1161
+ encoder.setBytes(ptr, length: argLens?[i] ?? 0, index: i)
1162
+ }
1163
+ }
1164
+
1165
+ encoder.dispatchThreadgroups(MTLSize(width: gx, height: gy, depth: gz),
1166
+ threadsPerThreadgroup: MTLSize(width: bx, height: by, depth: bz))
1167
+ guard let cb else {
1168
+ ctx.didEncodeBatchDispatch()
1169
+ return 0
1170
+ }
1171
+ encoder.endEncoding()
1172
+ cb.commit()
1173
+ if wait != 0 {
1174
+ cb.waitUntilCompleted()
1175
+ if let error = cb.error {
1176
+ setError(errorOut, error.localizedDescription)
1177
+ return -1
1178
+ }
1179
+ } else {
1180
+ ctx.track(cb)
1181
+ }
1182
+ return 0
1183
+ }
1184
+ }
1185
+
1186
+ @_cdecl("gpu_bridge_synchronize")
1187
+ public func gpu_bridge_synchronize() {
1188
+ BridgeContext.shared.synchronize()
1189
+ }
1190
+
1191
+ // Optional ABI: synchronize, and report the first asynchronous command buffer
1192
+ // that failed since the last call. GEMM and wait:false dispatches return
1193
+ // before they run, so this boundary is the only place their failure can
1194
+ // surface.
1195
+ @_cdecl("gpu_bridge_synchronize_checked")
1196
+ public func gpu_bridge_synchronize_checked(
1197
+ _ errorOut: UnsafeMutablePointer<UnsafeMutablePointer<CChar>?>?) -> Int32 {
1198
+ let ctx = BridgeContext.shared
1199
+ ctx.synchronize()
1200
+ guard let error = ctx.takeAsyncError() else { return 0 }
1201
+ setError(errorOut, error)
1202
+ return -1
1203
+ }