tilework-metal 0.18.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE +21 -0
- data/ext/tilework/metal/bridge.swift +1203 -0
- data/ext/tilework/metal/extconf.rb +65 -0
- data/ext/tilework/metal/gpu_bridge.c +840 -0
- data/lib/tilework/backend/metal.rb +40 -0
- metadata +66 -0
|
@@ -0,0 +1,1203 @@
|
|
|
1
|
+
import Metal
|
|
2
|
+
import MetalPerformanceShaders
|
|
3
|
+
import MetalPerformanceShadersGraph
|
|
4
|
+
import Foundation
|
|
5
|
+
|
|
6
|
+
// The C ABI exported to the Ruby C shim (gpu_bridge.c). All policy (type
|
|
7
|
+
// conversion, validation, caching, exception hierarchy) lives in Ruby; this
|
|
8
|
+
// provides Metal mechanism only. Handles hand ownership to the C side
|
|
9
|
+
// (TypedData's free) via Unmanaged.passRetained.
|
|
10
|
+
|
|
11
|
+
final class BridgeContext {
|
|
12
|
+
static let shared = BridgeContext()
|
|
13
|
+
let device: MTLDevice
|
|
14
|
+
let queue: MTLCommandQueue
|
|
15
|
+
private var pending: [MTLCommandBuffer] = []
|
|
16
|
+
private var batchCommandBuffer: MTLCommandBuffer?
|
|
17
|
+
private var batchEncoder: MTLComputeCommandEncoder?
|
|
18
|
+
private var batchDispatches = 0
|
|
19
|
+
private let batchLimit = max(1, Int(ProcessInfo.processInfo.environment["TILEWORK_METAL_BATCH_SIZE"] ?? "8") ?? 8)
|
|
20
|
+
private var asyncError: String?
|
|
21
|
+
private let lock = NSLock()
|
|
22
|
+
|
|
23
|
+
private init() {
|
|
24
|
+
guard let device = MTLCreateSystemDefaultDevice(),
|
|
25
|
+
let queue = device.makeCommandQueue() else {
|
|
26
|
+
fatalError("tilework/metal: no Metal device available")
|
|
27
|
+
}
|
|
28
|
+
self.device = device
|
|
29
|
+
self.queue = queue
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
func track(_ cb: MTLCommandBuffer) {
|
|
33
|
+
lock.lock()
|
|
34
|
+
pruneFinishedLocked()
|
|
35
|
+
pending.append(cb)
|
|
36
|
+
lock.unlock()
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
// Nothing waits on an asynchronous command buffer at the call that
|
|
40
|
+
// submitted it, so its failure has no caller to return to. Keep the first
|
|
41
|
+
// one and report it at the next synchronization boundary instead of
|
|
42
|
+
// dropping it. Call with `lock` held.
|
|
43
|
+
private func recordFailureLocked(_ cb: MTLCommandBuffer) {
|
|
44
|
+
guard asyncError == nil, let error = cb.error else { return }
|
|
45
|
+
asyncError = error.localizedDescription
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
private func pruneFinishedLocked() {
|
|
49
|
+
pending.removeAll { cb in
|
|
50
|
+
if cb.status == .error { recordFailureLocked(cb) }
|
|
51
|
+
return cb.status == .completed || cb.status == .error
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
// The first failure of an asynchronous command buffer since the last call.
|
|
56
|
+
func takeAsyncError() -> String? {
|
|
57
|
+
lock.lock()
|
|
58
|
+
defer { lock.unlock() }
|
|
59
|
+
let error = asyncError
|
|
60
|
+
asyncError = nil
|
|
61
|
+
return error
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
// Async dispatches in one Tilework.submit/Graph boundary share a command
|
|
65
|
+
// buffer and compute encoder. setBytes copies scalar payloads at encode
|
|
66
|
+
// time, so callers do not need to retain the Ruby strings after dispatch.
|
|
67
|
+
func encoderForBatch() -> MTLComputeCommandEncoder? {
|
|
68
|
+
lock.lock()
|
|
69
|
+
defer { lock.unlock() }
|
|
70
|
+
if let encoder = batchEncoder { return encoder }
|
|
71
|
+
guard let cb = queue.makeCommandBuffer(),
|
|
72
|
+
let encoder = cb.makeComputeCommandEncoder() else { return nil }
|
|
73
|
+
batchCommandBuffer = cb
|
|
74
|
+
batchEncoder = encoder
|
|
75
|
+
return encoder
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
private func commitBatch() {
|
|
79
|
+
lock.lock()
|
|
80
|
+
guard let cb = batchCommandBuffer, let encoder = batchEncoder else {
|
|
81
|
+
lock.unlock()
|
|
82
|
+
return
|
|
83
|
+
}
|
|
84
|
+
batchCommandBuffer = nil
|
|
85
|
+
batchEncoder = nil
|
|
86
|
+
batchDispatches = 0
|
|
87
|
+
encoder.endEncoding()
|
|
88
|
+
cb.commit()
|
|
89
|
+
pending.append(cb)
|
|
90
|
+
lock.unlock()
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
// Delegated MPS / MPSGraph operations use their own command buffer.
|
|
94
|
+
// Commit any open custom-kernel batch first: an encoded but uncommitted
|
|
95
|
+
// dispatch is not on the queue yet, so a delegated command buffer
|
|
96
|
+
// committed now would run *before* the kernels that produce its inputs.
|
|
97
|
+
// No host wait is needed, because the queue executes committed command
|
|
98
|
+
// buffers in order. A path that touches `contents()` on the host must
|
|
99
|
+
// call `synchronize()` instead.
|
|
100
|
+
func flushBatch() {
|
|
101
|
+
commitBatch()
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
func didEncodeBatchDispatch() {
|
|
105
|
+
lock.lock()
|
|
106
|
+
batchDispatches += 1
|
|
107
|
+
let full = batchDispatches >= batchLimit
|
|
108
|
+
lock.unlock()
|
|
109
|
+
if full { commitBatch() }
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
func synchronize() {
|
|
113
|
+
commitBatch()
|
|
114
|
+
lock.lock()
|
|
115
|
+
let toWait = pending
|
|
116
|
+
pending.removeAll()
|
|
117
|
+
lock.unlock()
|
|
118
|
+
for cb in toWait { cb.waitUntilCompleted() }
|
|
119
|
+
lock.lock()
|
|
120
|
+
for cb in toWait { recordFailureLocked(cb) }
|
|
121
|
+
lock.unlock()
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
// A shared MTLBuffer that was dirtied through `contents()` keeps its
|
|
125
|
+
// system-memory pages resident after its last strong reference goes away.
|
|
126
|
+
// Marking it empty tells Metal that both the resource and its contents can
|
|
127
|
+
// be discarded. It must happen after the last async command buffer that
|
|
128
|
+
// could still use it; the command queue's FIFO ordering makes the newest
|
|
129
|
+
// tracked buffer a sufficient fence for all earlier launches.
|
|
130
|
+
func discard(_ buffer: MTLBuffer) {
|
|
131
|
+
// The buffer may still be referenced by an open async encoder. Commit
|
|
132
|
+
// that batch before deciding whether an ordered purge fence is needed.
|
|
133
|
+
commitBatch()
|
|
134
|
+
lock.lock()
|
|
135
|
+
pruneFinishedLocked()
|
|
136
|
+
let hasPendingWork = !pending.isEmpty
|
|
137
|
+
lock.unlock()
|
|
138
|
+
|
|
139
|
+
guard hasPendingWork else {
|
|
140
|
+
_ = buffer.setPurgeableState(.empty)
|
|
141
|
+
return
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
// Metal requires completion handlers to be registered before commit.
|
|
145
|
+
// This empty command buffer is ordered after every previously
|
|
146
|
+
// submitted command on `queue`, and so is a safe, non-blocking fence.
|
|
147
|
+
guard let fence = queue.makeCommandBuffer() else {
|
|
148
|
+
return
|
|
149
|
+
}
|
|
150
|
+
fence.addCompletedHandler { [buffer] _ in
|
|
151
|
+
autoreleasepool {
|
|
152
|
+
_ = buffer.setPurgeableState(.empty)
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
fence.commit()
|
|
156
|
+
track(fence)
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
final class BufferBox {
|
|
161
|
+
let buffer: MTLBuffer
|
|
162
|
+
init(_ buffer: MTLBuffer) { self.buffer = buffer }
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
final class PipelineBox {
|
|
166
|
+
let pso: MTLComputePipelineState
|
|
167
|
+
init(_ pso: MTLComputePipelineState) { self.pso = pso }
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
private func setError(_ out: UnsafeMutablePointer<UnsafeMutablePointer<CChar>?>?, _ message: String) {
|
|
171
|
+
out?.pointee = strdup(message)
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
@_cdecl("gpu_bridge_device_name")
|
|
175
|
+
public func gpu_bridge_device_name() -> UnsafeMutablePointer<CChar>? {
|
|
176
|
+
return strdup(BridgeContext.shared.device.name)
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
@_cdecl("gpu_bridge_device_available")
|
|
180
|
+
public func gpu_bridge_device_available() -> Int32 {
|
|
181
|
+
return MTLCreateSystemDefaultDevice() != nil ? 1 : 0
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
// MARK: - Buffer
|
|
185
|
+
|
|
186
|
+
@_cdecl("gpu_bridge_buffer_alloc")
|
|
187
|
+
public func gpu_bridge_buffer_alloc(_ bytes: Int) -> UnsafeMutableRawPointer? {
|
|
188
|
+
// Ruby invokes this entrypoint directly, without an AppKit/Foundation run
|
|
189
|
+
// loop to drain Objective-C autorelease pools. `makeBuffer` returns an
|
|
190
|
+
// autoreleased Metal object; BufferBox retains it for the lifetime of the
|
|
191
|
+
// Ruby handle, but the original autorelease must still be drained here.
|
|
192
|
+
// Without this pool every allocation keeps an extra MTLBuffer reference
|
|
193
|
+
// until process exit, so explicit Buffer#free cannot reclaim inference
|
|
194
|
+
// weights.
|
|
195
|
+
return autoreleasepool { () -> UnsafeMutableRawPointer? in
|
|
196
|
+
guard let buf = BridgeContext.shared.device.makeBuffer(length: max(bytes, 1),
|
|
197
|
+
options: .storageModeShared) else {
|
|
198
|
+
return nil
|
|
199
|
+
}
|
|
200
|
+
return Unmanaged.passRetained(BufferBox(buf)).toOpaque()
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
// With storageModeShared, buffer_write / buffer_read are **raw memcpys from
|
|
205
|
+
// the host** and never go on the command queue, so calling them while an
|
|
206
|
+
// in-flight kernel touches the same buffer is a data race. Waiting is core's
|
|
207
|
+
// job (Buffer#write / #to_a etc. call Tilework.synchronize) — waiting here
|
|
208
|
+
// every time would drag synchronization into paths made of many small
|
|
209
|
+
// transfers, such as offload.
|
|
210
|
+
@_cdecl("gpu_bridge_buffer_write")
|
|
211
|
+
public func gpu_bridge_buffer_write(_ handle: UnsafeMutableRawPointer,
|
|
212
|
+
_ src: UnsafeRawPointer,
|
|
213
|
+
_ bytes: Int, _ offset: Int) {
|
|
214
|
+
let box = Unmanaged<BufferBox>.fromOpaque(handle).takeUnretainedValue()
|
|
215
|
+
box.buffer.contents().advanced(by: offset).copyMemory(from: src, byteCount: bytes)
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
@_cdecl("gpu_bridge_buffer_read")
|
|
219
|
+
public func gpu_bridge_buffer_read(_ handle: UnsafeMutableRawPointer,
|
|
220
|
+
_ dst: UnsafeMutableRawPointer,
|
|
221
|
+
_ bytes: Int, _ offset: Int) {
|
|
222
|
+
let box = Unmanaged<BufferBox>.fromOpaque(handle).takeUnretainedValue()
|
|
223
|
+
dst.copyMemory(from: box.buffer.contents().advanced(by: offset), byteCount: bytes)
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
@_cdecl("gpu_bridge_buffer_free")
|
|
227
|
+
public func gpu_bridge_buffer_free(_ handle: UnsafeMutableRawPointer) {
|
|
228
|
+
let box = Unmanaged<BufferBox>.fromOpaque(handle).takeUnretainedValue()
|
|
229
|
+
BridgeContext.shared.discard(box.buffer)
|
|
230
|
+
Unmanaged<BufferBox>.fromOpaque(handle).release()
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
// MARK: - Pipeline
|
|
234
|
+
|
|
235
|
+
@_cdecl("gpu_bridge_pipeline_create")
|
|
236
|
+
public func gpu_bridge_pipeline_create(_ source: UnsafePointer<CChar>,
|
|
237
|
+
_ fnName: UnsafePointer<CChar>,
|
|
238
|
+
_ errorOut: UnsafeMutablePointer<UnsafeMutablePointer<CChar>?>?)
|
|
239
|
+
-> UnsafeMutableRawPointer? {
|
|
240
|
+
// Like buffers, pipeline creation can be called repeatedly from Ruby with
|
|
241
|
+
// no ambient autorelease pool. PipelineBox owns the compiled state; this
|
|
242
|
+
// pool only releases the temporary library/function objects.
|
|
243
|
+
return autoreleasepool { () -> UnsafeMutableRawPointer? in
|
|
244
|
+
let ctx = BridgeContext.shared
|
|
245
|
+
let src = String(cString: source)
|
|
246
|
+
let name = String(cString: fnName)
|
|
247
|
+
do {
|
|
248
|
+
let library = try ctx.device.makeLibrary(source: src, options: nil)
|
|
249
|
+
guard let fn = library.makeFunction(name: name) else {
|
|
250
|
+
setError(errorOut, "kernel function '\(name)' not found in compiled library")
|
|
251
|
+
return nil
|
|
252
|
+
}
|
|
253
|
+
let pso = try ctx.device.makeComputePipelineState(function: fn)
|
|
254
|
+
return Unmanaged.passRetained(PipelineBox(pso)).toOpaque()
|
|
255
|
+
} catch {
|
|
256
|
+
setError(errorOut, error.localizedDescription)
|
|
257
|
+
return nil
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
@_cdecl("gpu_bridge_pipeline_max_threads")
|
|
263
|
+
public func gpu_bridge_pipeline_max_threads(_ handle: UnsafeMutableRawPointer) -> Int {
|
|
264
|
+
let box = Unmanaged<PipelineBox>.fromOpaque(handle).takeUnretainedValue()
|
|
265
|
+
return box.pso.maxTotalThreadsPerThreadgroup
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
@_cdecl("gpu_bridge_pipeline_free")
|
|
269
|
+
public func gpu_bridge_pipeline_free(_ handle: UnsafeMutableRawPointer) {
|
|
270
|
+
Unmanaged<PipelineBox>.fromOpaque(handle).release()
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
// MARK: - GEMM (MPSMatrixMultiplication)
|
|
274
|
+
//
|
|
275
|
+
// The two GEMM entrypoints have *different* buffer-layout contracts (see
|
|
276
|
+
// bridge.cpp, the CUDA reference impl, and tensor_core.rb):
|
|
277
|
+
//
|
|
278
|
+
// - `gemm` (plain, below): a/b/c are Tilework::Buffer's native row-major
|
|
279
|
+
// layout. MPS is natively row-major too, so this needs no reinterpretation
|
|
280
|
+
// — read A/B/C directly as row-major matrices.
|
|
281
|
+
//
|
|
282
|
+
// - `gemm_strided_batched` (encodeGemm, this section): a/b/c are genuinely
|
|
283
|
+
// column-major (lda/ldb/ldc/trans_a/trans_b are the caller's cuBLAS-style
|
|
284
|
+
// column-major description; NN.multi_head_attention builds these itself).
|
|
285
|
+
// Since MPS is row-major, we read each column-major buffer directly as
|
|
286
|
+
// its own row-major transpose instead of copying/transposing — a col-
|
|
287
|
+
// major (r, c) buffer with leading dimension ld is bit-identical to a
|
|
288
|
+
// row-major (c, r) buffer with rowBytes = ld * elemSize. Combined with
|
|
289
|
+
// cuBLAS's own "swap A/B, swap m/n, keep trans flags" identity for
|
|
290
|
+
// row-major libraries, this lets the call be encoded with zero extra
|
|
291
|
+
// data movement:
|
|
292
|
+
//
|
|
293
|
+
// leftMatrix = rowview(B), transpose = trans_b (represents op(B)^T)
|
|
294
|
+
// rightMatrix = rowview(A), transpose = trans_a (represents op(A)^T)
|
|
295
|
+
// result = rowview(C) (represents C^T, i.e. C's
|
|
296
|
+
// raw bytes read row-major)
|
|
297
|
+
//
|
|
298
|
+
// See core/lib/tilework/math/tensor_core.rb for the full ABI contract.
|
|
299
|
+
|
|
300
|
+
private func mpsDataType(_ code: Int32) -> MPSDataType {
|
|
301
|
+
if code == 1 { return .float16 }
|
|
302
|
+
if code == 2 { return .bFloat16 }
|
|
303
|
+
return .float32
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
private func elemSize(_ code: Int32) -> Int {
|
|
307
|
+
return code == 0 ? 4 : 2
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
// `handle` is always the base BufferBox pointer returned by buffer_alloc —
|
|
311
|
+
// never offset (see gpu_bridge_dispatch's comment in gpu_bridge.c: CUDA does
|
|
312
|
+
// pointer arithmetic, Metal takes the offset separately). Views pass their
|
|
313
|
+
// byte offset in `offset`, applied here via MPSMatrix's own offset param
|
|
314
|
+
// rather than by shifting the opaque handle, which is not a raw device
|
|
315
|
+
// pointer and cannot be offset like one.
|
|
316
|
+
private func mpsMatrix(_ handle: UnsafeMutableRawPointer, offset: Int, rows: Int, columns: Int,
|
|
317
|
+
rowBytes: Int, matrices: Int, matrixBytes: Int,
|
|
318
|
+
dataType: MPSDataType) -> MPSMatrix {
|
|
319
|
+
let box = Unmanaged<BufferBox>.fromOpaque(handle).takeUnretainedValue()
|
|
320
|
+
let desc = MPSMatrixDescriptor(rows: rows, columns: columns, matrices: matrices,
|
|
321
|
+
rowBytes: rowBytes, matrixBytes: matrixBytes,
|
|
322
|
+
dataType: dataType)
|
|
323
|
+
return MPSMatrix(buffer: box.buffer, offset: offset, descriptor: desc)
|
|
324
|
+
}
|
|
325
|
+
|
|
326
|
+
private func encodeGemm(_ a: UnsafeMutableRawPointer, _ aOff: Int,
|
|
327
|
+
_ b: UnsafeMutableRawPointer, _ bOff: Int,
|
|
328
|
+
_ c: UnsafeMutableRawPointer, _ cOff: Int,
|
|
329
|
+
m: Int, k: Int, n: Int,
|
|
330
|
+
lda: Int, ldb: Int, ldc: Int,
|
|
331
|
+
strideA: Int, strideB: Int, strideC: Int,
|
|
332
|
+
transA: Bool, transB: Bool,
|
|
333
|
+
batch: Int, alpha: Double, inDtype: Int32,
|
|
334
|
+
errorOut: UnsafeMutablePointer<UnsafeMutablePointer<CChar>?>?) -> Int32 {
|
|
335
|
+
let ctx = BridgeContext.shared
|
|
336
|
+
ctx.flushBatch()
|
|
337
|
+
let inType = mpsDataType(inDtype)
|
|
338
|
+
let inElem = elemSize(inDtype)
|
|
339
|
+
let outElem = 4 // ABI contract: the output is always float32
|
|
340
|
+
|
|
341
|
+
// Raw dimensions in BLAS convention (before transposes are applied).
|
|
342
|
+
let aRawRows = transA ? k : m
|
|
343
|
+
let aRawCols = transA ? m : k
|
|
344
|
+
let bRawRows = transB ? n : k
|
|
345
|
+
let bRawCols = transB ? k : n
|
|
346
|
+
|
|
347
|
+
let leftMat = mpsMatrix(b, offset: bOff, rows: bRawCols, columns: bRawRows,
|
|
348
|
+
rowBytes: ldb * inElem, matrices: batch,
|
|
349
|
+
matrixBytes: strideB * inElem, dataType: inType)
|
|
350
|
+
let rightMat = mpsMatrix(a, offset: aOff, rows: aRawCols, columns: aRawRows,
|
|
351
|
+
rowBytes: lda * inElem, matrices: batch,
|
|
352
|
+
matrixBytes: strideA * inElem, dataType: inType)
|
|
353
|
+
let resultMat = mpsMatrix(c, offset: cOff, rows: n, columns: m,
|
|
354
|
+
rowBytes: ldc * outElem, matrices: batch,
|
|
355
|
+
matrixBytes: strideC * outElem, dataType: .float32)
|
|
356
|
+
|
|
357
|
+
let mul = MPSMatrixMultiplication(device: ctx.device,
|
|
358
|
+
transposeLeft: transB, transposeRight: transA,
|
|
359
|
+
resultRows: n, resultColumns: m, interiorColumns: k,
|
|
360
|
+
alpha: alpha, beta: 0.0)
|
|
361
|
+
|
|
362
|
+
guard let cb = ctx.queue.makeCommandBuffer() else {
|
|
363
|
+
setError(errorOut, "failed to create command buffer for gemm")
|
|
364
|
+
return -1
|
|
365
|
+
}
|
|
366
|
+
mul.encode(commandBuffer: cb, leftMatrix: leftMat, rightMatrix: rightMat, resultMatrix: resultMat)
|
|
367
|
+
cb.commit()
|
|
368
|
+
ctx.track(cb)
|
|
369
|
+
return 0
|
|
370
|
+
}
|
|
371
|
+
|
|
372
|
+
// MARK: - GEMM: tiled simdgroup_matrix kernel (Phase 2, fp32 only)
|
|
373
|
+
//
|
|
374
|
+
// MPS (above) covers every shape correctly, but for large aligned fp32
|
|
375
|
+
// matmuls a hand-tiled kernel using simdgroup_float8x8 fragments can beat it
|
|
376
|
+
// by reusing each global-memory read across a whole threadgroup tile instead
|
|
377
|
+
// of paying MPS's more general dispatch overhead. This kernel deliberately
|
|
378
|
+
// mirrors the existing WMMA path's own tradeoff (see TensorCore's BLOCK_M/N/K
|
|
379
|
+
// comments in tensor_core.rb): it only accepts shapes that tile evenly, and
|
|
380
|
+
// anything else falls straight back to MPS — no partial-tile bounds checks,
|
|
381
|
+
// no attempt to be a general-purpose kernel.
|
|
382
|
+
//
|
|
383
|
+
// One simdgroup (32 threads) computes one 32x32 output tile as a 4x4 grid of
|
|
384
|
+
// 8x8 fragments, staging a 32x32 slice of A and B through threadgroup memory
|
|
385
|
+
// per K-step so the 32x32=1024 global loads are shared across all 16
|
|
386
|
+
// fragment multiplies that use them, instead of re-reading per fragment.
|
|
387
|
+
private let simdgroupGemmTile = 32
|
|
388
|
+
private let simdgroupGemmFrag = 8
|
|
389
|
+
|
|
390
|
+
private let simdgroupGemmSource = """
|
|
391
|
+
#include <metal_stdlib>
|
|
392
|
+
using namespace metal;
|
|
393
|
+
|
|
394
|
+
constant int TILE = \(simdgroupGemmTile);
|
|
395
|
+
constant int FRAG = \(simdgroupGemmFrag);
|
|
396
|
+
constant int FRAGS = TILE / FRAG;
|
|
397
|
+
|
|
398
|
+
kernel void gemm_simdgroup_f32(
|
|
399
|
+
device const float* A [[buffer(0)]],
|
|
400
|
+
device const float* B [[buffer(1)]],
|
|
401
|
+
device float* C [[buffer(2)]],
|
|
402
|
+
constant int& M [[buffer(3)]],
|
|
403
|
+
constant int& K [[buffer(4)]],
|
|
404
|
+
constant int& N [[buffer(5)]],
|
|
405
|
+
uint2 tgid [[threadgroup_position_in_grid]],
|
|
406
|
+
uint lane [[thread_index_in_simdgroup]])
|
|
407
|
+
{
|
|
408
|
+
threadgroup float As[TILE][TILE];
|
|
409
|
+
threadgroup float Bs[TILE][TILE];
|
|
410
|
+
|
|
411
|
+
int tileRow = int(tgid.y) * TILE;
|
|
412
|
+
int tileCol = int(tgid.x) * TILE;
|
|
413
|
+
|
|
414
|
+
simdgroup_float8x8 acc[FRAGS][FRAGS];
|
|
415
|
+
for (int i = 0; i < FRAGS; i++)
|
|
416
|
+
for (int j = 0; j < FRAGS; j++)
|
|
417
|
+
acc[i][j] = simdgroup_float8x8(0);
|
|
418
|
+
|
|
419
|
+
// Caller (gpu_bridge_gemm) only takes this path when M/N/K are all
|
|
420
|
+
// multiples of TILE, so every tile below is fully in-bounds.
|
|
421
|
+
for (int k0 = 0; k0 < K; k0 += TILE) {
|
|
422
|
+
for (int e = int(lane); e < TILE * TILE; e += 32) {
|
|
423
|
+
int r = e / TILE, c = e % TILE;
|
|
424
|
+
As[r][c] = A[(tileRow + r) * K + (k0 + c)];
|
|
425
|
+
Bs[r][c] = B[(k0 + r) * N + (tileCol + c)];
|
|
426
|
+
}
|
|
427
|
+
threadgroup_barrier(mem_flags::mem_threadgroup);
|
|
428
|
+
|
|
429
|
+
for (int kk = 0; kk < TILE; kk += FRAG) {
|
|
430
|
+
simdgroup_float8x8 aFrag[FRAGS], bFrag[FRAGS];
|
|
431
|
+
for (int i = 0; i < FRAGS; i++)
|
|
432
|
+
simdgroup_load(aFrag[i], &As[i * FRAG][kk], TILE);
|
|
433
|
+
for (int j = 0; j < FRAGS; j++)
|
|
434
|
+
simdgroup_load(bFrag[j], &Bs[kk][j * FRAG], TILE);
|
|
435
|
+
for (int i = 0; i < FRAGS; i++)
|
|
436
|
+
for (int j = 0; j < FRAGS; j++)
|
|
437
|
+
simdgroup_multiply_accumulate(acc[i][j], aFrag[i], bFrag[j], acc[i][j]);
|
|
438
|
+
}
|
|
439
|
+
threadgroup_barrier(mem_flags::mem_threadgroup);
|
|
440
|
+
}
|
|
441
|
+
|
|
442
|
+
for (int i = 0; i < FRAGS; i++)
|
|
443
|
+
for (int j = 0; j < FRAGS; j++)
|
|
444
|
+
simdgroup_store(acc[i][j], &C[(tileRow + i * FRAG) * N + (tileCol + j * FRAG)], N);
|
|
445
|
+
}
|
|
446
|
+
"""
|
|
447
|
+
|
|
448
|
+
private final class SimdgroupGemmKernel {
|
|
449
|
+
let pso: MTLComputePipelineState
|
|
450
|
+
private init(pso: MTLComputePipelineState) { self.pso = pso }
|
|
451
|
+
|
|
452
|
+
// Compiled once and cached. Where compilation itself fails (unsupported GPU
|
|
453
|
+
// family etc.) it is nil and callers quietly fall back to MPS — it is an
|
|
454
|
+
// optimization for supported GPUs only, so there is no reason to error.
|
|
455
|
+
static let shared: SimdgroupGemmKernel? = {
|
|
456
|
+
let ctx = BridgeContext.shared
|
|
457
|
+
guard let library = try? ctx.device.makeLibrary(source: simdgroupGemmSource, options: nil),
|
|
458
|
+
let fn = library.makeFunction(name: "gemm_simdgroup_f32"),
|
|
459
|
+
let pso = try? ctx.device.makeComputePipelineState(function: fn) else {
|
|
460
|
+
return nil
|
|
461
|
+
}
|
|
462
|
+
return SimdgroupGemmKernel(pso: pso)
|
|
463
|
+
}()
|
|
464
|
+
}
|
|
465
|
+
|
|
466
|
+
// Measured (M4-class, fp32, square 256/512/1024) at only 0.2–0.5x of MPS on
|
|
467
|
+
// every size (no double buffering or register blocking). The code is kept as
|
|
468
|
+
// a fallback for when MPS is unavailable, with the threshold at infinity so it
|
|
469
|
+
// is never chosen. Restore a measured value if multi-simdgroup tiling and
|
|
470
|
+
// double buffering bring it close to MPS.
|
|
471
|
+
private let simdgroupGemmMinElements = Int.max
|
|
472
|
+
|
|
473
|
+
private func encodeSimdgroupGemm(_ kernel: SimdgroupGemmKernel,
|
|
474
|
+
_ a: UnsafeMutableRawPointer, _ b: UnsafeMutableRawPointer,
|
|
475
|
+
_ c: UnsafeMutableRawPointer,
|
|
476
|
+
m: Int, k: Int, n: Int,
|
|
477
|
+
errorOut: UnsafeMutablePointer<UnsafeMutablePointer<CChar>?>?) -> Int32 {
|
|
478
|
+
let ctx = BridgeContext.shared
|
|
479
|
+
ctx.flushBatch()
|
|
480
|
+
let boxA = Unmanaged<BufferBox>.fromOpaque(a).takeUnretainedValue()
|
|
481
|
+
let boxB = Unmanaged<BufferBox>.fromOpaque(b).takeUnretainedValue()
|
|
482
|
+
let boxC = Unmanaged<BufferBox>.fromOpaque(c).takeUnretainedValue()
|
|
483
|
+
|
|
484
|
+
guard let cb = ctx.queue.makeCommandBuffer(),
|
|
485
|
+
let encoder = cb.makeComputeCommandEncoder() else {
|
|
486
|
+
setError(errorOut, "failed to create command buffer/encoder for simdgroup gemm")
|
|
487
|
+
return -1
|
|
488
|
+
}
|
|
489
|
+
var m32 = Int32(m), k32 = Int32(k), n32 = Int32(n)
|
|
490
|
+
encoder.setComputePipelineState(kernel.pso)
|
|
491
|
+
encoder.setBuffer(boxA.buffer, offset: 0, index: 0)
|
|
492
|
+
encoder.setBuffer(boxB.buffer, offset: 0, index: 1)
|
|
493
|
+
encoder.setBuffer(boxC.buffer, offset: 0, index: 2)
|
|
494
|
+
encoder.setBytes(&m32, length: 4, index: 3)
|
|
495
|
+
encoder.setBytes(&k32, length: 4, index: 4)
|
|
496
|
+
encoder.setBytes(&n32, length: 4, index: 5)
|
|
497
|
+
encoder.dispatchThreadgroups(MTLSize(width: n / simdgroupGemmTile, height: m / simdgroupGemmTile, depth: 1),
|
|
498
|
+
threadsPerThreadgroup: MTLSize(width: 32, height: 1, depth: 1))
|
|
499
|
+
encoder.endEncoding()
|
|
500
|
+
cb.commit()
|
|
501
|
+
ctx.track(cb)
|
|
502
|
+
return 0
|
|
503
|
+
}
|
|
504
|
+
|
|
505
|
+
// Plain GEMM: a/b/c are row-major (see the GEMM section above).
|
|
506
|
+
@_cdecl("gpu_bridge_gemm")
|
|
507
|
+
public func gpu_bridge_gemm(_ a: UnsafeMutableRawPointer, _ aOff: Int,
|
|
508
|
+
_ b: UnsafeMutableRawPointer, _ bOff: Int,
|
|
509
|
+
_ c: UnsafeMutableRawPointer, _ cOff: Int,
|
|
510
|
+
_ m: Int, _ k: Int, _ n: Int,
|
|
511
|
+
_ allowTF32: Int32, _ inDtype: Int32,
|
|
512
|
+
_ errorOut: UnsafeMutablePointer<UnsafeMutablePointer<CChar>?>?) -> Int32 {
|
|
513
|
+
return autoreleasepool { () -> Int32 in
|
|
514
|
+
// allowTF32 is CUDA-specific (TensorFloat-32) and meaningless on Metal, so it is ignored.
|
|
515
|
+
// The hand-tiled kernel does not accept views (offset != 0), so any call
|
|
516
|
+
// with an offset goes to MPS unconditionally.
|
|
517
|
+
let tileAligned = m % simdgroupGemmTile == 0 && n % simdgroupGemmTile == 0 && k % simdgroupGemmTile == 0
|
|
518
|
+
if inDtype == 0, tileAligned, aOff == 0, bOff == 0, cOff == 0,
|
|
519
|
+
m * n * k >= simdgroupGemmMinElements,
|
|
520
|
+
let kernel = SimdgroupGemmKernel.shared {
|
|
521
|
+
return encodeSimdgroupGemm(kernel, a, b, c, m: m, k: k, n: n, errorOut: errorOut)
|
|
522
|
+
}
|
|
523
|
+
|
|
524
|
+
let ctx = BridgeContext.shared
|
|
525
|
+
ctx.flushBatch()
|
|
526
|
+
let inType = mpsDataType(inDtype)
|
|
527
|
+
let inElem = elemSize(inDtype)
|
|
528
|
+
|
|
529
|
+
let matA = mpsMatrix(a, offset: aOff, rows: m, columns: k, rowBytes: k * inElem,
|
|
530
|
+
matrices: 1, matrixBytes: m * k * inElem, dataType: inType)
|
|
531
|
+
let matB = mpsMatrix(b, offset: bOff, rows: k, columns: n, rowBytes: n * inElem,
|
|
532
|
+
matrices: 1, matrixBytes: k * n * inElem, dataType: inType)
|
|
533
|
+
let matC = mpsMatrix(c, offset: cOff, rows: m, columns: n, rowBytes: n * 4,
|
|
534
|
+
matrices: 1, matrixBytes: m * n * 4, dataType: .float32)
|
|
535
|
+
|
|
536
|
+
let mul = MPSMatrixMultiplication(device: ctx.device,
|
|
537
|
+
transposeLeft: false, transposeRight: false,
|
|
538
|
+
resultRows: m, resultColumns: n, interiorColumns: k,
|
|
539
|
+
alpha: 1.0, beta: 0.0)
|
|
540
|
+
|
|
541
|
+
guard let cb = ctx.queue.makeCommandBuffer() else {
|
|
542
|
+
setError(errorOut, "failed to create command buffer for gemm")
|
|
543
|
+
return -1
|
|
544
|
+
}
|
|
545
|
+
mul.encode(commandBuffer: cb, leftMatrix: matA, rightMatrix: matB, resultMatrix: matC)
|
|
546
|
+
cb.commit()
|
|
547
|
+
ctx.track(cb)
|
|
548
|
+
return 0
|
|
549
|
+
}
|
|
550
|
+
}
|
|
551
|
+
|
|
552
|
+
@_cdecl("gpu_bridge_gemm_strided_batched")
|
|
553
|
+
public func gpu_bridge_gemm_strided_batched(_ a: UnsafeMutableRawPointer, _ aOff: Int,
|
|
554
|
+
_ b: UnsafeMutableRawPointer, _ bOff: Int,
|
|
555
|
+
_ c: UnsafeMutableRawPointer, _ cOff: Int,
|
|
556
|
+
_ m: Int, _ k: Int, _ n: Int,
|
|
557
|
+
_ lda: Int, _ ldb: Int, _ ldc: Int,
|
|
558
|
+
_ strideA: Int, _ strideB: Int, _ strideC: Int,
|
|
559
|
+
_ transA: Int32, _ transB: Int32,
|
|
560
|
+
_ batch: Int, _ alpha: Double,
|
|
561
|
+
_ allowTF32: Int32, _ inDtype: Int32,
|
|
562
|
+
_ errorOut: UnsafeMutablePointer<UnsafeMutablePointer<CChar>?>?) -> Int32 {
|
|
563
|
+
return autoreleasepool { () -> Int32 in
|
|
564
|
+
return encodeGemm(a, aOff, b, bOff, c, cOff, m: m, k: k, n: n,
|
|
565
|
+
lda: lda, ldb: ldb, ldc: ldc,
|
|
566
|
+
strideA: strideA, strideB: strideB, strideC: strideC,
|
|
567
|
+
transA: transA != 0, transB: transB != 0,
|
|
568
|
+
batch: batch, alpha: alpha, inDtype: inDtype,
|
|
569
|
+
errorOut: errorOut)
|
|
570
|
+
}
|
|
571
|
+
}
|
|
572
|
+
|
|
573
|
+
// MARK: - Random (MPSMatrixRandomPhilox, cuRAND analogue)
|
|
574
|
+
//
|
|
575
|
+
// See random.rb for kind/mean/stddev. UNIFORM = 0, NORMAL = 1.
|
|
576
|
+
// cuRAND's convention makes uniform **(0, 1]** (excluding 0, including 1),
|
|
577
|
+
// but MPS's uniformDistributionDescriptor is [minimum, maximum) — the open end
|
|
578
|
+
// is on the other side, so the result is flipped with 1.0 - x afterwards.
|
|
579
|
+
|
|
580
|
+
private let RANDOM_KIND_UNIFORM: Int32 = 0
|
|
581
|
+
|
|
582
|
+
@_cdecl("gpu_bridge_random")
|
|
583
|
+
public func gpu_bridge_random(_ handle: UnsafeMutableRawPointer, _ n: Int, _ kind: Int32,
|
|
584
|
+
_ seed: UInt64, _ mean: Float, _ stddev: Float,
|
|
585
|
+
_ errorOut: UnsafeMutablePointer<UnsafeMutablePointer<CChar>?>?) -> Int32 {
|
|
586
|
+
return autoreleasepool { () -> Int32 in
|
|
587
|
+
let ctx = BridgeContext.shared
|
|
588
|
+
let box = Unmanaged<BufferBox>.fromOpaque(handle).takeUnretainedValue()
|
|
589
|
+
|
|
590
|
+
let desc: MPSMatrixRandomDistributionDescriptor
|
|
591
|
+
if kind == RANDOM_KIND_UNIFORM {
|
|
592
|
+
desc = MPSMatrixRandomDistributionDescriptor.uniformDistributionDescriptor(withMinimum: 0.0, maximum: 1.0)
|
|
593
|
+
} else {
|
|
594
|
+
desc = MPSMatrixRandomDistributionDescriptor.normalDistributionDescriptor(withMean: mean, standardDeviation: stddev)
|
|
595
|
+
}
|
|
596
|
+
|
|
597
|
+
let rng = MPSMatrixRandomPhilox(device: ctx.device, destinationDataType: .float32,
|
|
598
|
+
seed: Int(bitPattern: UInt(truncatingIfNeeded: seed)),
|
|
599
|
+
distributionDescriptor: desc)
|
|
600
|
+
|
|
601
|
+
// MPSMatrixRandomPhilox emits four elements at a time. A count that is
|
|
602
|
+
// not a multiple of four **kills the process with an assert** (not an
|
|
603
|
+
// exception, so Ruby cannot catch it). Generate into a scratch buffer
|
|
604
|
+
// rounded up to a multiple of four and copy back the first n elements.
|
|
605
|
+
let padded = (n + 3) & ~3
|
|
606
|
+
let scratch: MTLBuffer?
|
|
607
|
+
if padded == n {
|
|
608
|
+
scratch = nil
|
|
609
|
+
} else {
|
|
610
|
+
guard let buf = ctx.device.makeBuffer(length: padded * 4, options: .storageModeShared) else {
|
|
611
|
+
setError(errorOut, "failed to allocate \(padded * 4) bytes for random")
|
|
612
|
+
return -1
|
|
613
|
+
}
|
|
614
|
+
scratch = buf
|
|
615
|
+
}
|
|
616
|
+
let dest = scratch ?? box.buffer
|
|
617
|
+
|
|
618
|
+
let matDesc = MPSMatrixDescriptor(rows: 1, columns: padded, rowBytes: padded * 4, dataType: .float32)
|
|
619
|
+
let matrix = MPSMatrix(buffer: dest, descriptor: matDesc)
|
|
620
|
+
|
|
621
|
+
guard let cb = ctx.queue.makeCommandBuffer() else {
|
|
622
|
+
setError(errorOut, "failed to create command buffer for random")
|
|
623
|
+
return -1
|
|
624
|
+
}
|
|
625
|
+
ctx.flushBatch()
|
|
626
|
+
rng.encode(commandBuffer: cb, destinationMatrix: matrix)
|
|
627
|
+
cb.commit()
|
|
628
|
+
cb.waitUntilCompleted()
|
|
629
|
+
if let error = cb.error {
|
|
630
|
+
setError(errorOut, error.localizedDescription)
|
|
631
|
+
return -1
|
|
632
|
+
}
|
|
633
|
+
|
|
634
|
+
if let scratch = scratch {
|
|
635
|
+
memcpy(box.buffer.contents(), scratch.contents(), n * 4)
|
|
636
|
+
}
|
|
637
|
+
|
|
638
|
+
if kind == RANDOM_KIND_UNIFORM {
|
|
639
|
+
let ptr = box.buffer.contents().bindMemory(to: Float.self, capacity: n)
|
|
640
|
+
for i in 0..<n { ptr[i] = 1.0 - ptr[i] }
|
|
641
|
+
}
|
|
642
|
+
|
|
643
|
+
return 0
|
|
644
|
+
}
|
|
645
|
+
}
|
|
646
|
+
|
|
647
|
+
// MARK: - FFT (MPSGraph, cuFFT analogue)
|
|
648
|
+
//
|
|
649
|
+
// See fft.rb for kind/direction. R2C = 0, C2R = 1, C2C = 2, FORWARD = 1,
|
|
650
|
+
// INVERSE = -1. Normalization (1/n) is fft.rb's job, so this always uses
|
|
651
|
+
// MPSGraphFFTScalingModeNone (no scaling).
|
|
652
|
+
//
|
|
653
|
+
// Complex numbers are interleaved float32 ([re0, im0, re1, im1, ...]) per the
|
|
654
|
+
// core contract. MPSDataTypeComplexFloat32 is a pair of 2x32-bit (re, im), so
|
|
655
|
+
// a Buffer's raw bytes can be read as complexFloat32 directly — no copy.
|
|
656
|
+
//
|
|
657
|
+
// Graph compilation is expensive, so graphs are cached per shape key (kind,
|
|
658
|
+
// direction, n, batch), the same idea as cuFFT's plan cache.
|
|
659
|
+
|
|
660
|
+
private struct FFTPlanKey: Hashable {
|
|
661
|
+
let kind: Int32
|
|
662
|
+
let direction: Int32
|
|
663
|
+
let n: Int
|
|
664
|
+
let batch: Int
|
|
665
|
+
}
|
|
666
|
+
|
|
667
|
+
private final class FFTPlan {
|
|
668
|
+
let graph: MPSGraph
|
|
669
|
+
let inputTensor: MPSGraphTensor
|
|
670
|
+
let outputTensor: MPSGraphTensor
|
|
671
|
+
let inputShape: [NSNumber]
|
|
672
|
+
let outputShape: [NSNumber]
|
|
673
|
+
let inputDataType: MPSDataType
|
|
674
|
+
let outputDataType: MPSDataType
|
|
675
|
+
|
|
676
|
+
init(graph: MPSGraph, inputTensor: MPSGraphTensor, outputTensor: MPSGraphTensor,
|
|
677
|
+
inputShape: [NSNumber], outputShape: [NSNumber],
|
|
678
|
+
inputDataType: MPSDataType, outputDataType: MPSDataType) {
|
|
679
|
+
self.graph = graph
|
|
680
|
+
self.inputTensor = inputTensor
|
|
681
|
+
self.outputTensor = outputTensor
|
|
682
|
+
self.inputShape = inputShape
|
|
683
|
+
self.outputShape = outputShape
|
|
684
|
+
self.inputDataType = inputDataType
|
|
685
|
+
self.outputDataType = outputDataType
|
|
686
|
+
}
|
|
687
|
+
}
|
|
688
|
+
|
|
689
|
+
private let FFT_KIND_R2C: Int32 = 0
|
|
690
|
+
private let FFT_KIND_C2R: Int32 = 1
|
|
691
|
+
private let FFT_DIR_INVERSE: Int32 = -1
|
|
692
|
+
|
|
693
|
+
private var fftPlanCache: [FFTPlanKey: FFTPlan] = [:]
|
|
694
|
+
private let fftPlanLock = NSLock()
|
|
695
|
+
|
|
696
|
+
private func fftPlan(kind: Int32, direction: Int32, n: Int, batch: Int) -> FFTPlan {
|
|
697
|
+
let key = FFTPlanKey(kind: kind, direction: direction, n: n, batch: batch)
|
|
698
|
+
|
|
699
|
+
fftPlanLock.lock()
|
|
700
|
+
defer { fftPlanLock.unlock() }
|
|
701
|
+
if let cached = fftPlanCache[key] { return cached }
|
|
702
|
+
|
|
703
|
+
let graph = MPSGraph()
|
|
704
|
+
let desc = MPSGraphFFTDescriptor()
|
|
705
|
+
desc.scalingMode = .none
|
|
706
|
+
desc.inverse = direction == FFT_DIR_INVERSE
|
|
707
|
+
|
|
708
|
+
let plan: FFTPlan
|
|
709
|
+
switch kind {
|
|
710
|
+
case FFT_KIND_R2C:
|
|
711
|
+
let inShape: [NSNumber] = [NSNumber(value: batch), NSNumber(value: n)]
|
|
712
|
+
let input = graph.placeholder(shape: inShape, dataType: .float32, name: nil)
|
|
713
|
+
let output = graph.realToHermiteanFFT(input, axes: [1], descriptor: desc, name: nil)
|
|
714
|
+
let outShape: [NSNumber] = [NSNumber(value: batch), NSNumber(value: n / 2 + 1)]
|
|
715
|
+
plan = FFTPlan(graph: graph, inputTensor: input, outputTensor: output,
|
|
716
|
+
inputShape: inShape, outputShape: outShape,
|
|
717
|
+
inputDataType: .float32, outputDataType: .complexFloat32)
|
|
718
|
+
case FFT_KIND_C2R:
|
|
719
|
+
let inSize = n / 2 + 1
|
|
720
|
+
let inShape: [NSNumber] = [NSNumber(value: batch), NSNumber(value: inSize)]
|
|
721
|
+
desc.roundToOddHermitean = n % 2 != 0
|
|
722
|
+
let input = graph.placeholder(shape: inShape, dataType: .complexFloat32, name: nil)
|
|
723
|
+
let output = graph.HermiteanToRealFFT(input, axes: [1], descriptor: desc, name: nil)
|
|
724
|
+
let outShape: [NSNumber] = [NSNumber(value: batch), NSNumber(value: n)]
|
|
725
|
+
plan = FFTPlan(graph: graph, inputTensor: input, outputTensor: output,
|
|
726
|
+
inputShape: inShape, outputShape: outShape,
|
|
727
|
+
inputDataType: .complexFloat32, outputDataType: .float32)
|
|
728
|
+
default: // C2C
|
|
729
|
+
let inShape: [NSNumber] = [NSNumber(value: batch), NSNumber(value: n)]
|
|
730
|
+
let input = graph.placeholder(shape: inShape, dataType: .complexFloat32, name: nil)
|
|
731
|
+
let output = graph.fastFourierTransform(input, axes: [1], descriptor: desc, name: nil)
|
|
732
|
+
plan = FFTPlan(graph: graph, inputTensor: input, outputTensor: output,
|
|
733
|
+
inputShape: inShape, outputShape: inShape,
|
|
734
|
+
inputDataType: .complexFloat32, outputDataType: .complexFloat32)
|
|
735
|
+
}
|
|
736
|
+
|
|
737
|
+
fftPlanCache[key] = plan
|
|
738
|
+
return plan
|
|
739
|
+
}
|
|
740
|
+
|
|
741
|
+
@_cdecl("gpu_bridge_fft")
|
|
742
|
+
public func gpu_bridge_fft(_ inHandle: UnsafeMutableRawPointer, _ outHandle: UnsafeMutableRawPointer,
|
|
743
|
+
_ n: Int, _ batch: Int, _ kind: Int32, _ direction: Int32,
|
|
744
|
+
_ errorOut: UnsafeMutablePointer<UnsafeMutablePointer<CChar>?>?) -> Int32 {
|
|
745
|
+
return autoreleasepool { () -> Int32 in
|
|
746
|
+
let ctx = BridgeContext.shared
|
|
747
|
+
let inBox = Unmanaged<BufferBox>.fromOpaque(inHandle).takeUnretainedValue()
|
|
748
|
+
let outBox = Unmanaged<BufferBox>.fromOpaque(outHandle).takeUnretainedValue()
|
|
749
|
+
|
|
750
|
+
ctx.flushBatch()
|
|
751
|
+
let plan = fftPlan(kind: kind, direction: direction, n: n, batch: batch)
|
|
752
|
+
let inputData = MPSGraphTensorData(inBox.buffer, shape: plan.inputShape, dataType: plan.inputDataType)
|
|
753
|
+
let outputData = MPSGraphTensorData(outBox.buffer, shape: plan.outputShape, dataType: plan.outputDataType)
|
|
754
|
+
|
|
755
|
+
plan.graph.run(with: ctx.queue, feeds: [plan.inputTensor: inputData],
|
|
756
|
+
targetOperations: nil,
|
|
757
|
+
resultsDictionary: [plan.outputTensor: outputData])
|
|
758
|
+
|
|
759
|
+
return 0
|
|
760
|
+
}
|
|
761
|
+
}
|
|
762
|
+
|
|
763
|
+
// MARK: - Conv2D (MPSGraph, cuDNN analogue)
|
|
764
|
+
//
|
|
765
|
+
// The layout is NCHW / weights OIHW per the core contract (see conv.rb). dtype
|
|
766
|
+
// uses the same codes as gemm (0 = float32, 1 = float16, 2 = bfloat16), so
|
|
767
|
+
// mpsDataType(_:) is reused. Unlike gemm, the output dtype follows the input
|
|
768
|
+
// (accumulation precision is left to MPSGraph's implementation).
|
|
769
|
+
//
|
|
770
|
+
// For mode see gpu_bridge_conv2d_backward's calling convention (conv.rb):
|
|
771
|
+
// 0 = input gradient (a=weights, b=grad_output), 1 = weight gradient
|
|
772
|
+
// (a=input, b=grad_output). All three are (tensorA, tensorB) -> output with
|
|
773
|
+
// two inputs and one output, so one ConvPlan struct represents them.
|
|
774
|
+
|
|
775
|
+
private let CONV_OP_FORWARD: Int32 = 0
|
|
776
|
+
private let CONV_OP_BACKWARD_DATA: Int32 = 1
|
|
777
|
+
private let CONV_OP_BACKWARD_FILTER: Int32 = 2
|
|
778
|
+
|
|
779
|
+
private struct ConvPlanKey: Hashable {
|
|
780
|
+
let op: Int32
|
|
781
|
+
let n: Int32, cin: Int32, h: Int32, w: Int32
|
|
782
|
+
let cout: Int32, kh: Int32, kw: Int32
|
|
783
|
+
let padH: Int32, padW: Int32, strideH: Int32, strideW: Int32
|
|
784
|
+
let dilationH: Int32, dilationW: Int32, groups: Int32
|
|
785
|
+
let dtype: Int32
|
|
786
|
+
}
|
|
787
|
+
|
|
788
|
+
private final class ConvPlan {
|
|
789
|
+
let graph: MPSGraph
|
|
790
|
+
let tensorA: MPSGraphTensor
|
|
791
|
+
let tensorB: MPSGraphTensor
|
|
792
|
+
let output: MPSGraphTensor
|
|
793
|
+
let shapeA: [NSNumber]
|
|
794
|
+
let shapeB: [NSNumber]
|
|
795
|
+
let outputShape: [NSNumber]
|
|
796
|
+
let dataType: MPSDataType
|
|
797
|
+
|
|
798
|
+
init(graph: MPSGraph, tensorA: MPSGraphTensor, tensorB: MPSGraphTensor, output: MPSGraphTensor,
|
|
799
|
+
shapeA: [NSNumber], shapeB: [NSNumber], outputShape: [NSNumber], dataType: MPSDataType) {
|
|
800
|
+
self.graph = graph
|
|
801
|
+
self.tensorA = tensorA
|
|
802
|
+
self.tensorB = tensorB
|
|
803
|
+
self.output = output
|
|
804
|
+
self.shapeA = shapeA
|
|
805
|
+
self.shapeB = shapeB
|
|
806
|
+
self.outputShape = outputShape
|
|
807
|
+
self.dataType = dataType
|
|
808
|
+
}
|
|
809
|
+
}
|
|
810
|
+
|
|
811
|
+
private var convPlanCache: [ConvPlanKey: ConvPlan] = [:]
|
|
812
|
+
private let convPlanLock = NSLock()
|
|
813
|
+
|
|
814
|
+
// The Ruby shim registers this as an end proc. MPSGraph plans are held by
|
|
815
|
+
// process-global dictionaries rather than Ruby TypedData, so release them
|
|
816
|
+
// explicitly after queued work has drained. Avoid touching BridgeContext when
|
|
817
|
+
// neither cache was ever populated; shutdown must not initialize Metal merely
|
|
818
|
+
// because the extension was required.
|
|
819
|
+
@_cdecl("gpu_bridge_shutdown")
|
|
820
|
+
public func gpu_bridge_shutdown() {
|
|
821
|
+
fftPlanLock.lock()
|
|
822
|
+
let hasFFTPlans = !fftPlanCache.isEmpty
|
|
823
|
+
fftPlanLock.unlock()
|
|
824
|
+
convPlanLock.lock()
|
|
825
|
+
let hasConvPlans = !convPlanCache.isEmpty
|
|
826
|
+
convPlanLock.unlock()
|
|
827
|
+
guard hasFFTPlans || hasConvPlans else { return }
|
|
828
|
+
|
|
829
|
+
BridgeContext.shared.synchronize()
|
|
830
|
+
autoreleasepool {
|
|
831
|
+
fftPlanLock.lock()
|
|
832
|
+
fftPlanCache.removeAll()
|
|
833
|
+
fftPlanLock.unlock()
|
|
834
|
+
convPlanLock.lock()
|
|
835
|
+
convPlanCache.removeAll()
|
|
836
|
+
convPlanLock.unlock()
|
|
837
|
+
}
|
|
838
|
+
}
|
|
839
|
+
|
|
840
|
+
private func convPlan(op: Int32, n: Int32, cin: Int32, h: Int32, w: Int32,
|
|
841
|
+
cout: Int32, kh: Int32, kw: Int32,
|
|
842
|
+
padH: Int32, padW: Int32, strideH: Int32, strideW: Int32,
|
|
843
|
+
dilationH: Int32, dilationW: Int32, groups: Int32, dtype: Int32) -> ConvPlan {
|
|
844
|
+
let key = ConvPlanKey(op: op, n: n, cin: cin, h: h, w: w, cout: cout, kh: kh, kw: kw,
|
|
845
|
+
padH: padH, padW: padW, strideH: strideH, strideW: strideW,
|
|
846
|
+
dilationH: dilationH, dilationW: dilationW, groups: groups, dtype: dtype)
|
|
847
|
+
|
|
848
|
+
convPlanLock.lock()
|
|
849
|
+
defer { convPlanLock.unlock() }
|
|
850
|
+
if let cached = convPlanCache[key] { return cached }
|
|
851
|
+
|
|
852
|
+
let graph = MPSGraph()
|
|
853
|
+
let mpsType = mpsDataType(dtype)
|
|
854
|
+
let effectiveKh = (Int(kh) - 1) * Int(dilationH) + 1
|
|
855
|
+
let effectiveKw = (Int(kw) - 1) * Int(dilationW) + 1
|
|
856
|
+
let oh = (Int(h) + 2 * Int(padH) - effectiveKh) / Int(strideH) + 1
|
|
857
|
+
let ow = (Int(w) + 2 * Int(padW) - effectiveKw) / Int(strideW) + 1
|
|
858
|
+
|
|
859
|
+
let desc = MPSGraphConvolution2DOpDescriptor(strideInX: Int(strideW), strideInY: Int(strideH),
|
|
860
|
+
dilationRateInX: Int(dilationW), dilationRateInY: Int(dilationH),
|
|
861
|
+
groups: Int(groups),
|
|
862
|
+
paddingLeft: Int(padW), paddingRight: Int(padW),
|
|
863
|
+
paddingTop: Int(padH), paddingBottom: Int(padH),
|
|
864
|
+
paddingStyle: .explicit,
|
|
865
|
+
dataLayout: .NCHW, weightsLayout: .OIHW)!
|
|
866
|
+
|
|
867
|
+
let cinPerGroup = cin / groups
|
|
868
|
+
let srcShape: [NSNumber] = [n, cin, h, w].map { NSNumber(value: $0) }
|
|
869
|
+
let weightsShape: [NSNumber] = [cout, cinPerGroup, kh, kw].map { NSNumber(value: $0) }
|
|
870
|
+
let outShape: [NSNumber] = [n, cout, Int32(oh), Int32(ow)].map { NSNumber(value: $0) }
|
|
871
|
+
|
|
872
|
+
let plan: ConvPlan
|
|
873
|
+
switch op {
|
|
874
|
+
case CONV_OP_BACKWARD_DATA:
|
|
875
|
+
let weights = graph.placeholder(shape: weightsShape, dataType: mpsType, name: nil)
|
|
876
|
+
let gradOutput = graph.placeholder(shape: outShape, dataType: mpsType, name: nil)
|
|
877
|
+
let dx = graph.convolution2DDataGradient(gradOutput, weights: weights, outputShape: srcShape,
|
|
878
|
+
forwardConvolutionDescriptor: desc, name: nil)
|
|
879
|
+
plan = ConvPlan(graph: graph, tensorA: weights, tensorB: gradOutput, output: dx,
|
|
880
|
+
shapeA: weightsShape, shapeB: outShape, outputShape: srcShape, dataType: mpsType)
|
|
881
|
+
case CONV_OP_BACKWARD_FILTER:
|
|
882
|
+
let src = graph.placeholder(shape: srcShape, dataType: mpsType, name: nil)
|
|
883
|
+
let gradOutput = graph.placeholder(shape: outShape, dataType: mpsType, name: nil)
|
|
884
|
+
let dw = graph.convolution2DWeightsGradient(gradOutput, source: src, outputShape: weightsShape,
|
|
885
|
+
forwardConvolutionDescriptor: desc, name: nil)
|
|
886
|
+
plan = ConvPlan(graph: graph, tensorA: src, tensorB: gradOutput, output: dw,
|
|
887
|
+
shapeA: srcShape, shapeB: outShape, outputShape: weightsShape, dataType: mpsType)
|
|
888
|
+
default: // CONV_OP_FORWARD
|
|
889
|
+
let src = graph.placeholder(shape: srcShape, dataType: mpsType, name: nil)
|
|
890
|
+
let weights = graph.placeholder(shape: weightsShape, dataType: mpsType, name: nil)
|
|
891
|
+
let out = graph.convolution2D(src, weights: weights, descriptor: desc, name: nil)
|
|
892
|
+
plan = ConvPlan(graph: graph, tensorA: src, tensorB: weights, output: out,
|
|
893
|
+
shapeA: srcShape, shapeB: weightsShape, outputShape: outShape, dataType: mpsType)
|
|
894
|
+
}
|
|
895
|
+
|
|
896
|
+
convPlanCache[key] = plan
|
|
897
|
+
return plan
|
|
898
|
+
}
|
|
899
|
+
|
|
900
|
+
private func runConvPlan(_ plan: ConvPlan, bufA: MTLBuffer, bufB: MTLBuffer, bufOut: MTLBuffer) {
|
|
901
|
+
let ctx = BridgeContext.shared
|
|
902
|
+
ctx.flushBatch()
|
|
903
|
+
let dataA = MPSGraphTensorData(bufA, shape: plan.shapeA, dataType: plan.dataType)
|
|
904
|
+
let dataB = MPSGraphTensorData(bufB, shape: plan.shapeB, dataType: plan.dataType)
|
|
905
|
+
let dataOut = MPSGraphTensorData(bufOut, shape: plan.outputShape, dataType: plan.dataType)
|
|
906
|
+
|
|
907
|
+
plan.graph.run(with: ctx.queue, feeds: [plan.tensorA: dataA, plan.tensorB: dataB],
|
|
908
|
+
targetOperations: nil,
|
|
909
|
+
resultsDictionary: [plan.output: dataOut])
|
|
910
|
+
}
|
|
911
|
+
|
|
912
|
+
private func runConvPlan(_ plan: ConvPlan, a: UnsafeMutableRawPointer, b: UnsafeMutableRawPointer,
|
|
913
|
+
out: UnsafeMutableRawPointer) {
|
|
914
|
+
let boxA = Unmanaged<BufferBox>.fromOpaque(a).takeUnretainedValue()
|
|
915
|
+
let boxB = Unmanaged<BufferBox>.fromOpaque(b).takeUnretainedValue()
|
|
916
|
+
let boxOut = Unmanaged<BufferBox>.fromOpaque(out).takeUnretainedValue()
|
|
917
|
+
runConvPlan(plan, bufA: boxA.buffer, bufB: boxB.buffer, bufOut: boxOut.buffer)
|
|
918
|
+
}
|
|
919
|
+
|
|
920
|
+
// MPSGraph's convolution **silently breaks** when output channels exceed 65536
|
|
921
|
+
// (rows past 65536 are never written and the first few rows differ; no error
|
|
922
|
+
// or warning).
|
|
923
|
+
//
|
|
924
|
+
// Up to exactly 65536 it is correct, so cout is split into chunks of that
|
|
925
|
+
// size. Buffers are storageModeShared, so slicing is a memcpy (graph.run is
|
|
926
|
+
// synchronous).
|
|
927
|
+
private let CONV_MAX_COUT: Int32 = 65536
|
|
928
|
+
|
|
929
|
+
private func convChunkBuffer(_ bytes: Int, _ what: String,
|
|
930
|
+
_ errorOut: UnsafeMutablePointer<UnsafeMutablePointer<CChar>?>?) -> MTLBuffer? {
|
|
931
|
+
guard let buf = BridgeContext.shared.device.makeBuffer(length: max(bytes, 1), options: .storageModeShared) else {
|
|
932
|
+
setError(errorOut, "failed to allocate \(bytes) bytes for the chunked conv2d \(what)")
|
|
933
|
+
return nil
|
|
934
|
+
}
|
|
935
|
+
return buf
|
|
936
|
+
}
|
|
937
|
+
|
|
938
|
+
// Slice / write back along cout of [n, cout, plane]. plane is in bytes
|
|
939
|
+
// (oh*ow*elemSize), not elements.
|
|
940
|
+
private func convCopyChannels(dst: UnsafeMutableRawPointer, dstChannels: Int, dstOffset: Int,
|
|
941
|
+
src: UnsafeMutableRawPointer, srcChannels: Int, srcOffset: Int,
|
|
942
|
+
count: Int, batch: Int, planeBytes: Int) {
|
|
943
|
+
for i in 0..<batch {
|
|
944
|
+
let d = dst + (((i * dstChannels) + dstOffset) * planeBytes)
|
|
945
|
+
let s = src + (((i * srcChannels) + srcOffset) * planeBytes)
|
|
946
|
+
memcpy(d, s, count * planeBytes)
|
|
947
|
+
}
|
|
948
|
+
}
|
|
949
|
+
|
|
950
|
+
// dst += src elementwise. Only backward_data has chunk results landing on the
|
|
951
|
+
// same dx (splitting by output channel makes the input gradient the sum over
|
|
952
|
+
// all chunks), so it must accumulate rather than write.
|
|
953
|
+
private func convAccumulate(dst: UnsafeMutableRawPointer, src: UnsafeMutableRawPointer,
|
|
954
|
+
count: Int, dtype: Int32) {
|
|
955
|
+
if dtype == 1 {
|
|
956
|
+
let d = dst.bindMemory(to: Float16.self, capacity: count)
|
|
957
|
+
let s = src.bindMemory(to: Float16.self, capacity: count)
|
|
958
|
+
for i in 0..<count { d[i] += s[i] }
|
|
959
|
+
} else {
|
|
960
|
+
let d = dst.bindMemory(to: Float.self, capacity: count)
|
|
961
|
+
let s = src.bindMemory(to: Float.self, capacity: count)
|
|
962
|
+
for i in 0..<count { d[i] += s[i] }
|
|
963
|
+
}
|
|
964
|
+
}
|
|
965
|
+
|
|
966
|
+
// Taken only when cout > CONV_MAX_COUT. Ops differ only in what they slice and
|
|
967
|
+
// write back; the split (cout in steps of CONV_MAX_COUT) is shared.
|
|
968
|
+
private func runConvChunked(op: Int32, a: UnsafeMutableRawPointer, b: UnsafeMutableRawPointer,
|
|
969
|
+
out: UnsafeMutableRawPointer,
|
|
970
|
+
n: Int32, cin: Int32, h: Int32, width: Int32,
|
|
971
|
+
cout: Int32, kh: Int32, kw: Int32,
|
|
972
|
+
padH: Int32, padW: Int32, strideH: Int32, strideW: Int32,
|
|
973
|
+
dilationH: Int32, dilationW: Int32, groups: Int32,
|
|
974
|
+
dtype: Int32,
|
|
975
|
+
errorOut: UnsafeMutablePointer<UnsafeMutablePointer<CChar>?>?) -> Int32 {
|
|
976
|
+
// Grouped convolution (groups > 1) mixes up channels unless chunk
|
|
977
|
+
// boundaries align with group boundaries. A grouped conv with cout > 65536
|
|
978
|
+
// is unrealistic, so reject it explicitly rather than break silently.
|
|
979
|
+
if groups != 1 {
|
|
980
|
+
setError(errorOut, "conv2d: groups > 1 with cout > \(CONV_MAX_COUT) is not supported " +
|
|
981
|
+
"(chunked cout path only handles groups == 1)")
|
|
982
|
+
return -1
|
|
983
|
+
}
|
|
984
|
+
|
|
985
|
+
let boxA = Unmanaged<BufferBox>.fromOpaque(a).takeUnretainedValue()
|
|
986
|
+
let boxB = Unmanaged<BufferBox>.fromOpaque(b).takeUnretainedValue()
|
|
987
|
+
let boxOut = Unmanaged<BufferBox>.fromOpaque(out).takeUnretainedValue()
|
|
988
|
+
|
|
989
|
+
// This path slices its operands with memcpy on the host, so queue order is
|
|
990
|
+
// not enough here: everything that writes them must have finished.
|
|
991
|
+
BridgeContext.shared.synchronize()
|
|
992
|
+
|
|
993
|
+
let es = elemSize(dtype)
|
|
994
|
+
let effectiveKh = (Int(kh) - 1) * Int(dilationH) + 1
|
|
995
|
+
let effectiveKw = (Int(kw) - 1) * Int(dilationW) + 1
|
|
996
|
+
let oh = (Int(h) + 2 * Int(padH) - effectiveKh) / Int(strideH) + 1
|
|
997
|
+
let ow = (Int(width) + 2 * Int(padW) - effectiveKw) / Int(strideW) + 1
|
|
998
|
+
let outPlane = oh * ow * es // bytes per output channel
|
|
999
|
+
let weightPlane = Int(cin) * Int(kh) * Int(kw) * es // bytes per output channel of the weight
|
|
1000
|
+
let batch = Int(n)
|
|
1001
|
+
|
|
1002
|
+
// backward_data accumulates, so zero the destination first.
|
|
1003
|
+
if op == CONV_OP_BACKWARD_DATA {
|
|
1004
|
+
memset(boxOut.buffer.contents(), 0, batch * Int(cin) * Int(h) * Int(width) * es)
|
|
1005
|
+
}
|
|
1006
|
+
|
|
1007
|
+
var done: Int32 = 0
|
|
1008
|
+
while done < cout {
|
|
1009
|
+
let chunk = min(CONV_MAX_COUT, cout - done)
|
|
1010
|
+
let plan = convPlan(op: op, n: n, cin: cin, h: h, w: width,
|
|
1011
|
+
cout: chunk, kh: kh, kw: kw, padH: padH, padW: padW,
|
|
1012
|
+
strideH: strideH, strideW: strideW,
|
|
1013
|
+
dilationH: dilationH, dilationW: dilationW, groups: 1, dtype: dtype)
|
|
1014
|
+
|
|
1015
|
+
switch op {
|
|
1016
|
+
case CONV_OP_BACKWARD_DATA:
|
|
1017
|
+
// a = weight [cout, cin, kh, kw] (contiguous), b = output gradient [n, cout, oh, ow]
|
|
1018
|
+
guard let wTmp = convChunkBuffer(Int(chunk) * weightPlane, "weights", errorOut),
|
|
1019
|
+
let gTmp = convChunkBuffer(batch * Int(chunk) * outPlane, "grad_output", errorOut),
|
|
1020
|
+
let dxTmp = convChunkBuffer(batch * Int(cin) * Int(h) * Int(width) * es, "grad_input", errorOut)
|
|
1021
|
+
else { return -1 }
|
|
1022
|
+
memcpy(wTmp.contents(), boxA.buffer.contents() + Int(done) * weightPlane, Int(chunk) * weightPlane)
|
|
1023
|
+
convCopyChannels(dst: gTmp.contents(), dstChannels: Int(chunk), dstOffset: 0,
|
|
1024
|
+
src: boxB.buffer.contents(), srcChannels: Int(cout), srcOffset: Int(done),
|
|
1025
|
+
count: Int(chunk), batch: batch, planeBytes: outPlane)
|
|
1026
|
+
runConvPlan(plan, bufA: wTmp, bufB: gTmp, bufOut: dxTmp)
|
|
1027
|
+
convAccumulate(dst: boxOut.buffer.contents(), src: dxTmp.contents(),
|
|
1028
|
+
count: batch * Int(cin) * Int(h) * Int(width), dtype: dtype)
|
|
1029
|
+
|
|
1030
|
+
case CONV_OP_BACKWARD_FILTER:
|
|
1031
|
+
// a = input (all of it), b = output gradient, output = weight gradient [cout, cin, kh, kw]
|
|
1032
|
+
guard let gTmp = convChunkBuffer(batch * Int(chunk) * outPlane, "grad_output", errorOut),
|
|
1033
|
+
let dwTmp = convChunkBuffer(Int(chunk) * weightPlane, "grad_weight", errorOut)
|
|
1034
|
+
else { return -1 }
|
|
1035
|
+
convCopyChannels(dst: gTmp.contents(), dstChannels: Int(chunk), dstOffset: 0,
|
|
1036
|
+
src: boxB.buffer.contents(), srcChannels: Int(cout), srcOffset: Int(done),
|
|
1037
|
+
count: Int(chunk), batch: batch, planeBytes: outPlane)
|
|
1038
|
+
runConvPlan(plan, bufA: boxA.buffer, bufB: gTmp, bufOut: dwTmp)
|
|
1039
|
+
memcpy(boxOut.buffer.contents() + Int(done) * weightPlane, dwTmp.contents(), Int(chunk) * weightPlane)
|
|
1040
|
+
|
|
1041
|
+
default: // CONV_OP_FORWARD
|
|
1042
|
+
// a = input (all of it), b = weight [cout, cin, kh, kw] (contiguous)
|
|
1043
|
+
guard let wTmp = convChunkBuffer(Int(chunk) * weightPlane, "weights", errorOut),
|
|
1044
|
+
let yTmp = convChunkBuffer(batch * Int(chunk) * outPlane, "output", errorOut)
|
|
1045
|
+
else { return -1 }
|
|
1046
|
+
memcpy(wTmp.contents(), boxB.buffer.contents() + Int(done) * weightPlane, Int(chunk) * weightPlane)
|
|
1047
|
+
runConvPlan(plan, bufA: boxA.buffer, bufB: wTmp, bufOut: yTmp)
|
|
1048
|
+
convCopyChannels(dst: boxOut.buffer.contents(), dstChannels: Int(cout), dstOffset: Int(done),
|
|
1049
|
+
src: yTmp.contents(), srcChannels: Int(chunk), srcOffset: 0,
|
|
1050
|
+
count: Int(chunk), batch: batch, planeBytes: outPlane)
|
|
1051
|
+
}
|
|
1052
|
+
|
|
1053
|
+
done += chunk
|
|
1054
|
+
}
|
|
1055
|
+
return 0
|
|
1056
|
+
}
|
|
1057
|
+
|
|
1058
|
+
@_cdecl("gpu_bridge_conv2d")
|
|
1059
|
+
public func gpu_bridge_conv2d(_ x: UnsafeMutableRawPointer, _ w: UnsafeMutableRawPointer, _ y: UnsafeMutableRawPointer,
|
|
1060
|
+
_ n: Int32, _ cin: Int32, _ h: Int32, _ width: Int32,
|
|
1061
|
+
_ cout: Int32, _ kh: Int32, _ kw: Int32,
|
|
1062
|
+
_ padH: Int32, _ padW: Int32, _ strideH: Int32, _ strideW: Int32,
|
|
1063
|
+
_ dilationH: Int32, _ dilationW: Int32, _ groups: Int32,
|
|
1064
|
+
_ dtype: Int32,
|
|
1065
|
+
_ errorOut: UnsafeMutablePointer<UnsafeMutablePointer<CChar>?>?) -> Int32 {
|
|
1066
|
+
return autoreleasepool { () -> Int32 in
|
|
1067
|
+
if cout > CONV_MAX_COUT {
|
|
1068
|
+
return runConvChunked(op: CONV_OP_FORWARD, a: x, b: w, out: y,
|
|
1069
|
+
n: n, cin: cin, h: h, width: width, cout: cout, kh: kh, kw: kw,
|
|
1070
|
+
padH: padH, padW: padW, strideH: strideH, strideW: strideW,
|
|
1071
|
+
dilationH: dilationH, dilationW: dilationW, groups: groups,
|
|
1072
|
+
dtype: dtype, errorOut: errorOut)
|
|
1073
|
+
}
|
|
1074
|
+
let plan = convPlan(op: CONV_OP_FORWARD, n: n, cin: cin, h: h, w: width,
|
|
1075
|
+
cout: cout, kh: kh, kw: kw, padH: padH, padW: padW,
|
|
1076
|
+
strideH: strideH, strideW: strideW,
|
|
1077
|
+
dilationH: dilationH, dilationW: dilationW, groups: groups, dtype: dtype)
|
|
1078
|
+
runConvPlan(plan, a: x, b: w, out: y)
|
|
1079
|
+
return 0
|
|
1080
|
+
}
|
|
1081
|
+
}
|
|
1082
|
+
|
|
1083
|
+
@_cdecl("gpu_bridge_conv2d_backward")
|
|
1084
|
+
public func gpu_bridge_conv2d_backward(_ a: UnsafeMutableRawPointer, _ b: UnsafeMutableRawPointer,
|
|
1085
|
+
_ out: UnsafeMutableRawPointer,
|
|
1086
|
+
_ n: Int32, _ cin: Int32, _ h: Int32, _ width: Int32,
|
|
1087
|
+
_ cout: Int32, _ kh: Int32, _ kw: Int32,
|
|
1088
|
+
_ padH: Int32, _ padW: Int32, _ strideH: Int32, _ strideW: Int32,
|
|
1089
|
+
_ dilationH: Int32, _ dilationW: Int32, _ groups: Int32,
|
|
1090
|
+
_ dtype: Int32, _ mode: Int32,
|
|
1091
|
+
_ errorOut: UnsafeMutablePointer<UnsafeMutablePointer<CChar>?>?) -> Int32 {
|
|
1092
|
+
return autoreleasepool { () -> Int32 in
|
|
1093
|
+
let op = mode == 0 ? CONV_OP_BACKWARD_DATA : CONV_OP_BACKWARD_FILTER
|
|
1094
|
+
if cout > CONV_MAX_COUT {
|
|
1095
|
+
return runConvChunked(op: op, a: a, b: b, out: out,
|
|
1096
|
+
n: n, cin: cin, h: h, width: width, cout: cout, kh: kh, kw: kw,
|
|
1097
|
+
padH: padH, padW: padW, strideH: strideH, strideW: strideW,
|
|
1098
|
+
dilationH: dilationH, dilationW: dilationW, groups: groups,
|
|
1099
|
+
dtype: dtype, errorOut: errorOut)
|
|
1100
|
+
}
|
|
1101
|
+
let plan = convPlan(op: op, n: n, cin: cin, h: h, w: width,
|
|
1102
|
+
cout: cout, kh: kh, kw: kw, padH: padH, padW: padW,
|
|
1103
|
+
strideH: strideH, strideW: strideW,
|
|
1104
|
+
dilationH: dilationH, dilationW: dilationW, groups: groups, dtype: dtype)
|
|
1105
|
+
runConvPlan(plan, a: a, b: b, out: out)
|
|
1106
|
+
return 0
|
|
1107
|
+
}
|
|
1108
|
+
}
|
|
1109
|
+
|
|
1110
|
+
// MARK: - Dispatch
|
|
1111
|
+
|
|
1112
|
+
// args is an array of kinds: kind=0 is an MTLBuffer handle, kind=1 raw setBytes bytes.
|
|
1113
|
+
@_cdecl("gpu_bridge_dispatch")
|
|
1114
|
+
public func gpu_bridge_dispatch(_ pipelineHandle: UnsafeMutableRawPointer,
|
|
1115
|
+
_ argKinds: UnsafePointer<Int32>?,
|
|
1116
|
+
_ argPtrs: UnsafePointer<UnsafeMutableRawPointer?>?,
|
|
1117
|
+
_ argLens: UnsafePointer<Int>?,
|
|
1118
|
+
_ argCount: Int32,
|
|
1119
|
+
_ gx: Int, _ gy: Int, _ gz: Int,
|
|
1120
|
+
_ bx: Int, _ by: Int, _ bz: Int,
|
|
1121
|
+
_ wait: Int32,
|
|
1122
|
+
_ errorOut: UnsafeMutablePointer<UnsafeMutablePointer<CChar>?>?) -> Int32 {
|
|
1123
|
+
return autoreleasepool { () -> Int32 in
|
|
1124
|
+
let ctx = BridgeContext.shared
|
|
1125
|
+
let pso = Unmanaged<PipelineBox>.fromOpaque(pipelineHandle).takeUnretainedValue().pso
|
|
1126
|
+
|
|
1127
|
+
let cb: MTLCommandBuffer?
|
|
1128
|
+
let encoder: MTLComputeCommandEncoder
|
|
1129
|
+
if wait == 0 {
|
|
1130
|
+
cb = nil
|
|
1131
|
+
guard let shared = ctx.encoderForBatch() else {
|
|
1132
|
+
setError(errorOut, "failed to create batched command buffer/encoder")
|
|
1133
|
+
return -1
|
|
1134
|
+
}
|
|
1135
|
+
encoder = shared
|
|
1136
|
+
} else {
|
|
1137
|
+
// Preserve queue order if an async batch is still open.
|
|
1138
|
+
ctx.synchronize()
|
|
1139
|
+
guard let commandBuffer = ctx.queue.makeCommandBuffer(),
|
|
1140
|
+
let synchronous = commandBuffer.makeComputeCommandEncoder() else {
|
|
1141
|
+
setError(errorOut, "failed to create command buffer/encoder")
|
|
1142
|
+
return -1
|
|
1143
|
+
}
|
|
1144
|
+
cb = commandBuffer
|
|
1145
|
+
encoder = synchronous
|
|
1146
|
+
}
|
|
1147
|
+
encoder.setComputePipelineState(pso)
|
|
1148
|
+
|
|
1149
|
+
for i in 0..<Int(argCount) {
|
|
1150
|
+
guard let ptr = argPtrs?[i] else {
|
|
1151
|
+
if wait != 0 { encoder.endEncoding() }
|
|
1152
|
+
setError(errorOut, "argument \(i) is null")
|
|
1153
|
+
return -1
|
|
1154
|
+
}
|
|
1155
|
+
if argKinds?[i] == 0 {
|
|
1156
|
+
let box = Unmanaged<BufferBox>.fromOpaque(ptr).takeUnretainedValue()
|
|
1157
|
+
// For buffer arguments argLens carries the view's byte offset
|
|
1158
|
+
// (see gpu_bridge.c); 0 for an ordinary buffer.
|
|
1159
|
+
encoder.setBuffer(box.buffer, offset: argLens?[i] ?? 0, index: i)
|
|
1160
|
+
} else {
|
|
1161
|
+
encoder.setBytes(ptr, length: argLens?[i] ?? 0, index: i)
|
|
1162
|
+
}
|
|
1163
|
+
}
|
|
1164
|
+
|
|
1165
|
+
encoder.dispatchThreadgroups(MTLSize(width: gx, height: gy, depth: gz),
|
|
1166
|
+
threadsPerThreadgroup: MTLSize(width: bx, height: by, depth: bz))
|
|
1167
|
+
guard let cb else {
|
|
1168
|
+
ctx.didEncodeBatchDispatch()
|
|
1169
|
+
return 0
|
|
1170
|
+
}
|
|
1171
|
+
encoder.endEncoding()
|
|
1172
|
+
cb.commit()
|
|
1173
|
+
if wait != 0 {
|
|
1174
|
+
cb.waitUntilCompleted()
|
|
1175
|
+
if let error = cb.error {
|
|
1176
|
+
setError(errorOut, error.localizedDescription)
|
|
1177
|
+
return -1
|
|
1178
|
+
}
|
|
1179
|
+
} else {
|
|
1180
|
+
ctx.track(cb)
|
|
1181
|
+
}
|
|
1182
|
+
return 0
|
|
1183
|
+
}
|
|
1184
|
+
}
|
|
1185
|
+
|
|
1186
|
+
@_cdecl("gpu_bridge_synchronize")
|
|
1187
|
+
public func gpu_bridge_synchronize() {
|
|
1188
|
+
BridgeContext.shared.synchronize()
|
|
1189
|
+
}
|
|
1190
|
+
|
|
1191
|
+
// Optional ABI: synchronize, and report the first asynchronous command buffer
|
|
1192
|
+
// that failed since the last call. GEMM and wait:false dispatches return
|
|
1193
|
+
// before they run, so this boundary is the only place their failure can
|
|
1194
|
+
// surface.
|
|
1195
|
+
@_cdecl("gpu_bridge_synchronize_checked")
|
|
1196
|
+
public func gpu_bridge_synchronize_checked(
|
|
1197
|
+
_ errorOut: UnsafeMutablePointer<UnsafeMutablePointer<CChar>?>?) -> Int32 {
|
|
1198
|
+
let ctx = BridgeContext.shared
|
|
1199
|
+
ctx.synchronize()
|
|
1200
|
+
guard let error = ctx.takeAsyncError() else { return 0 }
|
|
1201
|
+
setError(errorOut, error)
|
|
1202
|
+
return -1
|
|
1203
|
+
}
|