tilework-cuda 0.18.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,1236 @@
1
+ // tilework native bridge (CUDA)
2
+ //
3
+ // Implements the gpu_bridge_* C ABI declared by gpu_bridge.c with the CUDA
4
+ // Driver API + NVRTC. Same contract as bridge.swift (Metal,
5
+ // ../../../metal/ext/tilework/metal/): mechanism only, with policy on the
6
+ // Ruby side (core/lib/tilework.rb).
7
+ //
8
+ // Loading run-time-compiled PTX needs the Driver API (cu*), so memory
9
+ // allocation uses the Driver API too, for uniformity.
10
+ //
11
+ // NVRTC compiles kernels at run time, so this file has no __global__. It is
12
+ // pure host code and builds with the host C++ compiler rather than nvcc
13
+ // (see extconf.rb).
14
+
15
+ #include <cuda.h>
16
+ #include <nvrtc.h>
17
+ // The vendor libraries are optional: extconf.rb defines TILEWORK_HAS_<LIB>
18
+ // only for the ones it finds, and a missing one leaves its gpu_bridge_*
19
+ // function undefined, so Ruby sees the weak symbol absent and falls back.
20
+ #ifdef TILEWORK_HAS_CUBLAS
21
+ #include <cublas_v2.h>
22
+ #endif
23
+ #ifdef TILEWORK_HAS_CURAND
24
+ #include <curand.h>
25
+ #endif
26
+ #ifdef TILEWORK_HAS_CUFFT
27
+ #include <cufft.h>
28
+ #endif
29
+ #ifdef TILEWORK_HAS_CUDNN
30
+ #include <cudnn.h>
31
+ #endif
32
+
33
+ #include <fcntl.h>
34
+ #include <unistd.h>
35
+ #include <errno.h>
36
+
37
+ #include <stdint.h>
38
+ #include <stdio.h>
39
+ #include <stdlib.h>
40
+ #include <string.h>
41
+
42
+ #include <atomic>
43
+ #include <map>
44
+ #include <string>
45
+ #include <tuple>
46
+ #include <thread>
47
+ #include <vector>
48
+
49
+ namespace {
50
+
51
+ CUcontext g_ctx = nullptr;
52
+ CUdevice g_dev = 0;
53
+ bool g_init_failed = false;
54
+ // A compute-only stream. On the default stream, cuCtxSynchronize waits for
55
+ // every stream in the context — including offload's transfer stream — so
56
+ // prefetch never overlapped compute (raising depth stayed at 1.0x in
57
+ // measurements). Compute and transfers use separate streams and waits are
58
+ // per stream.
59
+ CUstream g_compute = nullptr;
60
+ // Stream-ordered memory pool (CUDA 11.2+). cuMemFreeAsync returns memory to
61
+ // the pool rather than the OS, so the next same-size allocation returns at
62
+ // once. On Jetson's unified memory cuMemAlloc involves page mapping and is
63
+ // very slow; a 16 MB allocation measured 3.208 ms → 0.136 ms (23.6x).
64
+ //
65
+ // Where unavailable, g_use_pool stays false and cuMemAlloc/cuMemFree are used.
66
+ // **Allocation and free must always take the same path**, so the choice is
67
+ // made once at initialization and never changes (mixing them crashes).
68
+ bool g_use_pool = false;
69
+ CUmemoryPool g_pool = nullptr;
70
+
71
+ #ifdef TILEWORK_HAS_CUBLAS
72
+ // cuBLAS handle, created lazily (creation is expensive) and reused. Orders of
73
+ // magnitude faster than our WMMA kernel (core/lib/tilework/math/tensor_core.rb).
74
+ cublasHandle_t g_blas = nullptr;
75
+ bool g_blas_failed = false;
76
+
77
+ bool ensure_blas() {
78
+ if (g_blas) return true;
79
+ if (g_blas_failed) return false;
80
+ if (cublasCreate(&g_blas) != CUBLAS_STATUS_SUCCESS) {
81
+ g_blas = nullptr;
82
+ g_blas_failed = true;
83
+ return false;
84
+ }
85
+ // ★ Bind to the compute stream. The pool is stream-ordered, so on the
86
+ // default stream ordering against allocation / zero-fill is not guaranteed.
87
+ cublasSetStream(g_blas, g_compute);
88
+ return true;
89
+ }
90
+ #endif // TILEWORK_HAS_CUBLAS
91
+
92
+ char *dup_cstr(const char *s) {
93
+ // The caller (gpu_bridge.c) releases it with free(), so allocate with malloc.
94
+ size_t n = strlen(s) + 1;
95
+ char *p = (char *)malloc(n);
96
+ if (p) memcpy(p, s, n);
97
+ return p;
98
+ }
99
+
100
+ void set_error(char **error_out, CUresult e, const char *what) {
101
+ if (!error_out) return;
102
+ const char *detail = nullptr;
103
+ cuGetErrorString(e, &detail);
104
+ char buf[512];
105
+ snprintf(buf, sizeof(buf), "%s: %s", what, detail ? detail : "unknown CUDA error");
106
+ *error_out = dup_cstr(buf);
107
+ }
108
+
109
+ // cuInit + context creation happen once; later calls just make it current
110
+ // again (Ruby assumes a single queue, so one context and one stream suffice).
111
+ bool ensure_ctx() {
112
+ if (g_ctx) {
113
+ cuCtxSetCurrent(g_ctx);
114
+ return true;
115
+ }
116
+ if (g_init_failed) return false;
117
+
118
+ if (cuInit(0) != CUDA_SUCCESS || cuDeviceGet(&g_dev, 0) != CUDA_SUCCESS) {
119
+ g_init_failed = true;
120
+ return false;
121
+ }
122
+ if (cuCtxCreate(&g_ctx, 0, g_dev) != CUDA_SUCCESS) {
123
+ g_ctx = nullptr;
124
+ g_init_failed = true;
125
+ return false;
126
+ }
127
+ if (cuStreamCreate(&g_compute, CU_STREAM_NON_BLOCKING) != CUDA_SUCCESS) {
128
+ // cuCtxCreate succeeded, so this function owns a live context even though
129
+ // initialization as a whole failed. Dropping only the pointer leaks all
130
+ // driver-side context state for the lifetime of the process.
131
+ cuCtxDestroy(g_ctx);
132
+ g_ctx = nullptr;
133
+ g_compute = nullptr;
134
+ g_init_failed = true;
135
+ return false;
136
+ }
137
+
138
+ int pools_supported = 0;
139
+ cuDeviceGetAttribute(&pools_supported, CU_DEVICE_ATTRIBUTE_MEMORY_POOLS_SUPPORTED, g_dev);
140
+ if (pools_supported && cuDeviceGetDefaultMemPool(&g_pool, g_dev) == CUDA_SUCCESS) {
141
+ // A pool hoarding without limit starves other allocations into OOM on
142
+ // unified-memory machines. Cap it at 1/4 of free memory (anything beyond
143
+ // goes back to the OS).
144
+ size_t free_bytes = 0, total_bytes = 0;
145
+ cuMemGetInfo(&free_bytes, &total_bytes);
146
+ cuuint64_t threshold = (cuuint64_t)(free_bytes / 4);
147
+ cuMemPoolSetAttribute(g_pool, CU_MEMPOOL_ATTR_RELEASE_THRESHOLD, &threshold);
148
+ g_use_pool = true;
149
+ }
150
+ return true;
151
+ }
152
+
153
+ // Offload chunk size. 16 MB by default (a middle size balancing speedup,
154
+ // hiding quality and synchronization overhead).
155
+ static const size_t OFFLOAD_CHUNK = 16u << 20;
156
+ // Header length at the start of a file; the body follows it.
157
+ static const long OFFLOAD_HEADER = 4096;
158
+
159
+ struct CuPipeline {
160
+ CUmodule mod;
161
+ CUfunction fn;
162
+ };
163
+ struct HostBacking { void *data; long bytes; };
164
+
165
+ // NVRTC does not bundle mma.h (measured on CUDA 12.6: without `-I`, "could
166
+ // not open source file mma.h"), so CUDA's include directory is always passed
167
+ // for the Tensor Core path. Passing header text as strings is an alternative,
168
+ // but -I is confirmed to work on 12.6.
169
+ const char *cuda_include_dir() {
170
+ static std::string dir;
171
+ if (dir.empty()) {
172
+ const char *home = getenv("CUDA_HOME");
173
+ dir = std::string((home && *home) ? home : "/usr/local/cuda") + "/include";
174
+ }
175
+ return dir.c_str();
176
+ }
177
+
178
+ // The PTX target architecture is built from the device rather than fixed at
179
+ // build time (compute_87 on a Jetson Orin, for example).
180
+ void arch_option(char *out, size_t len) {
181
+ int major = 0, minor = 0;
182
+ cuDeviceGetAttribute(&major, CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR, g_dev);
183
+ cuDeviceGetAttribute(&minor, CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MINOR, g_dev);
184
+ if (major == 0) { major = 5; minor = 0; } // on failure rely on PTX forward compatibility
185
+ snprintf(out, len, "--gpu-architecture=compute_%d%d", major, minor);
186
+ }
187
+
188
+ } // namespace
189
+
190
+ extern "C" {
191
+
192
+ char *gpu_bridge_device_name(void) {
193
+ if (!ensure_ctx()) return nullptr;
194
+ char name[256];
195
+ if (cuDeviceGetName(name, sizeof(name), g_dev) != CUDA_SUCCESS) return nullptr;
196
+ return dup_cstr(name);
197
+ }
198
+
199
+ int gpu_bridge_device_available(void) {
200
+ return ensure_ctx() ? 1 : 0;
201
+ }
202
+
203
+ void *gpu_bridge_buffer_alloc(long bytes) {
204
+ if (!ensure_ctx()) return nullptr;
205
+
206
+ CUdeviceptr dptr = 0;
207
+ if (g_use_pool) {
208
+ // Allocate on the compute stream. Every later operation on this pointer
209
+ // must be on g_compute or explicitly synchronized with it (a constraint of
210
+ // stream-ordered allocation).
211
+ CUresult e = cuMemAllocAsync(&dptr, (size_t)bytes, g_compute);
212
+ if (e != CUDA_SUCCESS) {
213
+ // Return what the pool holds to the OS and retry once; without this the
214
+ // state "free memory exists but allocation fails" is possible.
215
+ cuStreamSynchronize(g_compute);
216
+ cuMemPoolTrimTo(g_pool, 0);
217
+ e = cuMemAllocAsync(&dptr, (size_t)bytes, g_compute);
218
+ }
219
+ if (e != CUDA_SUCCESS) return nullptr;
220
+ } else if (cuMemAlloc(&dptr, (size_t)bytes) != CUDA_SUCCESS) {
221
+ return nullptr;
222
+ }
223
+
224
+ // Zero-initialize. The pool returns regions with old contents, so without
225
+ // this code assuming "a new buffer is zero" would read different values each
226
+ // run (Metal's storageModeShared buffers are effectively zero-initialized,
227
+ // and this keeps behavior the same across backends). The async version is
228
+ // used because synchronizing here would undo the faster allocation.
229
+ if (g_use_pool) {
230
+ cuMemsetD8Async(dptr, 0, (size_t)bytes, g_compute);
231
+ } else {
232
+ cuMemsetD8(dptr, 0, (size_t)bytes);
233
+ }
234
+
235
+ // CUdeviceptr is a 64-bit integer; it round-trips as a void* handle.
236
+ return (void *)(uintptr_t)dptr;
237
+ }
238
+
239
+ void gpu_bridge_buffer_write(void *handle, const void *src, long bytes, long offset) {
240
+ if (!ensure_ctx()) return;
241
+ CUdeviceptr dptr = (CUdeviceptr)(uintptr_t)handle;
242
+ // Done on the same g_compute as allocation and zero-fill. g_compute is
243
+ // NON_BLOCKING, so on the default stream implicit synchronization would not
244
+ // apply and the write could land before allocation or zero-fill finishes.
245
+ cuMemcpyHtoDAsync(dptr + (size_t)offset, src, (size_t)bytes, g_compute);
246
+ cuStreamSynchronize(g_compute); // Ruby expects the write to be complete
247
+ }
248
+
249
+ void gpu_bridge_buffer_read(void *handle, void *dst, long bytes, long offset) {
250
+ if (!ensure_ctx()) return;
251
+ CUdeviceptr dptr = (CUdeviceptr)(uintptr_t)handle;
252
+ // Read on the same g_compute as allocation, zero-fill and kernels so the
253
+ // value is right even if the previous launch_async has not finished (the
254
+ // same semantics as Metal's contents seeing committed buffers). Only the
255
+ // compute stream is waited on — no need to wait for in-flight offload transfers.
256
+ cuMemcpyDtoHAsync(dst, dptr + (size_t)offset, (size_t)bytes, g_compute);
257
+ cuStreamSynchronize(g_compute);
258
+ }
259
+
260
+ void gpu_bridge_buffer_free(void *handle) {
261
+ if (!handle) return;
262
+ if (!ensure_ctx()) return;
263
+
264
+ // Return it through the same path it was allocated by; mixing them crashes,
265
+ // hence the branch on g_use_pool.
266
+ if (g_use_pool) {
267
+ cuMemFreeAsync((CUdeviceptr)(uintptr_t)handle, g_compute);
268
+ } else {
269
+ cuMemFree((CUdeviceptr)(uintptr_t)handle);
270
+ }
271
+ }
272
+
273
+ int gpu_bridge_host_backing_available(void) { return ensure_ctx() ? 1 : 0; }
274
+ void *gpu_bridge_host_backing_alloc(long bytes) {
275
+ if (bytes <= 0) return nullptr;
276
+ auto *backing = static_cast<HostBacking *>(malloc(sizeof(HostBacking)));
277
+ if (!backing) return nullptr;
278
+ backing->data = malloc(static_cast<size_t>(bytes));
279
+ backing->bytes = bytes;
280
+ if (!backing->data) { free(backing); return nullptr; }
281
+ return backing;
282
+ }
283
+ void gpu_bridge_host_backing_free(void *handle) {
284
+ auto *backing = static_cast<HostBacking *>(handle);
285
+ if (!backing) return;
286
+ free(backing->data);
287
+ free(backing);
288
+ }
289
+ int gpu_bridge_host_backing_store(void *host, void *device, long bytes, long offset, char **error_out) {
290
+ auto *backing = static_cast<HostBacking *>(host);
291
+ if (!backing || !device || bytes < 0 || offset < 0 || offset + bytes > backing->bytes) {
292
+ if (error_out) *error_out = dup_cstr("host backing store range is out of bounds");
293
+ return 1;
294
+ }
295
+ if (!ensure_ctx()) { if (error_out) *error_out = dup_cstr("no CUDA device available"); return 1; }
296
+ CUresult status = cuMemcpyDtoHAsync(static_cast<char *>(backing->data) + offset,
297
+ (CUdeviceptr)(uintptr_t)device, static_cast<size_t>(bytes), g_compute);
298
+ if (status == CUDA_SUCCESS) status = cuStreamSynchronize(g_compute);
299
+ if (status != CUDA_SUCCESS) { set_error(error_out, status, "host backing device-to-host copy"); return 1; }
300
+ return 0;
301
+ }
302
+ int gpu_bridge_host_backing_restore(void *host, void *device, long bytes, long offset, char **error_out) {
303
+ auto *backing = static_cast<HostBacking *>(host);
304
+ if (!backing || !device || bytes < 0 || offset < 0 || offset + bytes > backing->bytes) {
305
+ if (error_out) *error_out = dup_cstr("host backing restore range is out of bounds");
306
+ return 1;
307
+ }
308
+ if (!ensure_ctx()) { if (error_out) *error_out = dup_cstr("no CUDA device available"); return 1; }
309
+ CUresult status = cuMemcpyHtoDAsync((CUdeviceptr)(uintptr_t)device,
310
+ static_cast<char *>(backing->data) + offset, static_cast<size_t>(bytes), g_compute);
311
+ if (status == CUDA_SUCCESS) status = cuStreamSynchronize(g_compute);
312
+ if (status != CUDA_SUCCESS) { set_error(error_out, status, "host backing host-to-device copy"); return 1; }
313
+ return 0;
314
+ }
315
+
316
+ void *gpu_bridge_pipeline_create(const char *src, const char *fn_name, char **error_out) {
317
+ if (!ensure_ctx()) {
318
+ if (error_out) *error_out = dup_cstr("no CUDA device available (cuInit/cuCtxCreate failed)");
319
+ return nullptr;
320
+ }
321
+
322
+ nvrtcProgram prog;
323
+ if (nvrtcCreateProgram(&prog, src, "kernel.cu", 0, nullptr, nullptr) != NVRTC_SUCCESS) {
324
+ if (error_out) *error_out = dup_cstr("nvrtcCreateProgram failed");
325
+ return nullptr;
326
+ }
327
+
328
+ char arch[64];
329
+ arch_option(arch, sizeof(arch));
330
+ std::string include_opt = std::string("-I") + cuda_include_dir();
331
+ const char *opts[] = { arch, "--std=c++14", include_opt.c_str() };
332
+
333
+ if (nvrtcCompileProgram(prog, 3, opts) != NVRTC_SUCCESS) {
334
+ // Pass the whole log with line numbers to Ruby (Tilework::CompileError) as is.
335
+ size_t n = 0;
336
+ nvrtcGetProgramLogSize(prog, &n);
337
+ char *log = (char *)malloc(n > 0 ? n : 1);
338
+ if (log) {
339
+ log[0] = '\0';
340
+ if (n > 0) nvrtcGetProgramLog(prog, log);
341
+ }
342
+ if (error_out) *error_out = log ? log : dup_cstr("NVRTC compile failed");
343
+ else if (log) free(log);
344
+ nvrtcDestroyProgram(&prog);
345
+ return nullptr;
346
+ }
347
+
348
+ size_t ptx_size = 0;
349
+ nvrtcGetPTXSize(prog, &ptx_size);
350
+ char *ptx = (char *)malloc(ptx_size);
351
+ if (!ptx) {
352
+ nvrtcDestroyProgram(&prog);
353
+ if (error_out) *error_out = dup_cstr("out of memory while retrieving PTX");
354
+ return nullptr;
355
+ }
356
+ nvrtcGetPTX(prog, ptx);
357
+ nvrtcDestroyProgram(&prog);
358
+
359
+ CuPipeline *pl = new CuPipeline{};
360
+ CUresult e = cuModuleLoadData(&pl->mod, ptx);
361
+ free(ptx);
362
+ if (e != CUDA_SUCCESS) {
363
+ set_error(error_out, e, "cuModuleLoadData failed");
364
+ delete pl;
365
+ return nullptr;
366
+ }
367
+
368
+ e = cuModuleGetFunction(&pl->fn, pl->mod, fn_name);
369
+ if (e != CUDA_SUCCESS) {
370
+ char buf[256];
371
+ snprintf(buf, sizeof(buf),
372
+ "kernel function '%s' not found in the compiled module "
373
+ "(is it declared with GPU_KERNEL so that extern \"C\" applies?)",
374
+ fn_name);
375
+ if (error_out) *error_out = dup_cstr(buf);
376
+ cuModuleUnload(pl->mod);
377
+ delete pl;
378
+ return nullptr;
379
+ }
380
+ return pl;
381
+ }
382
+
383
+ long gpu_bridge_pipeline_max_threads(void *handle) {
384
+ if (!ensure_ctx()) return 0;
385
+ CuPipeline *pl = (CuPipeline *)handle;
386
+ int v = 0;
387
+ if (cuFuncGetAttribute(&v, CU_FUNC_ATTRIBUTE_MAX_THREADS_PER_BLOCK, pl->fn) != CUDA_SUCCESS) {
388
+ return 0;
389
+ }
390
+ return (long)v;
391
+ }
392
+
393
+ void gpu_bridge_pipeline_free(void *handle) {
394
+ if (!handle) return;
395
+ CuPipeline *pl = (CuPipeline *)handle;
396
+ if (ensure_ctx()) cuModuleUnload(pl->mod);
397
+ delete pl;
398
+ }
399
+
400
+ int gpu_bridge_dispatch(void *pipeline,
401
+ const int32_t *arg_kinds, void *const *arg_ptrs,
402
+ const long *arg_lens, int32_t arg_count,
403
+ long gx, long gy, long gz,
404
+ long bx, long by, long bz,
405
+ int32_t wait, char **error_out) {
406
+ if (!ensure_ctx()) {
407
+ if (error_out) *error_out = dup_cstr("no CUDA device available");
408
+ return -1;
409
+ }
410
+ CuPipeline *pl = (CuPipeline *)pipeline;
411
+
412
+ // kernelParams is an array of pointers to argument values. A buffer (kind=0)
413
+ // passes the address of a local holding the CUdeviceptr; a scalar (kind=1)
414
+ // passes the pointer to its value bytes directly.
415
+ std::vector<CUdeviceptr> devptrs(arg_count > 0 ? arg_count : 1);
416
+ std::vector<void *> params(arg_count > 0 ? arg_count : 1);
417
+ for (int32_t i = 0; i < arg_count; i++) {
418
+ if (arg_kinds[i] == 0) {
419
+ // For buffer arguments arg_lens is the view's byte offset (see gpu_bridge.c)
420
+ devptrs[i] = (CUdeviceptr)(uintptr_t)arg_ptrs[i] + (size_t)arg_lens[i];
421
+ params[i] = &devptrs[i];
422
+ } else {
423
+ params[i] = arg_ptrs[i];
424
+ }
425
+ }
426
+
427
+ // grid is the number of blocks and block the threads per block. Ruby's
428
+ // convention matches cuLaunchKernel's semantics, so no conversion.
429
+ CUresult e = cuLaunchKernel(pl->fn,
430
+ (unsigned)gx, (unsigned)gy, (unsigned)gz,
431
+ (unsigned)bx, (unsigned)by, (unsigned)bz,
432
+ 0, g_compute, params.data(), nullptr);
433
+ if (e != CUDA_SUCCESS) {
434
+ set_error(error_out, e, "cuLaunchKernel failed");
435
+ return -1;
436
+ }
437
+
438
+ if (wait) {
439
+ e = cuStreamSynchronize(g_compute);
440
+ if (e != CUDA_SUCCESS) {
441
+ set_error(error_out, e, "cuStreamSynchronize failed");
442
+ return -1;
443
+ }
444
+ }
445
+ return 0;
446
+ }
447
+
448
+ void gpu_bridge_synchronize(void) {
449
+ if (!ensure_ctx()) return;
450
+ cuStreamSynchronize(g_compute);
451
+ }
452
+
453
+ } // extern "C"
454
+
455
+ namespace {
456
+ // On file systems without O_DIRECT this silently falls back to ordinary I/O.
457
+ int open_maybe_direct(const char *path, int flags) {
458
+ int fd = open(path, flags | O_DIRECT);
459
+ if (fd < 0 && errno == EINVAL) fd = open(path, flags);
460
+ return fd;
461
+ }
462
+
463
+ size_t staging_bytes(long bytes) {
464
+ size_t padded = ((size_t)bytes + 4095) & ~(size_t)4095;
465
+ if (padded == 0) padded = 4096;
466
+ return padded < OFFLOAD_CHUNK ? padded : OFFLOAD_CHUNK;
467
+ }
468
+
469
+ // The synchronous core. The async version (transfer_*) just runs this on a worker thread.
470
+ int staged_save(CUdeviceptr dptr, const char *path, long bytes, long off, std::string *err) {
471
+ // Staging takes the smaller of the chunk limit and the actual data length
472
+ // (rounded up to 4096). Always taking 16 MB let allocation cost dominate
473
+ // round trips of small buffers (measured: 1 KB and 64 KB both took 8 ms).
474
+ const size_t chunk = staging_bytes(bytes);
475
+ void *stage = nullptr;
476
+ if (cuMemAllocHost(&stage, chunk) != CUDA_SUCCESS) {
477
+ if (err) *err = "failed to allocate pinned staging buffer";
478
+ return -1;
479
+ }
480
+ // ★ A transfer-only stream. On the default stream it would synchronize with
481
+ // compute kernels and prefetch would not overlap compute.
482
+ CUstream xfer = nullptr;
483
+ cuStreamCreate(&xfer, CU_STREAM_NON_BLOCKING);
484
+ int fd = open_maybe_direct(path, O_WRONLY);
485
+ if (fd < 0) {
486
+ cuMemFreeHost(stage);
487
+ if (err) *err = strerror(errno);
488
+ return -1;
489
+ }
490
+
491
+ cuStreamSynchronize(g_compute); // settle pending kernels' writes before reading
492
+ int rc = 0;
493
+ for (long done = 0; done < bytes;) {
494
+ size_t n = (size_t)((bytes - done) < (long)chunk ? (bytes - done) : (long)chunk);
495
+ if (cuMemcpyDtoHAsync(stage, dptr + (size_t)off + (size_t)done, n, xfer) != CUDA_SUCCESS ||
496
+ cuStreamSynchronize(xfer) != CUDA_SUCCESS) {
497
+ if (err) *err = "device-to-host copy failed during offload";
498
+ rc = -1;
499
+ break;
500
+ }
501
+ // O_DIRECT needs length, offset and address all 4096-aligned. The last
502
+ // chunk writes through its rounded-up zero padding (matching the Ruby
503
+ // path's padding, so both paths' files are bit-identical).
504
+ size_t wn = (n + 4095) & ~(size_t)4095;
505
+ if (wn > n) memset((char *)stage + n, 0, wn - n);
506
+ if (pwrite(fd, stage, wn, OFFLOAD_HEADER + done) != (ssize_t)wn) {
507
+ if (err) *err = strerror(errno);
508
+ rc = -1;
509
+ break;
510
+ }
511
+ done += (long)n;
512
+ }
513
+
514
+ if (rc == 0) fsync(fd);
515
+ close(fd);
516
+ if (xfer) cuStreamDestroy(xfer);
517
+ cuMemFreeHost(stage);
518
+ return rc;
519
+ }
520
+
521
+ int staged_load(CUdeviceptr dptr, const char *path, long bytes, long off, std::string *err) {
522
+ // Staging is sized for the same reason as in staged_save.
523
+ const size_t chunk = staging_bytes(bytes);
524
+ void *stage = nullptr;
525
+ if (cuMemAllocHost(&stage, chunk) != CUDA_SUCCESS) {
526
+ if (err) *err = "failed to allocate pinned staging buffer";
527
+ return -1;
528
+ }
529
+ CUstream xfer = nullptr; // ★ transfer-only stream (see staged_save)
530
+ cuStreamCreate(&xfer, CU_STREAM_NON_BLOCKING);
531
+ int fd = open_maybe_direct(path, O_RDONLY);
532
+ if (fd < 0) {
533
+ cuMemFreeHost(stage);
534
+ if (err) *err = strerror(errno);
535
+ return -1;
536
+ }
537
+
538
+ // The destination was allocated stream-ordered (on g_compute). This is a
539
+ // different stream, so not explicitly waiting for allocation and zero-fill
540
+ // is undefined behavior. staged_save does the same synchronization up front.
541
+ cuStreamSynchronize(g_compute);
542
+
543
+ int rc = 0;
544
+ for (long done = 0; done < bytes;) {
545
+ size_t n = (size_t)((bytes - done) < (long)chunk ? (bytes - done) : (long)chunk);
546
+ size_t rn = (n + 4095) & ~(size_t)4095; // the file is padded
547
+ ssize_t got = pread(fd, stage, rn, OFFLOAD_HEADER + done);
548
+ if (got < (ssize_t)n) {
549
+ if (err) *err = (got < 0) ? strerror(errno) : "offload file is shorter than expected";
550
+ rc = -1;
551
+ break;
552
+ }
553
+ if (cuMemcpyHtoDAsync(dptr + (size_t)off + (size_t)done, stage, n, xfer) != CUDA_SUCCESS ||
554
+ cuStreamSynchronize(xfer) != CUDA_SUCCESS) {
555
+ if (err) *err = "host-to-device copy failed during restore";
556
+ rc = -1;
557
+ break;
558
+ }
559
+ done += (long)n;
560
+ }
561
+
562
+ close(fd);
563
+ if (xfer) cuStreamDestroy(xfer);
564
+ cuMemFreeHost(stage);
565
+ return rc;
566
+ }
567
+
568
+ // State of one async transfer. The bridge has no scheduler; it just runs one
569
+ // worker. The policy of what to prefetch in which order lives in Ruby.
570
+ struct CuTransfer {
571
+ std::thread worker;
572
+ std::atomic<int> status{0}; // 0 = in progress, 1 = done, -1 = error
573
+ std::string err;
574
+ };
575
+
576
+ } // namespace
577
+
578
+ // --- Optional ABI: offload transfers ---
579
+ //
580
+ // Pinned staging + O_DIRECT. Ruby has already written the header (the first
581
+ // 4096 bytes) with ordinary I/O; this reads/writes only the body on 4096
582
+ // boundaries.
583
+ //
584
+ // cuFile (GPUDirect Storage) is not used: on the test machine
585
+ // cuFileDriverOpen succeeds, but nvidia-fs is not loaded, so it only reaches
586
+ // compatibility mode, falls back to staging internally and gains nothing.
587
+
588
+ extern "C" {
589
+
590
+ int gpu_bridge_buffer_save(void *handle, const char *path, long bytes, long off, char **err) {
591
+ if (!ensure_ctx()) {
592
+ if (err) *err = dup_cstr("no CUDA device available");
593
+ return -1;
594
+ }
595
+ std::string msg;
596
+ int rc = staged_save((CUdeviceptr)(uintptr_t)handle, path, bytes, off, &msg);
597
+ if (rc != 0 && err) *err = dup_cstr(msg.c_str());
598
+ return rc;
599
+ }
600
+
601
+ int gpu_bridge_buffer_load(void *handle, const char *path, long bytes, long off, char **err) {
602
+ if (!ensure_ctx()) {
603
+ if (err) *err = dup_cstr("no CUDA device available");
604
+ return -1;
605
+ }
606
+ std::string msg;
607
+ int rc = staged_load((CUdeviceptr)(uintptr_t)handle, path, bytes, off, &msg);
608
+ if (rc != 0 && err) *err = dup_cstr(msg.c_str());
609
+ return rc;
610
+ }
611
+
612
+ // dir: 0=save (VRAM -> SSD), 1=load (SSD -> VRAM)
613
+ void *gpu_bridge_transfer_begin(void *handle, const char *path, long bytes, long off, int dir) {
614
+ if (!ensure_ctx()) return nullptr;
615
+
616
+ CUdeviceptr dptr = (CUdeviceptr)(uintptr_t)handle;
617
+ std::string p(path); // copied so it does not depend on the caller's string lifetime
618
+ auto *t = new CuTransfer();
619
+ t->worker = std::thread([t, dptr, p, bytes, off, dir]() {
620
+ // Bind the context to the worker thread too.
621
+ cuCtxSetCurrent(g_ctx);
622
+ std::string msg;
623
+ int rc = (dir == 0) ? staged_save(dptr, p.c_str(), bytes, off, &msg)
624
+ : staged_load(dptr, p.c_str(), bytes, off, &msg);
625
+ t->err = msg;
626
+ t->status.store(rc == 0 ? 1 : -1);
627
+ });
628
+ return t;
629
+ }
630
+
631
+ int gpu_bridge_transfer_poll(void *handle) {
632
+ auto *t = (CuTransfer *)handle;
633
+ return t ? t->status.load() : -1;
634
+ }
635
+
636
+ int gpu_bridge_transfer_end(void *handle, char **err) {
637
+ auto *t = (CuTransfer *)handle;
638
+ if (!t) return -1;
639
+
640
+ if (t->worker.joinable()) t->worker.join();
641
+ int status = t->status.load();
642
+ if (status != 1 && err) *err = dup_cstr(t->err.empty() ? "offload transfer failed" : t->err.c_str());
643
+ delete t;
644
+ return status == 1 ? 0 : -1;
645
+ }
646
+
647
+ } // extern "C"
648
+
649
+ #ifdef TILEWORK_HAS_CURAND
650
+ // --- Optional ABI: random generation delegated to cuRAND ---
651
+ //
652
+ // Philox is counter-based: the same seed deterministically gives the same
653
+ // sequence. Its state is small and generation fast, which makes it easier to
654
+ // handle than the default XORWOW.
655
+ namespace {
656
+ curandGenerator_t g_rng = nullptr;
657
+ bool g_rng_failed = false;
658
+
659
+ bool ensure_rng() {
660
+ if (g_rng) return true;
661
+ if (g_rng_failed) return false;
662
+ if (curandCreateGenerator(&g_rng, CURAND_RNG_PSEUDO_PHILOX4_32_10) != CURAND_STATUS_SUCCESS) {
663
+ g_rng = nullptr;
664
+ g_rng_failed = true;
665
+ return false;
666
+ }
667
+ // ★ Bind to the compute stream (the memory pool is stream-ordered)
668
+ curandSetStream(g_rng, g_compute);
669
+ return true;
670
+ }
671
+ } // namespace
672
+
673
+ extern "C" {
674
+
675
+ // kind: 0 = uniform (0, 1], 1 = normal (mean, stddev)
676
+ int gpu_bridge_random(void *handle, long n, int kind, unsigned long long seed,
677
+ float mean, float stddev, char **err) {
678
+ if (!ensure_ctx() || !ensure_rng()) {
679
+ if (err) *err = dup_cstr("cuRAND is not available");
680
+ return -1;
681
+ }
682
+
683
+ // Setting the seed alone does not reset the internal offset, so the same
684
+ // seed would continue the sequence on every call. Reset it to 0 explicitly
685
+ // for reproducibility.
686
+ curandSetPseudoRandomGeneratorSeed(g_rng, seed);
687
+ curandSetGeneratorOffset(g_rng, 0);
688
+ float *dst = (float *)(uintptr_t)handle;
689
+ curandStatus_t st;
690
+
691
+ if (kind == 0) {
692
+ st = curandGenerateUniform(g_rng, dst, (size_t)n);
693
+ } else if ((n % 2) == 0) {
694
+ st = curandGenerateNormal(g_rng, dst, (size_t)n, mean, stddev);
695
+ } else {
696
+ // curandGenerateNormal makes pairs via Box-Muller and rejects odd counts.
697
+ // For an odd count, allocate one extra separately and copy the first n.
698
+ CUdeviceptr tmp = 0;
699
+ size_t padded = (size_t)n + 1;
700
+ CUresult ce = g_use_pool ? cuMemAllocAsync(&tmp, padded * sizeof(float), g_compute)
701
+ : cuMemAlloc(&tmp, padded * sizeof(float));
702
+ if (ce != CUDA_SUCCESS) {
703
+ if (err) *err = dup_cstr("failed to allocate scratch for an odd-length normal draw");
704
+ return -1;
705
+ }
706
+ st = curandGenerateNormal(g_rng, (float *)tmp, padded, mean, stddev);
707
+ if (st == CURAND_STATUS_SUCCESS) {
708
+ cuMemcpyDtoDAsync((CUdeviceptr)(uintptr_t)handle, tmp, (size_t)n * sizeof(float), g_compute);
709
+ }
710
+ if (g_use_pool) {
711
+ cuMemFreeAsync(tmp, g_compute);
712
+ } else {
713
+ cuStreamSynchronize(g_compute);
714
+ cuMemFree(tmp);
715
+ }
716
+ }
717
+
718
+ if (st != CURAND_STATUS_SUCCESS) {
719
+ char buf[128];
720
+ snprintf(buf, sizeof(buf), "curandGenerate%s failed (status %d)",
721
+ kind == 0 ? "Uniform" : "Normal", (int)st);
722
+ if (err) *err = dup_cstr(buf);
723
+ return -1;
724
+ }
725
+ return 0;
726
+ }
727
+
728
+ } // extern "C"
729
+ #endif // TILEWORK_HAS_CURAND
730
+
731
+ extern "C" {
732
+
733
+ #ifdef TILEWORK_HAS_CUDNN
734
+ // --- Optional ABI: 2D convolution delegated to cuDNN ---
735
+ //
736
+ // A hand-written im2col + GEMM spends 74–84% of its time in im2col and reaches
737
+ // only 240–613 GF/s (measured). im2col writes out k^2 times the data and is
738
+ // memory-bound, which becomes the bottleneck. cuDNN's implicit GEMM has no
739
+ // expansion at all and reaches fp16 4747 GF/s (7.7x) on the same shape.
740
+ namespace {
741
+ cudnnHandle_t g_dnn = nullptr;
742
+ bool g_dnn_failed = false;
743
+
744
+ bool ensure_dnn() {
745
+ if (g_dnn) return true;
746
+ if (g_dnn_failed) return false;
747
+ if (cudnnCreate(&g_dnn) != CUDNN_STATUS_SUCCESS) {
748
+ g_dnn = nullptr;
749
+ g_dnn_failed = true;
750
+ return false;
751
+ }
752
+ // ★ Bind to the compute stream (the memory pool is stream-ordered)
753
+ cudnnSetStream(g_dnn, g_compute);
754
+ return true;
755
+ }
756
+
757
+ // Descriptor and algorithm selection is expensive, so they are reused per
758
+ // configuration. Descriptors are shared by forward and backward (a gradient
759
+ // has its tensor's shape, so dy uses y's descriptor, dx x's, dw w's).
760
+ struct ConvPlan {
761
+ cudnnTensorDescriptor_t x, y;
762
+ cudnnFilterDescriptor_t w;
763
+ cudnnConvolutionDescriptor_t conv;
764
+ cudnnConvolutionFwdAlgo_t algo;
765
+ cudnnConvolutionBwdDataAlgo_t bwd_data_algo;
766
+ cudnnConvolutionBwdFilterAlgo_t bwd_filter_algo;
767
+ size_t workspace; // forward
768
+ size_t bwd_data_workspace;
769
+ size_t bwd_filter_workspace;
770
+ };
771
+ std::map<std::vector<int>, ConvPlan> g_conv_plans;
772
+
773
+ void conv_plan_destroy(ConvPlan *plan) {
774
+ if (!plan) return;
775
+ if (plan->conv) cudnnDestroyConvolutionDescriptor(plan->conv);
776
+ if (plan->w) cudnnDestroyFilterDescriptor(plan->w);
777
+ if (plan->y) cudnnDestroyTensorDescriptor(plan->y);
778
+ if (plan->x) cudnnDestroyTensorDescriptor(plan->x);
779
+ *plan = ConvPlan{};
780
+ }
781
+
782
+ // Workspace comes from the pool (allocation takes 0.016 ms, so per call is fine).
783
+ bool conv_workspace_alloc(size_t bytes, CUdeviceptr *out) {
784
+ *out = 0;
785
+ if (bytes == 0) return true;
786
+ CUresult ce = g_use_pool ? cuMemAllocAsync(out, bytes, g_compute)
787
+ : cuMemAlloc(out, bytes);
788
+ return ce == CUDA_SUCCESS;
789
+ }
790
+
791
+ void conv_workspace_free(CUdeviceptr ws) {
792
+ if (!ws) return;
793
+ if (g_use_pool) {
794
+ cuMemFreeAsync(ws, g_compute);
795
+ } else {
796
+ cuStreamSynchronize(g_compute);
797
+ cuMemFree(ws);
798
+ }
799
+ }
800
+ } // namespace
801
+
802
+ extern "C" {
803
+
804
+ int gpu_bridge_conv2d(void *x, void *w, void *y,
805
+ int n, int cin, int h, int width,
806
+ int cout, int kh, int kw,
807
+ int pad_h, int pad_w, int stride_h, int stride_w,
808
+ int dilation_h, int dilation_w, int groups,
809
+ int dtype, char **err) {
810
+ if (!ensure_ctx() || !ensure_dnn()) {
811
+ if (err) *err = dup_cstr("cuDNN is not available");
812
+ return -1;
813
+ }
814
+
815
+ std::vector<int> key = {n, cin, h, width, cout, kh, kw, pad_h, pad_w, stride_h, stride_w,
816
+ dilation_h, dilation_w, groups, dtype};
817
+ auto it = g_conv_plans.find(key);
818
+ if (it == g_conv_plans.end()) {
819
+ ConvPlan p{};
820
+ cudnnDataType_t dt = dtype == 1 ? CUDNN_DATA_HALF :
821
+ (dtype == 2 ? CUDNN_DATA_BFLOAT16 : CUDNN_DATA_FLOAT);
822
+
823
+ if (cudnnCreateTensorDescriptor(&p.x) != CUDNN_STATUS_SUCCESS ||
824
+ cudnnCreateTensorDescriptor(&p.y) != CUDNN_STATUS_SUCCESS ||
825
+ cudnnCreateFilterDescriptor(&p.w) != CUDNN_STATUS_SUCCESS ||
826
+ cudnnCreateConvolutionDescriptor(&p.conv) != CUDNN_STATUS_SUCCESS) {
827
+ conv_plan_destroy(&p);
828
+ if (err) *err = dup_cstr("failed to create cuDNN descriptors");
829
+ return -1;
830
+ }
831
+
832
+ cudnnSetTensor4dDescriptor(p.x, CUDNN_TENSOR_NCHW, dt, n, cin, h, width);
833
+ cudnnSetFilter4dDescriptor(p.w, dt, CUDNN_TENSOR_NCHW, cout, cin / groups, kh, kw);
834
+ // Accumulation is always FP32
835
+ cudnnSetConvolution2dDescriptor(p.conv, pad_h, pad_w, stride_h, stride_w, dilation_h, dilation_w,
836
+ CUDNN_CROSS_CORRELATION, CUDNN_DATA_FLOAT);
837
+ cudnnSetConvolutionGroupCount(p.conv, groups);
838
+ if (dtype != 0) cudnnSetConvolutionMathType(p.conv, CUDNN_TENSOR_OP_MATH);
839
+
840
+ int on, oc, oh, ow;
841
+ cudnnGetConvolution2dForwardOutputDim(p.conv, p.x, p.w, &on, &oc, &oh, &ow);
842
+ cudnnSetTensor4dDescriptor(p.y, CUDNN_TENSOR_NCHW, dt, on, oc, oh, ow);
843
+
844
+ int count = 0;
845
+ cudnnConvolutionFwdAlgoPerf_t perf[8];
846
+ if (cudnnGetConvolutionForwardAlgorithm_v7(g_dnn, p.x, p.w, p.conv, p.y, 8, &count, perf)
847
+ != CUDNN_STATUS_SUCCESS || count == 0) {
848
+ conv_plan_destroy(&p);
849
+ if (err) *err = dup_cstr("cuDNN found no forward convolution algorithm");
850
+ return -1;
851
+ }
852
+ p.algo = perf[0].algo;
853
+ cudnnGetConvolutionForwardWorkspaceSize(g_dnn, p.x, p.w, p.conv, p.y, p.algo, &p.workspace);
854
+
855
+ // Backward algorithms are chosen from the same descriptors here too.
856
+ // Selection is expensive, so it is folded into the plan instead of being
857
+ // redone every training step.
858
+ cudnnConvolutionBwdDataAlgoPerf_t dperf[8];
859
+ count = 0;
860
+ if (cudnnGetConvolutionBackwardDataAlgorithm_v7(g_dnn, p.w, p.y, p.conv, p.x, 8, &count, dperf)
861
+ == CUDNN_STATUS_SUCCESS && count > 0) {
862
+ p.bwd_data_algo = dperf[0].algo;
863
+ cudnnGetConvolutionBackwardDataWorkspaceSize(g_dnn, p.w, p.y, p.conv, p.x,
864
+ p.bwd_data_algo, &p.bwd_data_workspace);
865
+ } else {
866
+ p.bwd_data_workspace = SIZE_MAX; // marks unavailable; errors when called
867
+ }
868
+
869
+ cudnnConvolutionBwdFilterAlgoPerf_t fperf[8];
870
+ count = 0;
871
+ if (cudnnGetConvolutionBackwardFilterAlgorithm_v7(g_dnn, p.x, p.y, p.conv, p.w, 8, &count, fperf)
872
+ == CUDNN_STATUS_SUCCESS && count > 0) {
873
+ p.bwd_filter_algo = fperf[0].algo;
874
+ cudnnGetConvolutionBackwardFilterWorkspaceSize(g_dnn, p.x, p.y, p.conv, p.w,
875
+ p.bwd_filter_algo, &p.bwd_filter_workspace);
876
+ } else {
877
+ p.bwd_filter_workspace = SIZE_MAX;
878
+ }
879
+
880
+ g_conv_plans[key] = p;
881
+ it = g_conv_plans.find(key);
882
+ }
883
+
884
+ ConvPlan &p = it->second;
885
+
886
+ CUdeviceptr ws = 0;
887
+ if (!conv_workspace_alloc(p.workspace, &ws)) {
888
+ if (err) *err = dup_cstr("failed to allocate the cuDNN workspace");
889
+ return -1;
890
+ }
891
+
892
+ const float alpha = 1.0f, beta = 0.0f;
893
+ cudnnStatus_t st = cudnnConvolutionForward(g_dnn, &alpha, p.x, x, p.w, w, p.conv, p.algo,
894
+ (void *)(uintptr_t)ws, p.workspace, &beta, p.y, y);
895
+ conv_workspace_free(ws);
896
+
897
+ if (st != CUDNN_STATUS_SUCCESS) {
898
+ char buf[128];
899
+ snprintf(buf, sizeof(buf), "cudnnConvolutionForward failed (status %d)", (int)st);
900
+ if (err) *err = dup_cstr(buf);
901
+ return -1;
902
+ }
903
+ return 0;
904
+ }
905
+
906
+ } // extern "C"
907
+ #endif // TILEWORK_HAS_CUDNN
908
+
909
+ #ifdef TILEWORK_HAS_CUFFT
910
+ // --- Optional ABI: FFT delegated to cuFFT ---
911
+ //
912
+ // Complex numbers are interleaved float32 (re, im, re, im, ...). That is
913
+ // exactly cufftComplex's layout, so a Buffer can be passed as is.
914
+ namespace {
915
+ // Plan creation is expensive, so plans are reused per (kind, n, batch).
916
+ std::map<std::tuple<int, long, long>, cufftHandle> g_fft_plans;
917
+
918
+ // kind: 0 = R2C, 1 = C2R, 2 = C2C
919
+ bool fft_plan(int kind, long n, long batch, cufftHandle *out) {
920
+ auto key = std::make_tuple(kind, n, batch);
921
+ auto it = g_fft_plans.find(key);
922
+ if (it != g_fft_plans.end()) {
923
+ *out = it->second;
924
+ return true;
925
+ }
926
+
927
+ cufftType type = kind == 0 ? CUFFT_R2C : (kind == 1 ? CUFFT_C2R : CUFFT_C2C);
928
+ cufftHandle plan;
929
+ int len = (int)n;
930
+ if (cufftPlanMany(&plan, 1, &len, nullptr, 1, 0, nullptr, 1, 0, type, (int)batch) != CUFFT_SUCCESS) {
931
+ return false;
932
+ }
933
+ // ★ Bind to the compute stream (the memory pool is stream-ordered)
934
+ cufftSetStream(plan, g_compute);
935
+ g_fft_plans[key] = plan;
936
+ *out = plan;
937
+ return true;
938
+ }
939
+ } // namespace
940
+
941
+ extern "C" {
942
+
943
+ int gpu_bridge_fft(void *in, void *out, long n, long batch, int kind, int direction,
944
+ char **err) {
945
+ if (!ensure_ctx()) {
946
+ if (err) *err = dup_cstr("no CUDA device available");
947
+ return -1;
948
+ }
949
+
950
+ cufftHandle plan;
951
+ if (!fft_plan(kind, n, batch, &plan)) {
952
+ if (err) *err = dup_cstr("cufftPlanMany failed");
953
+ return -1;
954
+ }
955
+
956
+ cufftResult st;
957
+ switch (kind) {
958
+ case 0:
959
+ st = cufftExecR2C(plan, (cufftReal *)(uintptr_t)in, (cufftComplex *)(uintptr_t)out);
960
+ break;
961
+ case 1:
962
+ st = cufftExecC2R(plan, (cufftComplex *)(uintptr_t)in, (cufftReal *)(uintptr_t)out);
963
+ break;
964
+ default:
965
+ st = cufftExecC2C(plan, (cufftComplex *)(uintptr_t)in, (cufftComplex *)(uintptr_t)out,
966
+ direction < 0 ? CUFFT_INVERSE : CUFFT_FORWARD);
967
+ break;
968
+ }
969
+
970
+ if (st != CUFFT_SUCCESS) {
971
+ char buf[128];
972
+ snprintf(buf, sizeof(buf), "cufftExec failed (status %d)", (int)st);
973
+ if (err) *err = dup_cstr(buf);
974
+ return -1;
975
+ }
976
+ return 0;
977
+ }
978
+
979
+ } // extern "C"
980
+ #endif // TILEWORK_HAS_CUFFT
981
+
982
+ #ifdef TILEWORK_HAS_CUBLAS
983
+ // --- Optional ABI: GEMM delegated to cuBLAS ---
984
+ //
985
+ // Without it Ruby falls back to the existing kernel path (WMMA or SIMT).
986
+ // allow_tf32 enables TF32 tensor cores (10-bit mantissa, about 1e-3 relative).
987
+
988
+ // in_dtype: 0 = float32, 1 = float16, 2 = bfloat16. **The output is always
989
+ // float32** (FP32 accumulation, float32 output buffer).
990
+ int gpu_bridge_gemm(void *a, long a_off, void *b, long b_off, void *c, long c_off,
991
+ long m, long k, long n,
992
+ int allow_tf32, int in_dtype, char **err) {
993
+ if (!ensure_ctx() || !ensure_blas()) {
994
+ if (err) *err = dup_cstr("cuBLAS is not available");
995
+ return -1;
996
+ }
997
+ a = (char *)a + a_off;
998
+ b = (char *)b + b_off;
999
+ c = (char *)c + c_off;
1000
+
1001
+ cublasSetMathMode(g_blas, allow_tf32 ? CUBLAS_TF32_TENSOR_OP_MATH : CUBLAS_DEFAULT_MATH);
1002
+
1003
+ // Accumulation is FP32, so alpha/beta are float; never pass __half.
1004
+ const float alpha = 1.0f, beta = 0.0f;
1005
+ cublasStatus_t st;
1006
+
1007
+ // Buffers are row-major but cuBLAS is column-major. Row-major C = A*B equals
1008
+ // computing C^T = B^T * A^T in column-major (no transpose copy needed).
1009
+ if (in_dtype == 1 || in_dtype == 2) {
1010
+ const cudaDataType_t input_type = in_dtype == 1 ? CUDA_R_16F : CUDA_R_16BF;
1011
+ // A/B are CUDA_R_16F and only C is CUDA_R_32F; easy to mix up.
1012
+ st = cublasGemmEx(g_blas, CUBLAS_OP_N, CUBLAS_OP_N,
1013
+ (int)n, (int)m, (int)k,
1014
+ &alpha,
1015
+ (const void *)(uintptr_t)b, input_type, (int)n,
1016
+ (const void *)(uintptr_t)a, input_type, (int)k,
1017
+ &beta,
1018
+ (void *)(uintptr_t)c, CUDA_R_32F, (int)n,
1019
+ CUBLAS_COMPUTE_32F, CUBLAS_GEMM_DEFAULT);
1020
+ } else {
1021
+ st = cublasSgemm(g_blas, CUBLAS_OP_N, CUBLAS_OP_N,
1022
+ (int)n, (int)m, (int)k,
1023
+ &alpha,
1024
+ (const float *)(uintptr_t)b, (int)n,
1025
+ (const float *)(uintptr_t)a, (int)k,
1026
+ &beta,
1027
+ (float *)(uintptr_t)c, (int)n);
1028
+ }
1029
+
1030
+ if (st != CUBLAS_STATUS_SUCCESS) {
1031
+ char buf[160];
1032
+ const char *operation = in_dtype == 1 ? "GemmEx (fp16)" :
1033
+ (in_dtype == 2 ? "GemmEx (bf16)" : "Sgemm");
1034
+ snprintf(buf, sizeof(buf), "cublas%s failed (status %d)",
1035
+ operation, (int)st);
1036
+ if (err) *err = dup_cstr(buf);
1037
+ return -1;
1038
+ }
1039
+ return 0;
1040
+ }
1041
+
1042
+ // --- Optional ABI: strided batched GEMM ---
1043
+ //
1044
+ // Issues `batch` GEMMs of the same shape in one call. Matrices spaced at even
1045
+ // intervals need not be gathered (stride_* is how many elements to the next
1046
+ // matrix). Multi-head attention's heads are laid out exactly like that, so
1047
+ // slicing heads out and writing them back disappears entirely.
1048
+ //
1049
+ // ld_* is how many elements apart one column of the matrix is (the
1050
+ // column-major leading dimension). To view a row-major [R, C] as part of a
1051
+ // larger matrix, pass **the parent's column count** as ld; that is why heads
1052
+ // can be views.
1053
+ //
1054
+ // Unlike gemm(), the caller builds m/k/n and ld/stride in column-major
1055
+ // (reinterpreting from row-major stays in core/lib/tilework/math/nn.rb).
1056
+ int gpu_bridge_gemm_strided_batched(void *a, long a_off, void *b, long b_off,
1057
+ void *c, long c_off,
1058
+ long m, long k, long n,
1059
+ long lda, long ldb, long ldc,
1060
+ long stride_a, long stride_b, long stride_c,
1061
+ int trans_a, int trans_b,
1062
+ long batch, double alpha_in,
1063
+ int allow_tf32, int in_dtype,
1064
+ char **err) {
1065
+ if (!ensure_ctx() || !ensure_blas()) {
1066
+ if (err) *err = dup_cstr("cuBLAS is not available");
1067
+ return -1;
1068
+ }
1069
+ a = (char *)a + a_off;
1070
+ b = (char *)b + b_off;
1071
+ c = (char *)c + c_off;
1072
+
1073
+ cublasSetMathMode(g_blas, allow_tf32 ? CUBLAS_TF32_TENSOR_OP_MATH : CUBLAS_DEFAULT_MATH);
1074
+
1075
+ // Accumulation is FP32, so alpha/beta are float. A scale after the product,
1076
+ // like attention's 1/sqrt(dh), is applied for free through alpha.
1077
+ const float alpha = (float)alpha_in, beta = 0.0f;
1078
+ const cublasOperation_t op_a = trans_a ? CUBLAS_OP_T : CUBLAS_OP_N;
1079
+ const cublasOperation_t op_b = trans_b ? CUBLAS_OP_T : CUBLAS_OP_N;
1080
+ const cudaDataType_t in_type = in_dtype == 1 ? CUDA_R_16F :
1081
+ (in_dtype == 2 ? CUDA_R_16BF : CUDA_R_32F);
1082
+
1083
+ cublasStatus_t st = cublasGemmStridedBatchedEx(
1084
+ g_blas, op_a, op_b,
1085
+ (int)m, (int)n, (int)k,
1086
+ &alpha,
1087
+ (const void *)(uintptr_t)a, in_type, (int)lda, (long long)stride_a,
1088
+ (const void *)(uintptr_t)b, in_type, (int)ldb, (long long)stride_b,
1089
+ &beta,
1090
+ (void *)(uintptr_t)c, CUDA_R_32F, (int)ldc, (long long)stride_c,
1091
+ (int)batch, CUBLAS_COMPUTE_32F, CUBLAS_GEMM_DEFAULT);
1092
+
1093
+ if (st != CUBLAS_STATUS_SUCCESS) {
1094
+ char buf[160];
1095
+ snprintf(buf, sizeof(buf),
1096
+ "cublasGemmStridedBatchedEx failed (status %d, batch %ld)", (int)st, batch);
1097
+ if (err) *err = dup_cstr(buf);
1098
+ return -1;
1099
+ }
1100
+ return 0;
1101
+ }
1102
+ #endif // TILEWORK_HAS_CUBLAS
1103
+
1104
+ // --- Optional ABI: capability queries ---
1105
+ //
1106
+ // Not a required ABI. gpu_bridge.c references it as a weak symbol, so linking
1107
+ // a bridge without it (bridge.swift) does not break, and Ruby sees that the
1108
+ // query is missing and falls back.
1109
+ int gpu_bridge_device_capabilities(int32_t *warp_size, int32_t *cc_major, int32_t *cc_minor,
1110
+ int32_t *integrated, long *memory_budget_bytes) {
1111
+ if (!ensure_ctx()) return -1;
1112
+
1113
+ int v = 0;
1114
+ cuDeviceGetAttribute(&v, CU_DEVICE_ATTRIBUTE_WARP_SIZE, g_dev);
1115
+ *warp_size = v;
1116
+ v = 0;
1117
+ cuDeviceGetAttribute(&v, CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR, g_dev);
1118
+ *cc_major = v;
1119
+ v = 0;
1120
+ cuDeviceGetAttribute(&v, CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MINOR, g_dev);
1121
+ *cc_minor = v;
1122
+ // On Jetson and APUs the GPU and CPU share the same physical memory; this
1123
+ // feeds the offload staging limit.
1124
+ v = 0;
1125
+ cuDeviceGetAttribute(&v, CU_DEVICE_ATTRIBUTE_INTEGRATED, g_dev);
1126
+ *integrated = v;
1127
+
1128
+ size_t free_bytes = 0, total_bytes = 0;
1129
+ *memory_budget_bytes = cuMemGetInfo(&free_bytes, &total_bytes) == CUDA_SUCCESS ? (long)free_bytes : 0;
1130
+ return 0;
1131
+ }
1132
+
1133
+ #ifdef TILEWORK_HAS_CUDNN
1134
+ // Backward. mode 0 = input gradient (dx), mode 1 = weight gradient (dw).
1135
+ // The arguments mean different things per mode:
1136
+ // mode 0: a = w (weight), b = dy (output gradient), out = dx
1137
+ // mode 1: a = x (input), b = dy (output gradient), out = dw
1138
+ // Descriptors are shared with the forward plan (a gradient has its tensor's shape).
1139
+ int gpu_bridge_conv2d_backward(void *a, void *b, void *out,
1140
+ int n, int cin, int h, int width,
1141
+ int cout, int kh, int kw,
1142
+ int pad_h, int pad_w, int stride_h, int stride_w,
1143
+ int dilation_h, int dilation_w, int groups,
1144
+ int dtype, int mode, char **err) {
1145
+ if (!ensure_ctx() || !ensure_dnn()) {
1146
+ if (err) *err = dup_cstr("cuDNN is not available");
1147
+ return -1;
1148
+ }
1149
+
1150
+ std::vector<int> key = {n, cin, h, width, cout, kh, kw, pad_h, pad_w, stride_h, stride_w,
1151
+ dilation_h, dilation_w, groups, dtype};
1152
+ auto it = g_conv_plans.find(key);
1153
+ if (it == g_conv_plans.end()) {
1154
+ // The forward pass creates the plan. Training always runs forward first, so
1155
+ // a missing plan is rejected explicitly as a call-order error.
1156
+ if (err) *err = dup_cstr("no cached convolution plan; run the forward pass first");
1157
+ return -1;
1158
+ }
1159
+
1160
+ ConvPlan &p = it->second;
1161
+ size_t need = mode == 0 ? p.bwd_data_workspace : p.bwd_filter_workspace;
1162
+ if (need == SIZE_MAX) {
1163
+ if (err) {
1164
+ *err = dup_cstr(mode == 0 ? "cuDNN found no backward-data convolution algorithm"
1165
+ : "cuDNN found no backward-filter convolution algorithm");
1166
+ }
1167
+ return -1;
1168
+ }
1169
+
1170
+ CUdeviceptr ws = 0;
1171
+ if (!conv_workspace_alloc(need, &ws)) {
1172
+ if (err) *err = dup_cstr("failed to allocate the cuDNN workspace");
1173
+ return -1;
1174
+ }
1175
+
1176
+ const float alpha = 1.0f, beta = 0.0f;
1177
+ cudnnStatus_t st;
1178
+ if (mode == 0) {
1179
+ st = cudnnConvolutionBackwardData(g_dnn, &alpha, p.w, a, p.y, b, p.conv,
1180
+ p.bwd_data_algo, (void *)(uintptr_t)ws, need,
1181
+ &beta, p.x, out);
1182
+ } else {
1183
+ st = cudnnConvolutionBackwardFilter(g_dnn, &alpha, p.x, a, p.y, b, p.conv,
1184
+ p.bwd_filter_algo, (void *)(uintptr_t)ws, need,
1185
+ &beta, p.w, out);
1186
+ }
1187
+ conv_workspace_free(ws);
1188
+
1189
+ if (st != CUDNN_STATUS_SUCCESS) {
1190
+ char buf[160];
1191
+ snprintf(buf, sizeof(buf), "%s failed (status %d)",
1192
+ mode == 0 ? "cudnnConvolutionBackwardData" : "cudnnConvolutionBackwardFilter",
1193
+ (int)st);
1194
+ if (err) *err = dup_cstr(buf);
1195
+ return -1;
1196
+ }
1197
+ return 0;
1198
+ }
1199
+ #endif // TILEWORK_HAS_CUDNN
1200
+
1201
+ // Ruby's end proc calls this while the CUDA runtime is still usable. These
1202
+ // caches own native descriptors/plans independently of Ruby TypedData, so OS
1203
+ // process teardown is not an adequate ownership boundary (embedding and
1204
+ // backend reloads can keep the process alive).
1205
+ void gpu_bridge_shutdown(void) {
1206
+ if (!g_ctx) return;
1207
+ cuCtxSetCurrent(g_ctx);
1208
+ if (g_compute) cuStreamSynchronize(g_compute);
1209
+
1210
+ #ifdef TILEWORK_HAS_CUDNN
1211
+ for (auto &entry : g_conv_plans) conv_plan_destroy(&entry.second);
1212
+ g_conv_plans.clear();
1213
+ if (g_dnn) {
1214
+ cudnnDestroy(g_dnn);
1215
+ g_dnn = nullptr;
1216
+ }
1217
+ #endif
1218
+ #ifdef TILEWORK_HAS_CUFFT
1219
+ for (auto &entry : g_fft_plans) cufftDestroy(entry.second);
1220
+ g_fft_plans.clear();
1221
+ #endif
1222
+ #ifdef TILEWORK_HAS_CUBLAS
1223
+ if (g_blas) {
1224
+ cublasDestroy(g_blas);
1225
+ g_blas = nullptr;
1226
+ }
1227
+ #endif
1228
+ #ifdef TILEWORK_HAS_CURAND
1229
+ if (g_rng) {
1230
+ curandDestroyGenerator(g_rng);
1231
+ g_rng = nullptr;
1232
+ }
1233
+ #endif
1234
+ }
1235
+
1236
+ } // extern "C"