tilework-cuda 0.18.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE +21 -0
- data/ext/tilework/cuda/bridge.cpp +1236 -0
- data/ext/tilework/cuda/extconf.rb +105 -0
- data/ext/tilework/cuda/gpu_bridge.c +840 -0
- data/lib/tilework/backend/cuda.rb +32 -0
- metadata +66 -0
|
@@ -0,0 +1,1236 @@
|
|
|
1
|
+
// tilework native bridge (CUDA)
|
|
2
|
+
//
|
|
3
|
+
// Implements the gpu_bridge_* C ABI declared by gpu_bridge.c with the CUDA
|
|
4
|
+
// Driver API + NVRTC. Same contract as bridge.swift (Metal,
|
|
5
|
+
// ../../../metal/ext/tilework/metal/): mechanism only, with policy on the
|
|
6
|
+
// Ruby side (core/lib/tilework.rb).
|
|
7
|
+
//
|
|
8
|
+
// Loading run-time-compiled PTX needs the Driver API (cu*), so memory
|
|
9
|
+
// allocation uses the Driver API too, for uniformity.
|
|
10
|
+
//
|
|
11
|
+
// NVRTC compiles kernels at run time, so this file has no __global__. It is
|
|
12
|
+
// pure host code and builds with the host C++ compiler rather than nvcc
|
|
13
|
+
// (see extconf.rb).
|
|
14
|
+
|
|
15
|
+
#include <cuda.h>
|
|
16
|
+
#include <nvrtc.h>
|
|
17
|
+
// The vendor libraries are optional: extconf.rb defines TILEWORK_HAS_<LIB>
|
|
18
|
+
// only for the ones it finds, and a missing one leaves its gpu_bridge_*
|
|
19
|
+
// function undefined, so Ruby sees the weak symbol absent and falls back.
|
|
20
|
+
#ifdef TILEWORK_HAS_CUBLAS
|
|
21
|
+
#include <cublas_v2.h>
|
|
22
|
+
#endif
|
|
23
|
+
#ifdef TILEWORK_HAS_CURAND
|
|
24
|
+
#include <curand.h>
|
|
25
|
+
#endif
|
|
26
|
+
#ifdef TILEWORK_HAS_CUFFT
|
|
27
|
+
#include <cufft.h>
|
|
28
|
+
#endif
|
|
29
|
+
#ifdef TILEWORK_HAS_CUDNN
|
|
30
|
+
#include <cudnn.h>
|
|
31
|
+
#endif
|
|
32
|
+
|
|
33
|
+
#include <fcntl.h>
|
|
34
|
+
#include <unistd.h>
|
|
35
|
+
#include <errno.h>
|
|
36
|
+
|
|
37
|
+
#include <stdint.h>
|
|
38
|
+
#include <stdio.h>
|
|
39
|
+
#include <stdlib.h>
|
|
40
|
+
#include <string.h>
|
|
41
|
+
|
|
42
|
+
#include <atomic>
|
|
43
|
+
#include <map>
|
|
44
|
+
#include <string>
|
|
45
|
+
#include <tuple>
|
|
46
|
+
#include <thread>
|
|
47
|
+
#include <vector>
|
|
48
|
+
|
|
49
|
+
namespace {
|
|
50
|
+
|
|
51
|
+
CUcontext g_ctx = nullptr;
|
|
52
|
+
CUdevice g_dev = 0;
|
|
53
|
+
bool g_init_failed = false;
|
|
54
|
+
// A compute-only stream. On the default stream, cuCtxSynchronize waits for
|
|
55
|
+
// every stream in the context — including offload's transfer stream — so
|
|
56
|
+
// prefetch never overlapped compute (raising depth stayed at 1.0x in
|
|
57
|
+
// measurements). Compute and transfers use separate streams and waits are
|
|
58
|
+
// per stream.
|
|
59
|
+
CUstream g_compute = nullptr;
|
|
60
|
+
// Stream-ordered memory pool (CUDA 11.2+). cuMemFreeAsync returns memory to
|
|
61
|
+
// the pool rather than the OS, so the next same-size allocation returns at
|
|
62
|
+
// once. On Jetson's unified memory cuMemAlloc involves page mapping and is
|
|
63
|
+
// very slow; a 16 MB allocation measured 3.208 ms → 0.136 ms (23.6x).
|
|
64
|
+
//
|
|
65
|
+
// Where unavailable, g_use_pool stays false and cuMemAlloc/cuMemFree are used.
|
|
66
|
+
// **Allocation and free must always take the same path**, so the choice is
|
|
67
|
+
// made once at initialization and never changes (mixing them crashes).
|
|
68
|
+
bool g_use_pool = false;
|
|
69
|
+
CUmemoryPool g_pool = nullptr;
|
|
70
|
+
|
|
71
|
+
#ifdef TILEWORK_HAS_CUBLAS
|
|
72
|
+
// cuBLAS handle, created lazily (creation is expensive) and reused. Orders of
|
|
73
|
+
// magnitude faster than our WMMA kernel (core/lib/tilework/math/tensor_core.rb).
|
|
74
|
+
cublasHandle_t g_blas = nullptr;
|
|
75
|
+
bool g_blas_failed = false;
|
|
76
|
+
|
|
77
|
+
bool ensure_blas() {
|
|
78
|
+
if (g_blas) return true;
|
|
79
|
+
if (g_blas_failed) return false;
|
|
80
|
+
if (cublasCreate(&g_blas) != CUBLAS_STATUS_SUCCESS) {
|
|
81
|
+
g_blas = nullptr;
|
|
82
|
+
g_blas_failed = true;
|
|
83
|
+
return false;
|
|
84
|
+
}
|
|
85
|
+
// ★ Bind to the compute stream. The pool is stream-ordered, so on the
|
|
86
|
+
// default stream ordering against allocation / zero-fill is not guaranteed.
|
|
87
|
+
cublasSetStream(g_blas, g_compute);
|
|
88
|
+
return true;
|
|
89
|
+
}
|
|
90
|
+
#endif // TILEWORK_HAS_CUBLAS
|
|
91
|
+
|
|
92
|
+
char *dup_cstr(const char *s) {
|
|
93
|
+
// The caller (gpu_bridge.c) releases it with free(), so allocate with malloc.
|
|
94
|
+
size_t n = strlen(s) + 1;
|
|
95
|
+
char *p = (char *)malloc(n);
|
|
96
|
+
if (p) memcpy(p, s, n);
|
|
97
|
+
return p;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
void set_error(char **error_out, CUresult e, const char *what) {
|
|
101
|
+
if (!error_out) return;
|
|
102
|
+
const char *detail = nullptr;
|
|
103
|
+
cuGetErrorString(e, &detail);
|
|
104
|
+
char buf[512];
|
|
105
|
+
snprintf(buf, sizeof(buf), "%s: %s", what, detail ? detail : "unknown CUDA error");
|
|
106
|
+
*error_out = dup_cstr(buf);
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
// cuInit + context creation happen once; later calls just make it current
|
|
110
|
+
// again (Ruby assumes a single queue, so one context and one stream suffice).
|
|
111
|
+
bool ensure_ctx() {
|
|
112
|
+
if (g_ctx) {
|
|
113
|
+
cuCtxSetCurrent(g_ctx);
|
|
114
|
+
return true;
|
|
115
|
+
}
|
|
116
|
+
if (g_init_failed) return false;
|
|
117
|
+
|
|
118
|
+
if (cuInit(0) != CUDA_SUCCESS || cuDeviceGet(&g_dev, 0) != CUDA_SUCCESS) {
|
|
119
|
+
g_init_failed = true;
|
|
120
|
+
return false;
|
|
121
|
+
}
|
|
122
|
+
if (cuCtxCreate(&g_ctx, 0, g_dev) != CUDA_SUCCESS) {
|
|
123
|
+
g_ctx = nullptr;
|
|
124
|
+
g_init_failed = true;
|
|
125
|
+
return false;
|
|
126
|
+
}
|
|
127
|
+
if (cuStreamCreate(&g_compute, CU_STREAM_NON_BLOCKING) != CUDA_SUCCESS) {
|
|
128
|
+
// cuCtxCreate succeeded, so this function owns a live context even though
|
|
129
|
+
// initialization as a whole failed. Dropping only the pointer leaks all
|
|
130
|
+
// driver-side context state for the lifetime of the process.
|
|
131
|
+
cuCtxDestroy(g_ctx);
|
|
132
|
+
g_ctx = nullptr;
|
|
133
|
+
g_compute = nullptr;
|
|
134
|
+
g_init_failed = true;
|
|
135
|
+
return false;
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
int pools_supported = 0;
|
|
139
|
+
cuDeviceGetAttribute(&pools_supported, CU_DEVICE_ATTRIBUTE_MEMORY_POOLS_SUPPORTED, g_dev);
|
|
140
|
+
if (pools_supported && cuDeviceGetDefaultMemPool(&g_pool, g_dev) == CUDA_SUCCESS) {
|
|
141
|
+
// A pool hoarding without limit starves other allocations into OOM on
|
|
142
|
+
// unified-memory machines. Cap it at 1/4 of free memory (anything beyond
|
|
143
|
+
// goes back to the OS).
|
|
144
|
+
size_t free_bytes = 0, total_bytes = 0;
|
|
145
|
+
cuMemGetInfo(&free_bytes, &total_bytes);
|
|
146
|
+
cuuint64_t threshold = (cuuint64_t)(free_bytes / 4);
|
|
147
|
+
cuMemPoolSetAttribute(g_pool, CU_MEMPOOL_ATTR_RELEASE_THRESHOLD, &threshold);
|
|
148
|
+
g_use_pool = true;
|
|
149
|
+
}
|
|
150
|
+
return true;
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
// Offload chunk size. 16 MB by default (a middle size balancing speedup,
|
|
154
|
+
// hiding quality and synchronization overhead).
|
|
155
|
+
static const size_t OFFLOAD_CHUNK = 16u << 20;
|
|
156
|
+
// Header length at the start of a file; the body follows it.
|
|
157
|
+
static const long OFFLOAD_HEADER = 4096;
|
|
158
|
+
|
|
159
|
+
struct CuPipeline {
|
|
160
|
+
CUmodule mod;
|
|
161
|
+
CUfunction fn;
|
|
162
|
+
};
|
|
163
|
+
struct HostBacking { void *data; long bytes; };
|
|
164
|
+
|
|
165
|
+
// NVRTC does not bundle mma.h (measured on CUDA 12.6: without `-I`, "could
|
|
166
|
+
// not open source file mma.h"), so CUDA's include directory is always passed
|
|
167
|
+
// for the Tensor Core path. Passing header text as strings is an alternative,
|
|
168
|
+
// but -I is confirmed to work on 12.6.
|
|
169
|
+
const char *cuda_include_dir() {
|
|
170
|
+
static std::string dir;
|
|
171
|
+
if (dir.empty()) {
|
|
172
|
+
const char *home = getenv("CUDA_HOME");
|
|
173
|
+
dir = std::string((home && *home) ? home : "/usr/local/cuda") + "/include";
|
|
174
|
+
}
|
|
175
|
+
return dir.c_str();
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
// The PTX target architecture is built from the device rather than fixed at
|
|
179
|
+
// build time (compute_87 on a Jetson Orin, for example).
|
|
180
|
+
void arch_option(char *out, size_t len) {
|
|
181
|
+
int major = 0, minor = 0;
|
|
182
|
+
cuDeviceGetAttribute(&major, CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR, g_dev);
|
|
183
|
+
cuDeviceGetAttribute(&minor, CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MINOR, g_dev);
|
|
184
|
+
if (major == 0) { major = 5; minor = 0; } // on failure rely on PTX forward compatibility
|
|
185
|
+
snprintf(out, len, "--gpu-architecture=compute_%d%d", major, minor);
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
} // namespace
|
|
189
|
+
|
|
190
|
+
extern "C" {
|
|
191
|
+
|
|
192
|
+
char *gpu_bridge_device_name(void) {
|
|
193
|
+
if (!ensure_ctx()) return nullptr;
|
|
194
|
+
char name[256];
|
|
195
|
+
if (cuDeviceGetName(name, sizeof(name), g_dev) != CUDA_SUCCESS) return nullptr;
|
|
196
|
+
return dup_cstr(name);
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
int gpu_bridge_device_available(void) {
|
|
200
|
+
return ensure_ctx() ? 1 : 0;
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
void *gpu_bridge_buffer_alloc(long bytes) {
|
|
204
|
+
if (!ensure_ctx()) return nullptr;
|
|
205
|
+
|
|
206
|
+
CUdeviceptr dptr = 0;
|
|
207
|
+
if (g_use_pool) {
|
|
208
|
+
// Allocate on the compute stream. Every later operation on this pointer
|
|
209
|
+
// must be on g_compute or explicitly synchronized with it (a constraint of
|
|
210
|
+
// stream-ordered allocation).
|
|
211
|
+
CUresult e = cuMemAllocAsync(&dptr, (size_t)bytes, g_compute);
|
|
212
|
+
if (e != CUDA_SUCCESS) {
|
|
213
|
+
// Return what the pool holds to the OS and retry once; without this the
|
|
214
|
+
// state "free memory exists but allocation fails" is possible.
|
|
215
|
+
cuStreamSynchronize(g_compute);
|
|
216
|
+
cuMemPoolTrimTo(g_pool, 0);
|
|
217
|
+
e = cuMemAllocAsync(&dptr, (size_t)bytes, g_compute);
|
|
218
|
+
}
|
|
219
|
+
if (e != CUDA_SUCCESS) return nullptr;
|
|
220
|
+
} else if (cuMemAlloc(&dptr, (size_t)bytes) != CUDA_SUCCESS) {
|
|
221
|
+
return nullptr;
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
// Zero-initialize. The pool returns regions with old contents, so without
|
|
225
|
+
// this code assuming "a new buffer is zero" would read different values each
|
|
226
|
+
// run (Metal's storageModeShared buffers are effectively zero-initialized,
|
|
227
|
+
// and this keeps behavior the same across backends). The async version is
|
|
228
|
+
// used because synchronizing here would undo the faster allocation.
|
|
229
|
+
if (g_use_pool) {
|
|
230
|
+
cuMemsetD8Async(dptr, 0, (size_t)bytes, g_compute);
|
|
231
|
+
} else {
|
|
232
|
+
cuMemsetD8(dptr, 0, (size_t)bytes);
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
// CUdeviceptr is a 64-bit integer; it round-trips as a void* handle.
|
|
236
|
+
return (void *)(uintptr_t)dptr;
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
void gpu_bridge_buffer_write(void *handle, const void *src, long bytes, long offset) {
|
|
240
|
+
if (!ensure_ctx()) return;
|
|
241
|
+
CUdeviceptr dptr = (CUdeviceptr)(uintptr_t)handle;
|
|
242
|
+
// Done on the same g_compute as allocation and zero-fill. g_compute is
|
|
243
|
+
// NON_BLOCKING, so on the default stream implicit synchronization would not
|
|
244
|
+
// apply and the write could land before allocation or zero-fill finishes.
|
|
245
|
+
cuMemcpyHtoDAsync(dptr + (size_t)offset, src, (size_t)bytes, g_compute);
|
|
246
|
+
cuStreamSynchronize(g_compute); // Ruby expects the write to be complete
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
void gpu_bridge_buffer_read(void *handle, void *dst, long bytes, long offset) {
|
|
250
|
+
if (!ensure_ctx()) return;
|
|
251
|
+
CUdeviceptr dptr = (CUdeviceptr)(uintptr_t)handle;
|
|
252
|
+
// Read on the same g_compute as allocation, zero-fill and kernels so the
|
|
253
|
+
// value is right even if the previous launch_async has not finished (the
|
|
254
|
+
// same semantics as Metal's contents seeing committed buffers). Only the
|
|
255
|
+
// compute stream is waited on — no need to wait for in-flight offload transfers.
|
|
256
|
+
cuMemcpyDtoHAsync(dst, dptr + (size_t)offset, (size_t)bytes, g_compute);
|
|
257
|
+
cuStreamSynchronize(g_compute);
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
void gpu_bridge_buffer_free(void *handle) {
|
|
261
|
+
if (!handle) return;
|
|
262
|
+
if (!ensure_ctx()) return;
|
|
263
|
+
|
|
264
|
+
// Return it through the same path it was allocated by; mixing them crashes,
|
|
265
|
+
// hence the branch on g_use_pool.
|
|
266
|
+
if (g_use_pool) {
|
|
267
|
+
cuMemFreeAsync((CUdeviceptr)(uintptr_t)handle, g_compute);
|
|
268
|
+
} else {
|
|
269
|
+
cuMemFree((CUdeviceptr)(uintptr_t)handle);
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
int gpu_bridge_host_backing_available(void) { return ensure_ctx() ? 1 : 0; }
|
|
274
|
+
void *gpu_bridge_host_backing_alloc(long bytes) {
|
|
275
|
+
if (bytes <= 0) return nullptr;
|
|
276
|
+
auto *backing = static_cast<HostBacking *>(malloc(sizeof(HostBacking)));
|
|
277
|
+
if (!backing) return nullptr;
|
|
278
|
+
backing->data = malloc(static_cast<size_t>(bytes));
|
|
279
|
+
backing->bytes = bytes;
|
|
280
|
+
if (!backing->data) { free(backing); return nullptr; }
|
|
281
|
+
return backing;
|
|
282
|
+
}
|
|
283
|
+
void gpu_bridge_host_backing_free(void *handle) {
|
|
284
|
+
auto *backing = static_cast<HostBacking *>(handle);
|
|
285
|
+
if (!backing) return;
|
|
286
|
+
free(backing->data);
|
|
287
|
+
free(backing);
|
|
288
|
+
}
|
|
289
|
+
int gpu_bridge_host_backing_store(void *host, void *device, long bytes, long offset, char **error_out) {
|
|
290
|
+
auto *backing = static_cast<HostBacking *>(host);
|
|
291
|
+
if (!backing || !device || bytes < 0 || offset < 0 || offset + bytes > backing->bytes) {
|
|
292
|
+
if (error_out) *error_out = dup_cstr("host backing store range is out of bounds");
|
|
293
|
+
return 1;
|
|
294
|
+
}
|
|
295
|
+
if (!ensure_ctx()) { if (error_out) *error_out = dup_cstr("no CUDA device available"); return 1; }
|
|
296
|
+
CUresult status = cuMemcpyDtoHAsync(static_cast<char *>(backing->data) + offset,
|
|
297
|
+
(CUdeviceptr)(uintptr_t)device, static_cast<size_t>(bytes), g_compute);
|
|
298
|
+
if (status == CUDA_SUCCESS) status = cuStreamSynchronize(g_compute);
|
|
299
|
+
if (status != CUDA_SUCCESS) { set_error(error_out, status, "host backing device-to-host copy"); return 1; }
|
|
300
|
+
return 0;
|
|
301
|
+
}
|
|
302
|
+
int gpu_bridge_host_backing_restore(void *host, void *device, long bytes, long offset, char **error_out) {
|
|
303
|
+
auto *backing = static_cast<HostBacking *>(host);
|
|
304
|
+
if (!backing || !device || bytes < 0 || offset < 0 || offset + bytes > backing->bytes) {
|
|
305
|
+
if (error_out) *error_out = dup_cstr("host backing restore range is out of bounds");
|
|
306
|
+
return 1;
|
|
307
|
+
}
|
|
308
|
+
if (!ensure_ctx()) { if (error_out) *error_out = dup_cstr("no CUDA device available"); return 1; }
|
|
309
|
+
CUresult status = cuMemcpyHtoDAsync((CUdeviceptr)(uintptr_t)device,
|
|
310
|
+
static_cast<char *>(backing->data) + offset, static_cast<size_t>(bytes), g_compute);
|
|
311
|
+
if (status == CUDA_SUCCESS) status = cuStreamSynchronize(g_compute);
|
|
312
|
+
if (status != CUDA_SUCCESS) { set_error(error_out, status, "host backing host-to-device copy"); return 1; }
|
|
313
|
+
return 0;
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
void *gpu_bridge_pipeline_create(const char *src, const char *fn_name, char **error_out) {
|
|
317
|
+
if (!ensure_ctx()) {
|
|
318
|
+
if (error_out) *error_out = dup_cstr("no CUDA device available (cuInit/cuCtxCreate failed)");
|
|
319
|
+
return nullptr;
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
nvrtcProgram prog;
|
|
323
|
+
if (nvrtcCreateProgram(&prog, src, "kernel.cu", 0, nullptr, nullptr) != NVRTC_SUCCESS) {
|
|
324
|
+
if (error_out) *error_out = dup_cstr("nvrtcCreateProgram failed");
|
|
325
|
+
return nullptr;
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
char arch[64];
|
|
329
|
+
arch_option(arch, sizeof(arch));
|
|
330
|
+
std::string include_opt = std::string("-I") + cuda_include_dir();
|
|
331
|
+
const char *opts[] = { arch, "--std=c++14", include_opt.c_str() };
|
|
332
|
+
|
|
333
|
+
if (nvrtcCompileProgram(prog, 3, opts) != NVRTC_SUCCESS) {
|
|
334
|
+
// Pass the whole log with line numbers to Ruby (Tilework::CompileError) as is.
|
|
335
|
+
size_t n = 0;
|
|
336
|
+
nvrtcGetProgramLogSize(prog, &n);
|
|
337
|
+
char *log = (char *)malloc(n > 0 ? n : 1);
|
|
338
|
+
if (log) {
|
|
339
|
+
log[0] = '\0';
|
|
340
|
+
if (n > 0) nvrtcGetProgramLog(prog, log);
|
|
341
|
+
}
|
|
342
|
+
if (error_out) *error_out = log ? log : dup_cstr("NVRTC compile failed");
|
|
343
|
+
else if (log) free(log);
|
|
344
|
+
nvrtcDestroyProgram(&prog);
|
|
345
|
+
return nullptr;
|
|
346
|
+
}
|
|
347
|
+
|
|
348
|
+
size_t ptx_size = 0;
|
|
349
|
+
nvrtcGetPTXSize(prog, &ptx_size);
|
|
350
|
+
char *ptx = (char *)malloc(ptx_size);
|
|
351
|
+
if (!ptx) {
|
|
352
|
+
nvrtcDestroyProgram(&prog);
|
|
353
|
+
if (error_out) *error_out = dup_cstr("out of memory while retrieving PTX");
|
|
354
|
+
return nullptr;
|
|
355
|
+
}
|
|
356
|
+
nvrtcGetPTX(prog, ptx);
|
|
357
|
+
nvrtcDestroyProgram(&prog);
|
|
358
|
+
|
|
359
|
+
CuPipeline *pl = new CuPipeline{};
|
|
360
|
+
CUresult e = cuModuleLoadData(&pl->mod, ptx);
|
|
361
|
+
free(ptx);
|
|
362
|
+
if (e != CUDA_SUCCESS) {
|
|
363
|
+
set_error(error_out, e, "cuModuleLoadData failed");
|
|
364
|
+
delete pl;
|
|
365
|
+
return nullptr;
|
|
366
|
+
}
|
|
367
|
+
|
|
368
|
+
e = cuModuleGetFunction(&pl->fn, pl->mod, fn_name);
|
|
369
|
+
if (e != CUDA_SUCCESS) {
|
|
370
|
+
char buf[256];
|
|
371
|
+
snprintf(buf, sizeof(buf),
|
|
372
|
+
"kernel function '%s' not found in the compiled module "
|
|
373
|
+
"(is it declared with GPU_KERNEL so that extern \"C\" applies?)",
|
|
374
|
+
fn_name);
|
|
375
|
+
if (error_out) *error_out = dup_cstr(buf);
|
|
376
|
+
cuModuleUnload(pl->mod);
|
|
377
|
+
delete pl;
|
|
378
|
+
return nullptr;
|
|
379
|
+
}
|
|
380
|
+
return pl;
|
|
381
|
+
}
|
|
382
|
+
|
|
383
|
+
long gpu_bridge_pipeline_max_threads(void *handle) {
|
|
384
|
+
if (!ensure_ctx()) return 0;
|
|
385
|
+
CuPipeline *pl = (CuPipeline *)handle;
|
|
386
|
+
int v = 0;
|
|
387
|
+
if (cuFuncGetAttribute(&v, CU_FUNC_ATTRIBUTE_MAX_THREADS_PER_BLOCK, pl->fn) != CUDA_SUCCESS) {
|
|
388
|
+
return 0;
|
|
389
|
+
}
|
|
390
|
+
return (long)v;
|
|
391
|
+
}
|
|
392
|
+
|
|
393
|
+
void gpu_bridge_pipeline_free(void *handle) {
|
|
394
|
+
if (!handle) return;
|
|
395
|
+
CuPipeline *pl = (CuPipeline *)handle;
|
|
396
|
+
if (ensure_ctx()) cuModuleUnload(pl->mod);
|
|
397
|
+
delete pl;
|
|
398
|
+
}
|
|
399
|
+
|
|
400
|
+
int gpu_bridge_dispatch(void *pipeline,
|
|
401
|
+
const int32_t *arg_kinds, void *const *arg_ptrs,
|
|
402
|
+
const long *arg_lens, int32_t arg_count,
|
|
403
|
+
long gx, long gy, long gz,
|
|
404
|
+
long bx, long by, long bz,
|
|
405
|
+
int32_t wait, char **error_out) {
|
|
406
|
+
if (!ensure_ctx()) {
|
|
407
|
+
if (error_out) *error_out = dup_cstr("no CUDA device available");
|
|
408
|
+
return -1;
|
|
409
|
+
}
|
|
410
|
+
CuPipeline *pl = (CuPipeline *)pipeline;
|
|
411
|
+
|
|
412
|
+
// kernelParams is an array of pointers to argument values. A buffer (kind=0)
|
|
413
|
+
// passes the address of a local holding the CUdeviceptr; a scalar (kind=1)
|
|
414
|
+
// passes the pointer to its value bytes directly.
|
|
415
|
+
std::vector<CUdeviceptr> devptrs(arg_count > 0 ? arg_count : 1);
|
|
416
|
+
std::vector<void *> params(arg_count > 0 ? arg_count : 1);
|
|
417
|
+
for (int32_t i = 0; i < arg_count; i++) {
|
|
418
|
+
if (arg_kinds[i] == 0) {
|
|
419
|
+
// For buffer arguments arg_lens is the view's byte offset (see gpu_bridge.c)
|
|
420
|
+
devptrs[i] = (CUdeviceptr)(uintptr_t)arg_ptrs[i] + (size_t)arg_lens[i];
|
|
421
|
+
params[i] = &devptrs[i];
|
|
422
|
+
} else {
|
|
423
|
+
params[i] = arg_ptrs[i];
|
|
424
|
+
}
|
|
425
|
+
}
|
|
426
|
+
|
|
427
|
+
// grid is the number of blocks and block the threads per block. Ruby's
|
|
428
|
+
// convention matches cuLaunchKernel's semantics, so no conversion.
|
|
429
|
+
CUresult e = cuLaunchKernel(pl->fn,
|
|
430
|
+
(unsigned)gx, (unsigned)gy, (unsigned)gz,
|
|
431
|
+
(unsigned)bx, (unsigned)by, (unsigned)bz,
|
|
432
|
+
0, g_compute, params.data(), nullptr);
|
|
433
|
+
if (e != CUDA_SUCCESS) {
|
|
434
|
+
set_error(error_out, e, "cuLaunchKernel failed");
|
|
435
|
+
return -1;
|
|
436
|
+
}
|
|
437
|
+
|
|
438
|
+
if (wait) {
|
|
439
|
+
e = cuStreamSynchronize(g_compute);
|
|
440
|
+
if (e != CUDA_SUCCESS) {
|
|
441
|
+
set_error(error_out, e, "cuStreamSynchronize failed");
|
|
442
|
+
return -1;
|
|
443
|
+
}
|
|
444
|
+
}
|
|
445
|
+
return 0;
|
|
446
|
+
}
|
|
447
|
+
|
|
448
|
+
void gpu_bridge_synchronize(void) {
|
|
449
|
+
if (!ensure_ctx()) return;
|
|
450
|
+
cuStreamSynchronize(g_compute);
|
|
451
|
+
}
|
|
452
|
+
|
|
453
|
+
} // extern "C"
|
|
454
|
+
|
|
455
|
+
namespace {
|
|
456
|
+
// On file systems without O_DIRECT this silently falls back to ordinary I/O.
|
|
457
|
+
int open_maybe_direct(const char *path, int flags) {
|
|
458
|
+
int fd = open(path, flags | O_DIRECT);
|
|
459
|
+
if (fd < 0 && errno == EINVAL) fd = open(path, flags);
|
|
460
|
+
return fd;
|
|
461
|
+
}
|
|
462
|
+
|
|
463
|
+
size_t staging_bytes(long bytes) {
|
|
464
|
+
size_t padded = ((size_t)bytes + 4095) & ~(size_t)4095;
|
|
465
|
+
if (padded == 0) padded = 4096;
|
|
466
|
+
return padded < OFFLOAD_CHUNK ? padded : OFFLOAD_CHUNK;
|
|
467
|
+
}
|
|
468
|
+
|
|
469
|
+
// The synchronous core. The async version (transfer_*) just runs this on a worker thread.
|
|
470
|
+
int staged_save(CUdeviceptr dptr, const char *path, long bytes, long off, std::string *err) {
|
|
471
|
+
// Staging takes the smaller of the chunk limit and the actual data length
|
|
472
|
+
// (rounded up to 4096). Always taking 16 MB let allocation cost dominate
|
|
473
|
+
// round trips of small buffers (measured: 1 KB and 64 KB both took 8 ms).
|
|
474
|
+
const size_t chunk = staging_bytes(bytes);
|
|
475
|
+
void *stage = nullptr;
|
|
476
|
+
if (cuMemAllocHost(&stage, chunk) != CUDA_SUCCESS) {
|
|
477
|
+
if (err) *err = "failed to allocate pinned staging buffer";
|
|
478
|
+
return -1;
|
|
479
|
+
}
|
|
480
|
+
// ★ A transfer-only stream. On the default stream it would synchronize with
|
|
481
|
+
// compute kernels and prefetch would not overlap compute.
|
|
482
|
+
CUstream xfer = nullptr;
|
|
483
|
+
cuStreamCreate(&xfer, CU_STREAM_NON_BLOCKING);
|
|
484
|
+
int fd = open_maybe_direct(path, O_WRONLY);
|
|
485
|
+
if (fd < 0) {
|
|
486
|
+
cuMemFreeHost(stage);
|
|
487
|
+
if (err) *err = strerror(errno);
|
|
488
|
+
return -1;
|
|
489
|
+
}
|
|
490
|
+
|
|
491
|
+
cuStreamSynchronize(g_compute); // settle pending kernels' writes before reading
|
|
492
|
+
int rc = 0;
|
|
493
|
+
for (long done = 0; done < bytes;) {
|
|
494
|
+
size_t n = (size_t)((bytes - done) < (long)chunk ? (bytes - done) : (long)chunk);
|
|
495
|
+
if (cuMemcpyDtoHAsync(stage, dptr + (size_t)off + (size_t)done, n, xfer) != CUDA_SUCCESS ||
|
|
496
|
+
cuStreamSynchronize(xfer) != CUDA_SUCCESS) {
|
|
497
|
+
if (err) *err = "device-to-host copy failed during offload";
|
|
498
|
+
rc = -1;
|
|
499
|
+
break;
|
|
500
|
+
}
|
|
501
|
+
// O_DIRECT needs length, offset and address all 4096-aligned. The last
|
|
502
|
+
// chunk writes through its rounded-up zero padding (matching the Ruby
|
|
503
|
+
// path's padding, so both paths' files are bit-identical).
|
|
504
|
+
size_t wn = (n + 4095) & ~(size_t)4095;
|
|
505
|
+
if (wn > n) memset((char *)stage + n, 0, wn - n);
|
|
506
|
+
if (pwrite(fd, stage, wn, OFFLOAD_HEADER + done) != (ssize_t)wn) {
|
|
507
|
+
if (err) *err = strerror(errno);
|
|
508
|
+
rc = -1;
|
|
509
|
+
break;
|
|
510
|
+
}
|
|
511
|
+
done += (long)n;
|
|
512
|
+
}
|
|
513
|
+
|
|
514
|
+
if (rc == 0) fsync(fd);
|
|
515
|
+
close(fd);
|
|
516
|
+
if (xfer) cuStreamDestroy(xfer);
|
|
517
|
+
cuMemFreeHost(stage);
|
|
518
|
+
return rc;
|
|
519
|
+
}
|
|
520
|
+
|
|
521
|
+
int staged_load(CUdeviceptr dptr, const char *path, long bytes, long off, std::string *err) {
|
|
522
|
+
// Staging is sized for the same reason as in staged_save.
|
|
523
|
+
const size_t chunk = staging_bytes(bytes);
|
|
524
|
+
void *stage = nullptr;
|
|
525
|
+
if (cuMemAllocHost(&stage, chunk) != CUDA_SUCCESS) {
|
|
526
|
+
if (err) *err = "failed to allocate pinned staging buffer";
|
|
527
|
+
return -1;
|
|
528
|
+
}
|
|
529
|
+
CUstream xfer = nullptr; // ★ transfer-only stream (see staged_save)
|
|
530
|
+
cuStreamCreate(&xfer, CU_STREAM_NON_BLOCKING);
|
|
531
|
+
int fd = open_maybe_direct(path, O_RDONLY);
|
|
532
|
+
if (fd < 0) {
|
|
533
|
+
cuMemFreeHost(stage);
|
|
534
|
+
if (err) *err = strerror(errno);
|
|
535
|
+
return -1;
|
|
536
|
+
}
|
|
537
|
+
|
|
538
|
+
// The destination was allocated stream-ordered (on g_compute). This is a
|
|
539
|
+
// different stream, so not explicitly waiting for allocation and zero-fill
|
|
540
|
+
// is undefined behavior. staged_save does the same synchronization up front.
|
|
541
|
+
cuStreamSynchronize(g_compute);
|
|
542
|
+
|
|
543
|
+
int rc = 0;
|
|
544
|
+
for (long done = 0; done < bytes;) {
|
|
545
|
+
size_t n = (size_t)((bytes - done) < (long)chunk ? (bytes - done) : (long)chunk);
|
|
546
|
+
size_t rn = (n + 4095) & ~(size_t)4095; // the file is padded
|
|
547
|
+
ssize_t got = pread(fd, stage, rn, OFFLOAD_HEADER + done);
|
|
548
|
+
if (got < (ssize_t)n) {
|
|
549
|
+
if (err) *err = (got < 0) ? strerror(errno) : "offload file is shorter than expected";
|
|
550
|
+
rc = -1;
|
|
551
|
+
break;
|
|
552
|
+
}
|
|
553
|
+
if (cuMemcpyHtoDAsync(dptr + (size_t)off + (size_t)done, stage, n, xfer) != CUDA_SUCCESS ||
|
|
554
|
+
cuStreamSynchronize(xfer) != CUDA_SUCCESS) {
|
|
555
|
+
if (err) *err = "host-to-device copy failed during restore";
|
|
556
|
+
rc = -1;
|
|
557
|
+
break;
|
|
558
|
+
}
|
|
559
|
+
done += (long)n;
|
|
560
|
+
}
|
|
561
|
+
|
|
562
|
+
close(fd);
|
|
563
|
+
if (xfer) cuStreamDestroy(xfer);
|
|
564
|
+
cuMemFreeHost(stage);
|
|
565
|
+
return rc;
|
|
566
|
+
}
|
|
567
|
+
|
|
568
|
+
// State of one async transfer. The bridge has no scheduler; it just runs one
|
|
569
|
+
// worker. The policy of what to prefetch in which order lives in Ruby.
|
|
570
|
+
struct CuTransfer {
|
|
571
|
+
std::thread worker;
|
|
572
|
+
std::atomic<int> status{0}; // 0 = in progress, 1 = done, -1 = error
|
|
573
|
+
std::string err;
|
|
574
|
+
};
|
|
575
|
+
|
|
576
|
+
} // namespace
|
|
577
|
+
|
|
578
|
+
// --- Optional ABI: offload transfers ---
|
|
579
|
+
//
|
|
580
|
+
// Pinned staging + O_DIRECT. Ruby has already written the header (the first
|
|
581
|
+
// 4096 bytes) with ordinary I/O; this reads/writes only the body on 4096
|
|
582
|
+
// boundaries.
|
|
583
|
+
//
|
|
584
|
+
// cuFile (GPUDirect Storage) is not used: on the test machine
|
|
585
|
+
// cuFileDriverOpen succeeds, but nvidia-fs is not loaded, so it only reaches
|
|
586
|
+
// compatibility mode, falls back to staging internally and gains nothing.
|
|
587
|
+
|
|
588
|
+
extern "C" {
|
|
589
|
+
|
|
590
|
+
int gpu_bridge_buffer_save(void *handle, const char *path, long bytes, long off, char **err) {
|
|
591
|
+
if (!ensure_ctx()) {
|
|
592
|
+
if (err) *err = dup_cstr("no CUDA device available");
|
|
593
|
+
return -1;
|
|
594
|
+
}
|
|
595
|
+
std::string msg;
|
|
596
|
+
int rc = staged_save((CUdeviceptr)(uintptr_t)handle, path, bytes, off, &msg);
|
|
597
|
+
if (rc != 0 && err) *err = dup_cstr(msg.c_str());
|
|
598
|
+
return rc;
|
|
599
|
+
}
|
|
600
|
+
|
|
601
|
+
int gpu_bridge_buffer_load(void *handle, const char *path, long bytes, long off, char **err) {
|
|
602
|
+
if (!ensure_ctx()) {
|
|
603
|
+
if (err) *err = dup_cstr("no CUDA device available");
|
|
604
|
+
return -1;
|
|
605
|
+
}
|
|
606
|
+
std::string msg;
|
|
607
|
+
int rc = staged_load((CUdeviceptr)(uintptr_t)handle, path, bytes, off, &msg);
|
|
608
|
+
if (rc != 0 && err) *err = dup_cstr(msg.c_str());
|
|
609
|
+
return rc;
|
|
610
|
+
}
|
|
611
|
+
|
|
612
|
+
// dir: 0=save (VRAM -> SSD), 1=load (SSD -> VRAM)
|
|
613
|
+
void *gpu_bridge_transfer_begin(void *handle, const char *path, long bytes, long off, int dir) {
|
|
614
|
+
if (!ensure_ctx()) return nullptr;
|
|
615
|
+
|
|
616
|
+
CUdeviceptr dptr = (CUdeviceptr)(uintptr_t)handle;
|
|
617
|
+
std::string p(path); // copied so it does not depend on the caller's string lifetime
|
|
618
|
+
auto *t = new CuTransfer();
|
|
619
|
+
t->worker = std::thread([t, dptr, p, bytes, off, dir]() {
|
|
620
|
+
// Bind the context to the worker thread too.
|
|
621
|
+
cuCtxSetCurrent(g_ctx);
|
|
622
|
+
std::string msg;
|
|
623
|
+
int rc = (dir == 0) ? staged_save(dptr, p.c_str(), bytes, off, &msg)
|
|
624
|
+
: staged_load(dptr, p.c_str(), bytes, off, &msg);
|
|
625
|
+
t->err = msg;
|
|
626
|
+
t->status.store(rc == 0 ? 1 : -1);
|
|
627
|
+
});
|
|
628
|
+
return t;
|
|
629
|
+
}
|
|
630
|
+
|
|
631
|
+
int gpu_bridge_transfer_poll(void *handle) {
|
|
632
|
+
auto *t = (CuTransfer *)handle;
|
|
633
|
+
return t ? t->status.load() : -1;
|
|
634
|
+
}
|
|
635
|
+
|
|
636
|
+
int gpu_bridge_transfer_end(void *handle, char **err) {
|
|
637
|
+
auto *t = (CuTransfer *)handle;
|
|
638
|
+
if (!t) return -1;
|
|
639
|
+
|
|
640
|
+
if (t->worker.joinable()) t->worker.join();
|
|
641
|
+
int status = t->status.load();
|
|
642
|
+
if (status != 1 && err) *err = dup_cstr(t->err.empty() ? "offload transfer failed" : t->err.c_str());
|
|
643
|
+
delete t;
|
|
644
|
+
return status == 1 ? 0 : -1;
|
|
645
|
+
}
|
|
646
|
+
|
|
647
|
+
} // extern "C"
|
|
648
|
+
|
|
649
|
+
#ifdef TILEWORK_HAS_CURAND
|
|
650
|
+
// --- Optional ABI: random generation delegated to cuRAND ---
|
|
651
|
+
//
|
|
652
|
+
// Philox is counter-based: the same seed deterministically gives the same
|
|
653
|
+
// sequence. Its state is small and generation fast, which makes it easier to
|
|
654
|
+
// handle than the default XORWOW.
|
|
655
|
+
namespace {
|
|
656
|
+
curandGenerator_t g_rng = nullptr;
|
|
657
|
+
bool g_rng_failed = false;
|
|
658
|
+
|
|
659
|
+
bool ensure_rng() {
|
|
660
|
+
if (g_rng) return true;
|
|
661
|
+
if (g_rng_failed) return false;
|
|
662
|
+
if (curandCreateGenerator(&g_rng, CURAND_RNG_PSEUDO_PHILOX4_32_10) != CURAND_STATUS_SUCCESS) {
|
|
663
|
+
g_rng = nullptr;
|
|
664
|
+
g_rng_failed = true;
|
|
665
|
+
return false;
|
|
666
|
+
}
|
|
667
|
+
// ★ Bind to the compute stream (the memory pool is stream-ordered)
|
|
668
|
+
curandSetStream(g_rng, g_compute);
|
|
669
|
+
return true;
|
|
670
|
+
}
|
|
671
|
+
} // namespace
|
|
672
|
+
|
|
673
|
+
extern "C" {
|
|
674
|
+
|
|
675
|
+
// kind: 0 = uniform (0, 1], 1 = normal (mean, stddev)
|
|
676
|
+
int gpu_bridge_random(void *handle, long n, int kind, unsigned long long seed,
|
|
677
|
+
float mean, float stddev, char **err) {
|
|
678
|
+
if (!ensure_ctx() || !ensure_rng()) {
|
|
679
|
+
if (err) *err = dup_cstr("cuRAND is not available");
|
|
680
|
+
return -1;
|
|
681
|
+
}
|
|
682
|
+
|
|
683
|
+
// Setting the seed alone does not reset the internal offset, so the same
|
|
684
|
+
// seed would continue the sequence on every call. Reset it to 0 explicitly
|
|
685
|
+
// for reproducibility.
|
|
686
|
+
curandSetPseudoRandomGeneratorSeed(g_rng, seed);
|
|
687
|
+
curandSetGeneratorOffset(g_rng, 0);
|
|
688
|
+
float *dst = (float *)(uintptr_t)handle;
|
|
689
|
+
curandStatus_t st;
|
|
690
|
+
|
|
691
|
+
if (kind == 0) {
|
|
692
|
+
st = curandGenerateUniform(g_rng, dst, (size_t)n);
|
|
693
|
+
} else if ((n % 2) == 0) {
|
|
694
|
+
st = curandGenerateNormal(g_rng, dst, (size_t)n, mean, stddev);
|
|
695
|
+
} else {
|
|
696
|
+
// curandGenerateNormal makes pairs via Box-Muller and rejects odd counts.
|
|
697
|
+
// For an odd count, allocate one extra separately and copy the first n.
|
|
698
|
+
CUdeviceptr tmp = 0;
|
|
699
|
+
size_t padded = (size_t)n + 1;
|
|
700
|
+
CUresult ce = g_use_pool ? cuMemAllocAsync(&tmp, padded * sizeof(float), g_compute)
|
|
701
|
+
: cuMemAlloc(&tmp, padded * sizeof(float));
|
|
702
|
+
if (ce != CUDA_SUCCESS) {
|
|
703
|
+
if (err) *err = dup_cstr("failed to allocate scratch for an odd-length normal draw");
|
|
704
|
+
return -1;
|
|
705
|
+
}
|
|
706
|
+
st = curandGenerateNormal(g_rng, (float *)tmp, padded, mean, stddev);
|
|
707
|
+
if (st == CURAND_STATUS_SUCCESS) {
|
|
708
|
+
cuMemcpyDtoDAsync((CUdeviceptr)(uintptr_t)handle, tmp, (size_t)n * sizeof(float), g_compute);
|
|
709
|
+
}
|
|
710
|
+
if (g_use_pool) {
|
|
711
|
+
cuMemFreeAsync(tmp, g_compute);
|
|
712
|
+
} else {
|
|
713
|
+
cuStreamSynchronize(g_compute);
|
|
714
|
+
cuMemFree(tmp);
|
|
715
|
+
}
|
|
716
|
+
}
|
|
717
|
+
|
|
718
|
+
if (st != CURAND_STATUS_SUCCESS) {
|
|
719
|
+
char buf[128];
|
|
720
|
+
snprintf(buf, sizeof(buf), "curandGenerate%s failed (status %d)",
|
|
721
|
+
kind == 0 ? "Uniform" : "Normal", (int)st);
|
|
722
|
+
if (err) *err = dup_cstr(buf);
|
|
723
|
+
return -1;
|
|
724
|
+
}
|
|
725
|
+
return 0;
|
|
726
|
+
}
|
|
727
|
+
|
|
728
|
+
} // extern "C"
|
|
729
|
+
#endif // TILEWORK_HAS_CURAND
|
|
730
|
+
|
|
731
|
+
extern "C" {
|
|
732
|
+
|
|
733
|
+
#ifdef TILEWORK_HAS_CUDNN
|
|
734
|
+
// --- Optional ABI: 2D convolution delegated to cuDNN ---
|
|
735
|
+
//
|
|
736
|
+
// A hand-written im2col + GEMM spends 74–84% of its time in im2col and reaches
|
|
737
|
+
// only 240–613 GF/s (measured). im2col writes out k^2 times the data and is
|
|
738
|
+
// memory-bound, which becomes the bottleneck. cuDNN's implicit GEMM has no
|
|
739
|
+
// expansion at all and reaches fp16 4747 GF/s (7.7x) on the same shape.
|
|
740
|
+
namespace {
|
|
741
|
+
cudnnHandle_t g_dnn = nullptr;
|
|
742
|
+
bool g_dnn_failed = false;
|
|
743
|
+
|
|
744
|
+
bool ensure_dnn() {
|
|
745
|
+
if (g_dnn) return true;
|
|
746
|
+
if (g_dnn_failed) return false;
|
|
747
|
+
if (cudnnCreate(&g_dnn) != CUDNN_STATUS_SUCCESS) {
|
|
748
|
+
g_dnn = nullptr;
|
|
749
|
+
g_dnn_failed = true;
|
|
750
|
+
return false;
|
|
751
|
+
}
|
|
752
|
+
// ★ Bind to the compute stream (the memory pool is stream-ordered)
|
|
753
|
+
cudnnSetStream(g_dnn, g_compute);
|
|
754
|
+
return true;
|
|
755
|
+
}
|
|
756
|
+
|
|
757
|
+
// Descriptor and algorithm selection is expensive, so they are reused per
|
|
758
|
+
// configuration. Descriptors are shared by forward and backward (a gradient
|
|
759
|
+
// has its tensor's shape, so dy uses y's descriptor, dx x's, dw w's).
|
|
760
|
+
struct ConvPlan {
|
|
761
|
+
cudnnTensorDescriptor_t x, y;
|
|
762
|
+
cudnnFilterDescriptor_t w;
|
|
763
|
+
cudnnConvolutionDescriptor_t conv;
|
|
764
|
+
cudnnConvolutionFwdAlgo_t algo;
|
|
765
|
+
cudnnConvolutionBwdDataAlgo_t bwd_data_algo;
|
|
766
|
+
cudnnConvolutionBwdFilterAlgo_t bwd_filter_algo;
|
|
767
|
+
size_t workspace; // forward
|
|
768
|
+
size_t bwd_data_workspace;
|
|
769
|
+
size_t bwd_filter_workspace;
|
|
770
|
+
};
|
|
771
|
+
std::map<std::vector<int>, ConvPlan> g_conv_plans;
|
|
772
|
+
|
|
773
|
+
void conv_plan_destroy(ConvPlan *plan) {
|
|
774
|
+
if (!plan) return;
|
|
775
|
+
if (plan->conv) cudnnDestroyConvolutionDescriptor(plan->conv);
|
|
776
|
+
if (plan->w) cudnnDestroyFilterDescriptor(plan->w);
|
|
777
|
+
if (plan->y) cudnnDestroyTensorDescriptor(plan->y);
|
|
778
|
+
if (plan->x) cudnnDestroyTensorDescriptor(plan->x);
|
|
779
|
+
*plan = ConvPlan{};
|
|
780
|
+
}
|
|
781
|
+
|
|
782
|
+
// Workspace comes from the pool (allocation takes 0.016 ms, so per call is fine).
|
|
783
|
+
bool conv_workspace_alloc(size_t bytes, CUdeviceptr *out) {
|
|
784
|
+
*out = 0;
|
|
785
|
+
if (bytes == 0) return true;
|
|
786
|
+
CUresult ce = g_use_pool ? cuMemAllocAsync(out, bytes, g_compute)
|
|
787
|
+
: cuMemAlloc(out, bytes);
|
|
788
|
+
return ce == CUDA_SUCCESS;
|
|
789
|
+
}
|
|
790
|
+
|
|
791
|
+
void conv_workspace_free(CUdeviceptr ws) {
|
|
792
|
+
if (!ws) return;
|
|
793
|
+
if (g_use_pool) {
|
|
794
|
+
cuMemFreeAsync(ws, g_compute);
|
|
795
|
+
} else {
|
|
796
|
+
cuStreamSynchronize(g_compute);
|
|
797
|
+
cuMemFree(ws);
|
|
798
|
+
}
|
|
799
|
+
}
|
|
800
|
+
} // namespace
|
|
801
|
+
|
|
802
|
+
extern "C" {
|
|
803
|
+
|
|
804
|
+
int gpu_bridge_conv2d(void *x, void *w, void *y,
|
|
805
|
+
int n, int cin, int h, int width,
|
|
806
|
+
int cout, int kh, int kw,
|
|
807
|
+
int pad_h, int pad_w, int stride_h, int stride_w,
|
|
808
|
+
int dilation_h, int dilation_w, int groups,
|
|
809
|
+
int dtype, char **err) {
|
|
810
|
+
if (!ensure_ctx() || !ensure_dnn()) {
|
|
811
|
+
if (err) *err = dup_cstr("cuDNN is not available");
|
|
812
|
+
return -1;
|
|
813
|
+
}
|
|
814
|
+
|
|
815
|
+
std::vector<int> key = {n, cin, h, width, cout, kh, kw, pad_h, pad_w, stride_h, stride_w,
|
|
816
|
+
dilation_h, dilation_w, groups, dtype};
|
|
817
|
+
auto it = g_conv_plans.find(key);
|
|
818
|
+
if (it == g_conv_plans.end()) {
|
|
819
|
+
ConvPlan p{};
|
|
820
|
+
cudnnDataType_t dt = dtype == 1 ? CUDNN_DATA_HALF :
|
|
821
|
+
(dtype == 2 ? CUDNN_DATA_BFLOAT16 : CUDNN_DATA_FLOAT);
|
|
822
|
+
|
|
823
|
+
if (cudnnCreateTensorDescriptor(&p.x) != CUDNN_STATUS_SUCCESS ||
|
|
824
|
+
cudnnCreateTensorDescriptor(&p.y) != CUDNN_STATUS_SUCCESS ||
|
|
825
|
+
cudnnCreateFilterDescriptor(&p.w) != CUDNN_STATUS_SUCCESS ||
|
|
826
|
+
cudnnCreateConvolutionDescriptor(&p.conv) != CUDNN_STATUS_SUCCESS) {
|
|
827
|
+
conv_plan_destroy(&p);
|
|
828
|
+
if (err) *err = dup_cstr("failed to create cuDNN descriptors");
|
|
829
|
+
return -1;
|
|
830
|
+
}
|
|
831
|
+
|
|
832
|
+
cudnnSetTensor4dDescriptor(p.x, CUDNN_TENSOR_NCHW, dt, n, cin, h, width);
|
|
833
|
+
cudnnSetFilter4dDescriptor(p.w, dt, CUDNN_TENSOR_NCHW, cout, cin / groups, kh, kw);
|
|
834
|
+
// Accumulation is always FP32
|
|
835
|
+
cudnnSetConvolution2dDescriptor(p.conv, pad_h, pad_w, stride_h, stride_w, dilation_h, dilation_w,
|
|
836
|
+
CUDNN_CROSS_CORRELATION, CUDNN_DATA_FLOAT);
|
|
837
|
+
cudnnSetConvolutionGroupCount(p.conv, groups);
|
|
838
|
+
if (dtype != 0) cudnnSetConvolutionMathType(p.conv, CUDNN_TENSOR_OP_MATH);
|
|
839
|
+
|
|
840
|
+
int on, oc, oh, ow;
|
|
841
|
+
cudnnGetConvolution2dForwardOutputDim(p.conv, p.x, p.w, &on, &oc, &oh, &ow);
|
|
842
|
+
cudnnSetTensor4dDescriptor(p.y, CUDNN_TENSOR_NCHW, dt, on, oc, oh, ow);
|
|
843
|
+
|
|
844
|
+
int count = 0;
|
|
845
|
+
cudnnConvolutionFwdAlgoPerf_t perf[8];
|
|
846
|
+
if (cudnnGetConvolutionForwardAlgorithm_v7(g_dnn, p.x, p.w, p.conv, p.y, 8, &count, perf)
|
|
847
|
+
!= CUDNN_STATUS_SUCCESS || count == 0) {
|
|
848
|
+
conv_plan_destroy(&p);
|
|
849
|
+
if (err) *err = dup_cstr("cuDNN found no forward convolution algorithm");
|
|
850
|
+
return -1;
|
|
851
|
+
}
|
|
852
|
+
p.algo = perf[0].algo;
|
|
853
|
+
cudnnGetConvolutionForwardWorkspaceSize(g_dnn, p.x, p.w, p.conv, p.y, p.algo, &p.workspace);
|
|
854
|
+
|
|
855
|
+
// Backward algorithms are chosen from the same descriptors here too.
|
|
856
|
+
// Selection is expensive, so it is folded into the plan instead of being
|
|
857
|
+
// redone every training step.
|
|
858
|
+
cudnnConvolutionBwdDataAlgoPerf_t dperf[8];
|
|
859
|
+
count = 0;
|
|
860
|
+
if (cudnnGetConvolutionBackwardDataAlgorithm_v7(g_dnn, p.w, p.y, p.conv, p.x, 8, &count, dperf)
|
|
861
|
+
== CUDNN_STATUS_SUCCESS && count > 0) {
|
|
862
|
+
p.bwd_data_algo = dperf[0].algo;
|
|
863
|
+
cudnnGetConvolutionBackwardDataWorkspaceSize(g_dnn, p.w, p.y, p.conv, p.x,
|
|
864
|
+
p.bwd_data_algo, &p.bwd_data_workspace);
|
|
865
|
+
} else {
|
|
866
|
+
p.bwd_data_workspace = SIZE_MAX; // marks unavailable; errors when called
|
|
867
|
+
}
|
|
868
|
+
|
|
869
|
+
cudnnConvolutionBwdFilterAlgoPerf_t fperf[8];
|
|
870
|
+
count = 0;
|
|
871
|
+
if (cudnnGetConvolutionBackwardFilterAlgorithm_v7(g_dnn, p.x, p.y, p.conv, p.w, 8, &count, fperf)
|
|
872
|
+
== CUDNN_STATUS_SUCCESS && count > 0) {
|
|
873
|
+
p.bwd_filter_algo = fperf[0].algo;
|
|
874
|
+
cudnnGetConvolutionBackwardFilterWorkspaceSize(g_dnn, p.x, p.y, p.conv, p.w,
|
|
875
|
+
p.bwd_filter_algo, &p.bwd_filter_workspace);
|
|
876
|
+
} else {
|
|
877
|
+
p.bwd_filter_workspace = SIZE_MAX;
|
|
878
|
+
}
|
|
879
|
+
|
|
880
|
+
g_conv_plans[key] = p;
|
|
881
|
+
it = g_conv_plans.find(key);
|
|
882
|
+
}
|
|
883
|
+
|
|
884
|
+
ConvPlan &p = it->second;
|
|
885
|
+
|
|
886
|
+
CUdeviceptr ws = 0;
|
|
887
|
+
if (!conv_workspace_alloc(p.workspace, &ws)) {
|
|
888
|
+
if (err) *err = dup_cstr("failed to allocate the cuDNN workspace");
|
|
889
|
+
return -1;
|
|
890
|
+
}
|
|
891
|
+
|
|
892
|
+
const float alpha = 1.0f, beta = 0.0f;
|
|
893
|
+
cudnnStatus_t st = cudnnConvolutionForward(g_dnn, &alpha, p.x, x, p.w, w, p.conv, p.algo,
|
|
894
|
+
(void *)(uintptr_t)ws, p.workspace, &beta, p.y, y);
|
|
895
|
+
conv_workspace_free(ws);
|
|
896
|
+
|
|
897
|
+
if (st != CUDNN_STATUS_SUCCESS) {
|
|
898
|
+
char buf[128];
|
|
899
|
+
snprintf(buf, sizeof(buf), "cudnnConvolutionForward failed (status %d)", (int)st);
|
|
900
|
+
if (err) *err = dup_cstr(buf);
|
|
901
|
+
return -1;
|
|
902
|
+
}
|
|
903
|
+
return 0;
|
|
904
|
+
}
|
|
905
|
+
|
|
906
|
+
} // extern "C"
|
|
907
|
+
#endif // TILEWORK_HAS_CUDNN
|
|
908
|
+
|
|
909
|
+
#ifdef TILEWORK_HAS_CUFFT
|
|
910
|
+
// --- Optional ABI: FFT delegated to cuFFT ---
|
|
911
|
+
//
|
|
912
|
+
// Complex numbers are interleaved float32 (re, im, re, im, ...). That is
|
|
913
|
+
// exactly cufftComplex's layout, so a Buffer can be passed as is.
|
|
914
|
+
namespace {
|
|
915
|
+
// Plan creation is expensive, so plans are reused per (kind, n, batch).
|
|
916
|
+
std::map<std::tuple<int, long, long>, cufftHandle> g_fft_plans;
|
|
917
|
+
|
|
918
|
+
// kind: 0 = R2C, 1 = C2R, 2 = C2C
|
|
919
|
+
bool fft_plan(int kind, long n, long batch, cufftHandle *out) {
|
|
920
|
+
auto key = std::make_tuple(kind, n, batch);
|
|
921
|
+
auto it = g_fft_plans.find(key);
|
|
922
|
+
if (it != g_fft_plans.end()) {
|
|
923
|
+
*out = it->second;
|
|
924
|
+
return true;
|
|
925
|
+
}
|
|
926
|
+
|
|
927
|
+
cufftType type = kind == 0 ? CUFFT_R2C : (kind == 1 ? CUFFT_C2R : CUFFT_C2C);
|
|
928
|
+
cufftHandle plan;
|
|
929
|
+
int len = (int)n;
|
|
930
|
+
if (cufftPlanMany(&plan, 1, &len, nullptr, 1, 0, nullptr, 1, 0, type, (int)batch) != CUFFT_SUCCESS) {
|
|
931
|
+
return false;
|
|
932
|
+
}
|
|
933
|
+
// ★ Bind to the compute stream (the memory pool is stream-ordered)
|
|
934
|
+
cufftSetStream(plan, g_compute);
|
|
935
|
+
g_fft_plans[key] = plan;
|
|
936
|
+
*out = plan;
|
|
937
|
+
return true;
|
|
938
|
+
}
|
|
939
|
+
} // namespace
|
|
940
|
+
|
|
941
|
+
extern "C" {
|
|
942
|
+
|
|
943
|
+
int gpu_bridge_fft(void *in, void *out, long n, long batch, int kind, int direction,
|
|
944
|
+
char **err) {
|
|
945
|
+
if (!ensure_ctx()) {
|
|
946
|
+
if (err) *err = dup_cstr("no CUDA device available");
|
|
947
|
+
return -1;
|
|
948
|
+
}
|
|
949
|
+
|
|
950
|
+
cufftHandle plan;
|
|
951
|
+
if (!fft_plan(kind, n, batch, &plan)) {
|
|
952
|
+
if (err) *err = dup_cstr("cufftPlanMany failed");
|
|
953
|
+
return -1;
|
|
954
|
+
}
|
|
955
|
+
|
|
956
|
+
cufftResult st;
|
|
957
|
+
switch (kind) {
|
|
958
|
+
case 0:
|
|
959
|
+
st = cufftExecR2C(plan, (cufftReal *)(uintptr_t)in, (cufftComplex *)(uintptr_t)out);
|
|
960
|
+
break;
|
|
961
|
+
case 1:
|
|
962
|
+
st = cufftExecC2R(plan, (cufftComplex *)(uintptr_t)in, (cufftReal *)(uintptr_t)out);
|
|
963
|
+
break;
|
|
964
|
+
default:
|
|
965
|
+
st = cufftExecC2C(plan, (cufftComplex *)(uintptr_t)in, (cufftComplex *)(uintptr_t)out,
|
|
966
|
+
direction < 0 ? CUFFT_INVERSE : CUFFT_FORWARD);
|
|
967
|
+
break;
|
|
968
|
+
}
|
|
969
|
+
|
|
970
|
+
if (st != CUFFT_SUCCESS) {
|
|
971
|
+
char buf[128];
|
|
972
|
+
snprintf(buf, sizeof(buf), "cufftExec failed (status %d)", (int)st);
|
|
973
|
+
if (err) *err = dup_cstr(buf);
|
|
974
|
+
return -1;
|
|
975
|
+
}
|
|
976
|
+
return 0;
|
|
977
|
+
}
|
|
978
|
+
|
|
979
|
+
} // extern "C"
|
|
980
|
+
#endif // TILEWORK_HAS_CUFFT
|
|
981
|
+
|
|
982
|
+
#ifdef TILEWORK_HAS_CUBLAS
|
|
983
|
+
// --- Optional ABI: GEMM delegated to cuBLAS ---
|
|
984
|
+
//
|
|
985
|
+
// Without it Ruby falls back to the existing kernel path (WMMA or SIMT).
|
|
986
|
+
// allow_tf32 enables TF32 tensor cores (10-bit mantissa, about 1e-3 relative).
|
|
987
|
+
|
|
988
|
+
// in_dtype: 0 = float32, 1 = float16, 2 = bfloat16. **The output is always
|
|
989
|
+
// float32** (FP32 accumulation, float32 output buffer).
|
|
990
|
+
int gpu_bridge_gemm(void *a, long a_off, void *b, long b_off, void *c, long c_off,
|
|
991
|
+
long m, long k, long n,
|
|
992
|
+
int allow_tf32, int in_dtype, char **err) {
|
|
993
|
+
if (!ensure_ctx() || !ensure_blas()) {
|
|
994
|
+
if (err) *err = dup_cstr("cuBLAS is not available");
|
|
995
|
+
return -1;
|
|
996
|
+
}
|
|
997
|
+
a = (char *)a + a_off;
|
|
998
|
+
b = (char *)b + b_off;
|
|
999
|
+
c = (char *)c + c_off;
|
|
1000
|
+
|
|
1001
|
+
cublasSetMathMode(g_blas, allow_tf32 ? CUBLAS_TF32_TENSOR_OP_MATH : CUBLAS_DEFAULT_MATH);
|
|
1002
|
+
|
|
1003
|
+
// Accumulation is FP32, so alpha/beta are float; never pass __half.
|
|
1004
|
+
const float alpha = 1.0f, beta = 0.0f;
|
|
1005
|
+
cublasStatus_t st;
|
|
1006
|
+
|
|
1007
|
+
// Buffers are row-major but cuBLAS is column-major. Row-major C = A*B equals
|
|
1008
|
+
// computing C^T = B^T * A^T in column-major (no transpose copy needed).
|
|
1009
|
+
if (in_dtype == 1 || in_dtype == 2) {
|
|
1010
|
+
const cudaDataType_t input_type = in_dtype == 1 ? CUDA_R_16F : CUDA_R_16BF;
|
|
1011
|
+
// A/B are CUDA_R_16F and only C is CUDA_R_32F; easy to mix up.
|
|
1012
|
+
st = cublasGemmEx(g_blas, CUBLAS_OP_N, CUBLAS_OP_N,
|
|
1013
|
+
(int)n, (int)m, (int)k,
|
|
1014
|
+
&alpha,
|
|
1015
|
+
(const void *)(uintptr_t)b, input_type, (int)n,
|
|
1016
|
+
(const void *)(uintptr_t)a, input_type, (int)k,
|
|
1017
|
+
&beta,
|
|
1018
|
+
(void *)(uintptr_t)c, CUDA_R_32F, (int)n,
|
|
1019
|
+
CUBLAS_COMPUTE_32F, CUBLAS_GEMM_DEFAULT);
|
|
1020
|
+
} else {
|
|
1021
|
+
st = cublasSgemm(g_blas, CUBLAS_OP_N, CUBLAS_OP_N,
|
|
1022
|
+
(int)n, (int)m, (int)k,
|
|
1023
|
+
&alpha,
|
|
1024
|
+
(const float *)(uintptr_t)b, (int)n,
|
|
1025
|
+
(const float *)(uintptr_t)a, (int)k,
|
|
1026
|
+
&beta,
|
|
1027
|
+
(float *)(uintptr_t)c, (int)n);
|
|
1028
|
+
}
|
|
1029
|
+
|
|
1030
|
+
if (st != CUBLAS_STATUS_SUCCESS) {
|
|
1031
|
+
char buf[160];
|
|
1032
|
+
const char *operation = in_dtype == 1 ? "GemmEx (fp16)" :
|
|
1033
|
+
(in_dtype == 2 ? "GemmEx (bf16)" : "Sgemm");
|
|
1034
|
+
snprintf(buf, sizeof(buf), "cublas%s failed (status %d)",
|
|
1035
|
+
operation, (int)st);
|
|
1036
|
+
if (err) *err = dup_cstr(buf);
|
|
1037
|
+
return -1;
|
|
1038
|
+
}
|
|
1039
|
+
return 0;
|
|
1040
|
+
}
|
|
1041
|
+
|
|
1042
|
+
// --- Optional ABI: strided batched GEMM ---
|
|
1043
|
+
//
|
|
1044
|
+
// Issues `batch` GEMMs of the same shape in one call. Matrices spaced at even
|
|
1045
|
+
// intervals need not be gathered (stride_* is how many elements to the next
|
|
1046
|
+
// matrix). Multi-head attention's heads are laid out exactly like that, so
|
|
1047
|
+
// slicing heads out and writing them back disappears entirely.
|
|
1048
|
+
//
|
|
1049
|
+
// ld_* is how many elements apart one column of the matrix is (the
|
|
1050
|
+
// column-major leading dimension). To view a row-major [R, C] as part of a
|
|
1051
|
+
// larger matrix, pass **the parent's column count** as ld; that is why heads
|
|
1052
|
+
// can be views.
|
|
1053
|
+
//
|
|
1054
|
+
// Unlike gemm(), the caller builds m/k/n and ld/stride in column-major
|
|
1055
|
+
// (reinterpreting from row-major stays in core/lib/tilework/math/nn.rb).
|
|
1056
|
+
int gpu_bridge_gemm_strided_batched(void *a, long a_off, void *b, long b_off,
|
|
1057
|
+
void *c, long c_off,
|
|
1058
|
+
long m, long k, long n,
|
|
1059
|
+
long lda, long ldb, long ldc,
|
|
1060
|
+
long stride_a, long stride_b, long stride_c,
|
|
1061
|
+
int trans_a, int trans_b,
|
|
1062
|
+
long batch, double alpha_in,
|
|
1063
|
+
int allow_tf32, int in_dtype,
|
|
1064
|
+
char **err) {
|
|
1065
|
+
if (!ensure_ctx() || !ensure_blas()) {
|
|
1066
|
+
if (err) *err = dup_cstr("cuBLAS is not available");
|
|
1067
|
+
return -1;
|
|
1068
|
+
}
|
|
1069
|
+
a = (char *)a + a_off;
|
|
1070
|
+
b = (char *)b + b_off;
|
|
1071
|
+
c = (char *)c + c_off;
|
|
1072
|
+
|
|
1073
|
+
cublasSetMathMode(g_blas, allow_tf32 ? CUBLAS_TF32_TENSOR_OP_MATH : CUBLAS_DEFAULT_MATH);
|
|
1074
|
+
|
|
1075
|
+
// Accumulation is FP32, so alpha/beta are float. A scale after the product,
|
|
1076
|
+
// like attention's 1/sqrt(dh), is applied for free through alpha.
|
|
1077
|
+
const float alpha = (float)alpha_in, beta = 0.0f;
|
|
1078
|
+
const cublasOperation_t op_a = trans_a ? CUBLAS_OP_T : CUBLAS_OP_N;
|
|
1079
|
+
const cublasOperation_t op_b = trans_b ? CUBLAS_OP_T : CUBLAS_OP_N;
|
|
1080
|
+
const cudaDataType_t in_type = in_dtype == 1 ? CUDA_R_16F :
|
|
1081
|
+
(in_dtype == 2 ? CUDA_R_16BF : CUDA_R_32F);
|
|
1082
|
+
|
|
1083
|
+
cublasStatus_t st = cublasGemmStridedBatchedEx(
|
|
1084
|
+
g_blas, op_a, op_b,
|
|
1085
|
+
(int)m, (int)n, (int)k,
|
|
1086
|
+
&alpha,
|
|
1087
|
+
(const void *)(uintptr_t)a, in_type, (int)lda, (long long)stride_a,
|
|
1088
|
+
(const void *)(uintptr_t)b, in_type, (int)ldb, (long long)stride_b,
|
|
1089
|
+
&beta,
|
|
1090
|
+
(void *)(uintptr_t)c, CUDA_R_32F, (int)ldc, (long long)stride_c,
|
|
1091
|
+
(int)batch, CUBLAS_COMPUTE_32F, CUBLAS_GEMM_DEFAULT);
|
|
1092
|
+
|
|
1093
|
+
if (st != CUBLAS_STATUS_SUCCESS) {
|
|
1094
|
+
char buf[160];
|
|
1095
|
+
snprintf(buf, sizeof(buf),
|
|
1096
|
+
"cublasGemmStridedBatchedEx failed (status %d, batch %ld)", (int)st, batch);
|
|
1097
|
+
if (err) *err = dup_cstr(buf);
|
|
1098
|
+
return -1;
|
|
1099
|
+
}
|
|
1100
|
+
return 0;
|
|
1101
|
+
}
|
|
1102
|
+
#endif // TILEWORK_HAS_CUBLAS
|
|
1103
|
+
|
|
1104
|
+
// --- Optional ABI: capability queries ---
|
|
1105
|
+
//
|
|
1106
|
+
// Not a required ABI. gpu_bridge.c references it as a weak symbol, so linking
|
|
1107
|
+
// a bridge without it (bridge.swift) does not break, and Ruby sees that the
|
|
1108
|
+
// query is missing and falls back.
|
|
1109
|
+
int gpu_bridge_device_capabilities(int32_t *warp_size, int32_t *cc_major, int32_t *cc_minor,
|
|
1110
|
+
int32_t *integrated, long *memory_budget_bytes) {
|
|
1111
|
+
if (!ensure_ctx()) return -1;
|
|
1112
|
+
|
|
1113
|
+
int v = 0;
|
|
1114
|
+
cuDeviceGetAttribute(&v, CU_DEVICE_ATTRIBUTE_WARP_SIZE, g_dev);
|
|
1115
|
+
*warp_size = v;
|
|
1116
|
+
v = 0;
|
|
1117
|
+
cuDeviceGetAttribute(&v, CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR, g_dev);
|
|
1118
|
+
*cc_major = v;
|
|
1119
|
+
v = 0;
|
|
1120
|
+
cuDeviceGetAttribute(&v, CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MINOR, g_dev);
|
|
1121
|
+
*cc_minor = v;
|
|
1122
|
+
// On Jetson and APUs the GPU and CPU share the same physical memory; this
|
|
1123
|
+
// feeds the offload staging limit.
|
|
1124
|
+
v = 0;
|
|
1125
|
+
cuDeviceGetAttribute(&v, CU_DEVICE_ATTRIBUTE_INTEGRATED, g_dev);
|
|
1126
|
+
*integrated = v;
|
|
1127
|
+
|
|
1128
|
+
size_t free_bytes = 0, total_bytes = 0;
|
|
1129
|
+
*memory_budget_bytes = cuMemGetInfo(&free_bytes, &total_bytes) == CUDA_SUCCESS ? (long)free_bytes : 0;
|
|
1130
|
+
return 0;
|
|
1131
|
+
}
|
|
1132
|
+
|
|
1133
|
+
#ifdef TILEWORK_HAS_CUDNN
|
|
1134
|
+
// Backward. mode 0 = input gradient (dx), mode 1 = weight gradient (dw).
|
|
1135
|
+
// The arguments mean different things per mode:
|
|
1136
|
+
// mode 0: a = w (weight), b = dy (output gradient), out = dx
|
|
1137
|
+
// mode 1: a = x (input), b = dy (output gradient), out = dw
|
|
1138
|
+
// Descriptors are shared with the forward plan (a gradient has its tensor's shape).
|
|
1139
|
+
int gpu_bridge_conv2d_backward(void *a, void *b, void *out,
|
|
1140
|
+
int n, int cin, int h, int width,
|
|
1141
|
+
int cout, int kh, int kw,
|
|
1142
|
+
int pad_h, int pad_w, int stride_h, int stride_w,
|
|
1143
|
+
int dilation_h, int dilation_w, int groups,
|
|
1144
|
+
int dtype, int mode, char **err) {
|
|
1145
|
+
if (!ensure_ctx() || !ensure_dnn()) {
|
|
1146
|
+
if (err) *err = dup_cstr("cuDNN is not available");
|
|
1147
|
+
return -1;
|
|
1148
|
+
}
|
|
1149
|
+
|
|
1150
|
+
std::vector<int> key = {n, cin, h, width, cout, kh, kw, pad_h, pad_w, stride_h, stride_w,
|
|
1151
|
+
dilation_h, dilation_w, groups, dtype};
|
|
1152
|
+
auto it = g_conv_plans.find(key);
|
|
1153
|
+
if (it == g_conv_plans.end()) {
|
|
1154
|
+
// The forward pass creates the plan. Training always runs forward first, so
|
|
1155
|
+
// a missing plan is rejected explicitly as a call-order error.
|
|
1156
|
+
if (err) *err = dup_cstr("no cached convolution plan; run the forward pass first");
|
|
1157
|
+
return -1;
|
|
1158
|
+
}
|
|
1159
|
+
|
|
1160
|
+
ConvPlan &p = it->second;
|
|
1161
|
+
size_t need = mode == 0 ? p.bwd_data_workspace : p.bwd_filter_workspace;
|
|
1162
|
+
if (need == SIZE_MAX) {
|
|
1163
|
+
if (err) {
|
|
1164
|
+
*err = dup_cstr(mode == 0 ? "cuDNN found no backward-data convolution algorithm"
|
|
1165
|
+
: "cuDNN found no backward-filter convolution algorithm");
|
|
1166
|
+
}
|
|
1167
|
+
return -1;
|
|
1168
|
+
}
|
|
1169
|
+
|
|
1170
|
+
CUdeviceptr ws = 0;
|
|
1171
|
+
if (!conv_workspace_alloc(need, &ws)) {
|
|
1172
|
+
if (err) *err = dup_cstr("failed to allocate the cuDNN workspace");
|
|
1173
|
+
return -1;
|
|
1174
|
+
}
|
|
1175
|
+
|
|
1176
|
+
const float alpha = 1.0f, beta = 0.0f;
|
|
1177
|
+
cudnnStatus_t st;
|
|
1178
|
+
if (mode == 0) {
|
|
1179
|
+
st = cudnnConvolutionBackwardData(g_dnn, &alpha, p.w, a, p.y, b, p.conv,
|
|
1180
|
+
p.bwd_data_algo, (void *)(uintptr_t)ws, need,
|
|
1181
|
+
&beta, p.x, out);
|
|
1182
|
+
} else {
|
|
1183
|
+
st = cudnnConvolutionBackwardFilter(g_dnn, &alpha, p.x, a, p.y, b, p.conv,
|
|
1184
|
+
p.bwd_filter_algo, (void *)(uintptr_t)ws, need,
|
|
1185
|
+
&beta, p.w, out);
|
|
1186
|
+
}
|
|
1187
|
+
conv_workspace_free(ws);
|
|
1188
|
+
|
|
1189
|
+
if (st != CUDNN_STATUS_SUCCESS) {
|
|
1190
|
+
char buf[160];
|
|
1191
|
+
snprintf(buf, sizeof(buf), "%s failed (status %d)",
|
|
1192
|
+
mode == 0 ? "cudnnConvolutionBackwardData" : "cudnnConvolutionBackwardFilter",
|
|
1193
|
+
(int)st);
|
|
1194
|
+
if (err) *err = dup_cstr(buf);
|
|
1195
|
+
return -1;
|
|
1196
|
+
}
|
|
1197
|
+
return 0;
|
|
1198
|
+
}
|
|
1199
|
+
#endif // TILEWORK_HAS_CUDNN
|
|
1200
|
+
|
|
1201
|
+
// Ruby's end proc calls this while the CUDA runtime is still usable. These
|
|
1202
|
+
// caches own native descriptors/plans independently of Ruby TypedData, so OS
|
|
1203
|
+
// process teardown is not an adequate ownership boundary (embedding and
|
|
1204
|
+
// backend reloads can keep the process alive).
|
|
1205
|
+
void gpu_bridge_shutdown(void) {
|
|
1206
|
+
if (!g_ctx) return;
|
|
1207
|
+
cuCtxSetCurrent(g_ctx);
|
|
1208
|
+
if (g_compute) cuStreamSynchronize(g_compute);
|
|
1209
|
+
|
|
1210
|
+
#ifdef TILEWORK_HAS_CUDNN
|
|
1211
|
+
for (auto &entry : g_conv_plans) conv_plan_destroy(&entry.second);
|
|
1212
|
+
g_conv_plans.clear();
|
|
1213
|
+
if (g_dnn) {
|
|
1214
|
+
cudnnDestroy(g_dnn);
|
|
1215
|
+
g_dnn = nullptr;
|
|
1216
|
+
}
|
|
1217
|
+
#endif
|
|
1218
|
+
#ifdef TILEWORK_HAS_CUFFT
|
|
1219
|
+
for (auto &entry : g_fft_plans) cufftDestroy(entry.second);
|
|
1220
|
+
g_fft_plans.clear();
|
|
1221
|
+
#endif
|
|
1222
|
+
#ifdef TILEWORK_HAS_CUBLAS
|
|
1223
|
+
if (g_blas) {
|
|
1224
|
+
cublasDestroy(g_blas);
|
|
1225
|
+
g_blas = nullptr;
|
|
1226
|
+
}
|
|
1227
|
+
#endif
|
|
1228
|
+
#ifdef TILEWORK_HAS_CURAND
|
|
1229
|
+
if (g_rng) {
|
|
1230
|
+
curandDestroyGenerator(g_rng);
|
|
1231
|
+
g_rng = nullptr;
|
|
1232
|
+
}
|
|
1233
|
+
#endif
|
|
1234
|
+
}
|
|
1235
|
+
|
|
1236
|
+
} // extern "C"
|