cumo 0.5.5 → 0.5.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +52 -0
- data/Rakefile +11 -2
- data/ext/cumo/cuda/driver.c +40 -15
- data/ext/cumo/cuda/memory_pool.cpp +13 -8
- data/ext/cumo/cuda/memory_pool_impl.cpp +80 -26
- data/ext/cumo/cuda/memory_pool_impl.hpp +94 -25
- data/ext/cumo/cuda/memory_pool_impl_test.cpp +245 -1
- data/ext/cumo/cuda/nvrtc.c +101 -21
- data/ext/cumo/depend.erb +1 -1
- data/ext/cumo/extconf.rb +1 -0
- data/ext/cumo/include/cumo/cuda/handle.h +85 -0
- data/ext/cumo/include/cumo/types/float_macro.h +6 -1
- data/ext/cumo/include/cumo/types/robj_macro.h +1 -8
- data/ext/cumo/include/cumo/types/robject.h +1 -1
- data/ext/cumo/include/cumo.h +1 -1
- data/ext/cumo/narray/data.c +27 -11
- data/ext/cumo/narray/gen/spec.rb +1 -0
- data/ext/cumo/narray/gen/tmpl/alloc_func.c +7 -5
- data/ext/cumo/narray/gen/tmpl/format.c +8 -0
- data/ext/cumo/narray/gen/tmpl/qsort.c +2 -5
- data/ext/cumo/narray/gen/tmpl/store_array.c +26 -8
- data/ext/cumo/narray/index.c +91 -42
- data/ext/cumo/narray/narray.c +28 -13
- data/lib/cumo/cuda/nvrtc_program.rb +3 -1
- data/lib/cumo/narray/extra.rb +42 -1
- data/test/cuda/driver_test.rb +92 -0
- data/test/cuda/memory_pool_test.rb +10 -0
- data/test/cuda/nvrtc_test.rb +68 -0
- data/test/narray_test.rb +494 -0
- metadata +5 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 3ddc7b756681afdf9fccd45652c422d2f67accfd1cf238d1fd05f6827fb5029d
|
|
4
|
+
data.tar.gz: 47848503fd8a388b7dfee79d2da0e627032fd4d8c8717645da70d8f09770f599
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: f59ddc27c8f96a5eceec7fb4ae6665d1fae6dd20ef039a83563c0e0f5a76780fd2ab9b5ebbe06e116a3f01cd59c7493eee06dc6c1166a0bb82ba546c43331f03
|
|
7
|
+
data.tar.gz: 16543c06d74f79089799442abc30741ffb2a26556da953410023b19711fd25e15adb05548f9664c79bf8430a3be9c8f15c0813dccfae4b5e918ffc55a5fc68c0
|
data/CHANGELOG.md
CHANGED
|
@@ -1,3 +1,55 @@
|
|
|
1
|
+
# 0.5.7 (2026/08/13)
|
|
2
|
+
|
|
3
|
+
Breaking changes:
|
|
4
|
+
|
|
5
|
+
* `reshape` raises `ArgumentError` instead of `RangeError` for a dimension above `INT_MAX` (PR #183)
|
|
6
|
+
* `parse` accepts only numeric literals and `true`/`false`/`nil`, so `parse("3/4")` raises `ArgumentError` instead of returning `[[0]]` (PR #190)
|
|
7
|
+
|
|
8
|
+
Fixes:
|
|
9
|
+
|
|
10
|
+
* Fix segfaults from CUDA handles the process did not create, and from destroying one twice (PR #192)
|
|
11
|
+
* Fix process abort from freeing NArray data with the wrong allocator, including a raise out of GC for `Cumo::RObject` (PR #191)
|
|
12
|
+
* Fix arbitrary code execution in `parse`, which ran `eval` on every token (PR #190)
|
|
13
|
+
* Fix stack buffer overflow in `Cumo::RObject#format` with an element longer than 47 characters (PR #189)
|
|
14
|
+
* Fix a freed chunk returning to the arena of the wrong stream (PR #187)
|
|
15
|
+
* Fix `Malloc` searching the free list with the default stream's index (PR #186)
|
|
16
|
+
* Fix data race on the memory pool chunk graph and on the per-device pool table (PR #185)
|
|
17
|
+
* Fix a SIGFPE on fractional range steps and an out-of-bounds read on newaxis (PR #184)
|
|
18
|
+
* Fix a SIGFPE and a silently corrupt shape from unchecked `reshape` arguments (PR #183)
|
|
19
|
+
* Fix segfault when `store_binary` receives a non-String (PR #182)
|
|
20
|
+
* Fix missing range check on NArray indices, which segfaulted on a negative one (PR #181)
|
|
21
|
+
|
|
22
|
+
Changes:
|
|
23
|
+
|
|
24
|
+
* Compile the memory pool test harness in CI, and run it from `rake test` (PR #188)
|
|
25
|
+
|
|
26
|
+
# 0.5.6 (2026/08/09)
|
|
27
|
+
|
|
28
|
+
Fixes:
|
|
29
|
+
|
|
30
|
+
* Fix out-of-tree build failing to create the narray/types object directory
|
|
31
|
+
* Fix segfaults from unchecked String arguments in Driver (#179)
|
|
32
|
+
* Fix out-of-bounds read of include_names in nvrtcCreateProgram (#178)
|
|
33
|
+
* Fix store into a strided view reading a released staging buffer (#177)
|
|
34
|
+
* Fix pinned host memory leak when indexing by an array (#176)
|
|
35
|
+
* Fix segfault in Module#get_global_var reading device memory as a host pointer (#174)
|
|
36
|
+
* Fix freeing pooled memory with cudaFree after disabling the pool (#173)
|
|
37
|
+
* Fix run-ctest failing to start the test binary (#172)
|
|
38
|
+
* Fix process abort when freeing device memory fails (#171)
|
|
39
|
+
* Fix memory pool handing out a tiny chunk for a huge allocation (#170)
|
|
40
|
+
* Fix out-of-bounds read of a shorter sub-narray in store (#169)
|
|
41
|
+
* Fix out-of-bounds device write in store when a sub-narray is too long (#168)
|
|
42
|
+
* Backport: fix na_flatten_dim for multi-dimensional empty arrays (#167)
|
|
43
|
+
* Backport: make qsort loop condition explicit to prevent incorrect optimization (#166)
|
|
44
|
+
* Backport: free previously allocated shape in cumo_na_alloc_shape (#165)
|
|
45
|
+
* Backport: free shape on deallocation regardless of size (#164)
|
|
46
|
+
* Backport: convert max to double in rand method of Cumo::RObject (#163)
|
|
47
|
+
* Backport: use an inline function to prevent double use of macro argument (#162)
|
|
48
|
+
* Fix out-of-bounds stridx access in at() when :new is given (#161)
|
|
49
|
+
* Backport: prevent out-of-bounds access to stridx when orig_dim exceeds ndim (#160)
|
|
50
|
+
* Backport: prevent negative array index in na_get_strides_nadata when ndim is zero (#159)
|
|
51
|
+
* Backport: use correct array index for mod result in divmod macro for Cumo::RObject (#158)
|
|
52
|
+
|
|
1
53
|
# 0.5.5 (2026/07/11)
|
|
2
54
|
|
|
3
55
|
Fixes:
|
data/Rakefile
CHANGED
|
@@ -12,10 +12,19 @@ end
|
|
|
12
12
|
require "rake/extensiontask"
|
|
13
13
|
Rake::ExtensionTask.new("cumo")
|
|
14
14
|
|
|
15
|
-
|
|
16
|
-
|
|
15
|
+
desc 'Build the C++ memory pool test harness without running it (running it needs a GPU)'
|
|
16
|
+
task :build_ctest do
|
|
17
|
+
sh 'cd ext/cumo && ruby extconf.rb && make build-ctest'
|
|
17
18
|
end
|
|
18
19
|
|
|
20
|
+
task :ctest => :build_ctest do
|
|
21
|
+
sh 'cd ext/cumo && make run-ctest'
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
# Both need a GPU, so CI runs neither. Tie them together so the one place
|
|
25
|
+
# they do run does not skip half of them.
|
|
26
|
+
task :test => :ctest
|
|
27
|
+
|
|
19
28
|
task :docs do
|
|
20
29
|
dir = "ext/cumo"
|
|
21
30
|
srcs = %w[array.c data.c index.c math.c narray.c rand.c struct.c].map { |s| File.join(dir, "narray", s) }
|
data/ext/cumo/cuda/driver.c
CHANGED
|
@@ -3,12 +3,16 @@
|
|
|
3
3
|
#include <cuda.h>
|
|
4
4
|
#include <cuda_runtime.h>
|
|
5
5
|
#include "cumo/cuda/driver.h"
|
|
6
|
+
#include "cumo/cuda/handle.h"
|
|
6
7
|
|
|
7
8
|
VALUE cumo_cuda_eDriverError;
|
|
8
9
|
VALUE cumo_cuda_mDriver;
|
|
9
10
|
#define eDriverError cumo_cuda_eDriverError
|
|
10
11
|
#define mDriver cumo_cuda_mDriver
|
|
11
12
|
|
|
13
|
+
static cumo_cuda_handle_set_t link_states;
|
|
14
|
+
static cumo_cuda_handle_set_t modules;
|
|
15
|
+
|
|
12
16
|
static void
|
|
13
17
|
check_status(CUresult status)
|
|
14
18
|
{
|
|
@@ -100,17 +104,21 @@ cuLinkAddData_without_gvl_cb(void *param)
|
|
|
100
104
|
static VALUE
|
|
101
105
|
rb_cuLinkAddData(VALUE self, VALUE state, VALUE type, VALUE data, VALUE name)
|
|
102
106
|
{
|
|
103
|
-
CUlinkState _state = (CUlinkState)NUM2SIZET(state);
|
|
104
107
|
CUjitInputType _type = (CUjitInputType)NUM2INT(type);
|
|
105
|
-
|
|
108
|
+
// The image may be a cubin, so it is taken by length and allowed to hold
|
|
109
|
+
// NUL bytes; the name is a plain C string cuLinkAddData reports errors with.
|
|
110
|
+
void* _data = (void *)StringValuePtr(data);
|
|
106
111
|
size_t _size = RSTRING_LEN(data);
|
|
107
|
-
const char* _name =
|
|
112
|
+
const char* _name = StringValueCStr(name);
|
|
113
|
+
CUlinkState _state = (CUlinkState)cumo_cuda_handle_get(&link_states, state, "CUlinkState");
|
|
108
114
|
CUresult status;
|
|
109
115
|
|
|
110
116
|
struct cuLinkAddDataParam param = {_state, _type, _data, _size, _name, 0, (CUjit_option*)0, (void**)0};
|
|
111
117
|
status = (CUresult)rb_thread_call_without_gvl(cuLinkAddData_without_gvl_cb, ¶m, NULL, NULL);
|
|
112
118
|
//status = cuLinkAddData(_state, _type, _data, _size, _name, 0, (CUjit_option*)0, (void**)0);
|
|
113
119
|
|
|
120
|
+
RB_GC_GUARD(data);
|
|
121
|
+
RB_GC_GUARD(name);
|
|
114
122
|
check_status(status);
|
|
115
123
|
return Qnil;
|
|
116
124
|
}
|
|
@@ -137,15 +145,16 @@ cuLinkAddFile_without_gvl_cb(void *param)
|
|
|
137
145
|
static VALUE
|
|
138
146
|
rb_cuLinkAddFile(VALUE self, VALUE state, VALUE type, VALUE path)
|
|
139
147
|
{
|
|
140
|
-
CUlinkState _state = (CUlinkState)NUM2SIZET(state);
|
|
141
148
|
CUjitInputType _type = (CUjitInputType)NUM2INT(type);
|
|
142
|
-
const char* _path =
|
|
149
|
+
const char* _path = StringValueCStr(path);
|
|
150
|
+
CUlinkState _state = (CUlinkState)cumo_cuda_handle_get(&link_states, state, "CUlinkState");
|
|
143
151
|
CUresult status;
|
|
144
152
|
|
|
145
153
|
struct cuLinkAddFileParam param = {_state, _type, _path, 0, (CUjit_option*)0, (void **)0};
|
|
146
154
|
status = (CUresult)rb_thread_call_without_gvl(cuLinkAddFile_without_gvl_cb, ¶m, NULL, NULL);
|
|
147
155
|
//status = cuLinkAddFile(_state, _type, _path, 0, (CUjit_option*)0, (void **)0);
|
|
148
156
|
|
|
157
|
+
RB_GC_GUARD(path);
|
|
149
158
|
check_status(status);
|
|
150
159
|
return Qnil;
|
|
151
160
|
}
|
|
@@ -168,7 +177,7 @@ cuLinkComplete_without_gvl_cb(void *param)
|
|
|
168
177
|
static VALUE
|
|
169
178
|
rb_cuLinkComplete(VALUE self, VALUE state)
|
|
170
179
|
{
|
|
171
|
-
CUlinkState _state = (CUlinkState)
|
|
180
|
+
CUlinkState _state = (CUlinkState)cumo_cuda_handle_get(&link_states, state, "CUlinkState");
|
|
172
181
|
void* _cubinOut;
|
|
173
182
|
size_t _sizeOut;
|
|
174
183
|
CUresult status;
|
|
@@ -209,6 +218,7 @@ rb_cuLinkCreate(VALUE self)
|
|
|
209
218
|
//status = cuLinkCreate(0, (CUjit_option*)0, (void**)0, &state);
|
|
210
219
|
|
|
211
220
|
check_status(status);
|
|
221
|
+
cumo_cuda_handle_set_add(&link_states, (size_t)state);
|
|
212
222
|
return SIZET2NUM((size_t)state);
|
|
213
223
|
}
|
|
214
224
|
|
|
@@ -228,7 +238,7 @@ cuLinkDestroy_without_gvl_cb(void *param)
|
|
|
228
238
|
static VALUE
|
|
229
239
|
rb_cuLinkDestroy(VALUE self, VALUE state)
|
|
230
240
|
{
|
|
231
|
-
CUlinkState _state = (CUlinkState)
|
|
241
|
+
CUlinkState _state = (CUlinkState)cumo_cuda_handle_take(&link_states, state, "CUlinkState");
|
|
232
242
|
CUresult status;
|
|
233
243
|
|
|
234
244
|
struct cuLinkDestroyParam param = {_state};
|
|
@@ -258,14 +268,15 @@ static VALUE
|
|
|
258
268
|
rb_cuModuleGetFunction(VALUE self, VALUE hmod, VALUE name)
|
|
259
269
|
{
|
|
260
270
|
CUfunction _hfunc;
|
|
261
|
-
|
|
262
|
-
|
|
271
|
+
const char* _name = StringValueCStr(name);
|
|
272
|
+
CUmodule _hmod = (CUmodule)cumo_cuda_handle_get(&modules, hmod, "CUmodule");
|
|
263
273
|
CUresult status;
|
|
264
274
|
|
|
265
275
|
struct cuModuleGetFunctionParam param = {&_hfunc, _hmod, _name};
|
|
266
276
|
status = (CUresult)rb_thread_call_without_gvl(cuModuleGetFunction_without_gvl_cb, ¶m, NULL, NULL);
|
|
267
277
|
//status = cuModuleGetFunction(&_hfunc, _hmod, _name);
|
|
268
278
|
|
|
279
|
+
RB_GC_GUARD(name);
|
|
269
280
|
check_status(status);
|
|
270
281
|
return SIZET2NUM((size_t)_hfunc);
|
|
271
282
|
}
|
|
@@ -291,16 +302,22 @@ rb_cuModuleGetGlobal(VALUE self, VALUE hmod, VALUE name)
|
|
|
291
302
|
{
|
|
292
303
|
CUdeviceptr _dptr;
|
|
293
304
|
size_t _bytes;
|
|
294
|
-
|
|
295
|
-
|
|
305
|
+
const char* _name = StringValueCStr(name);
|
|
306
|
+
CUmodule _hmod = (CUmodule)cumo_cuda_handle_get(&modules, hmod, "CUmodule");
|
|
296
307
|
CUresult status;
|
|
308
|
+
VALUE ret;
|
|
297
309
|
|
|
298
310
|
struct cuModuleGetGlobalParam param = {&_dptr, &_bytes, _hmod, _name};
|
|
299
311
|
status = (CUresult)rb_thread_call_without_gvl(cuModuleGetGlobal_without_gvl_cb, ¶m, NULL, NULL);
|
|
300
312
|
//status = cuModuleGetGlobal(&_dptr, &_bytes, _hmod, _name);
|
|
301
313
|
|
|
314
|
+
RB_GC_GUARD(name);
|
|
302
315
|
check_status(status);
|
|
303
|
-
|
|
316
|
+
|
|
317
|
+
// _dptr addresses device memory, which the host cannot read directly.
|
|
318
|
+
ret = rb_str_new(NULL, (long)_bytes);
|
|
319
|
+
check_status(cuMemcpyDtoH(RSTRING_PTR(ret), _dptr, _bytes));
|
|
320
|
+
return ret;
|
|
304
321
|
}
|
|
305
322
|
|
|
306
323
|
struct cuModuleLoadParam {
|
|
@@ -321,14 +338,16 @@ static VALUE
|
|
|
321
338
|
rb_cuModuleLoad(VALUE self, VALUE fname)
|
|
322
339
|
{
|
|
323
340
|
CUmodule _module;
|
|
324
|
-
const char* _fname =
|
|
341
|
+
const char* _fname = StringValueCStr(fname);
|
|
325
342
|
CUresult status;
|
|
326
343
|
|
|
327
344
|
struct cuModuleLoadParam param = {&_module, _fname};
|
|
328
345
|
status = (CUresult)rb_thread_call_without_gvl(cuModuleLoad_without_gvl_cb, ¶m, NULL, NULL);
|
|
329
346
|
//status = cuModuleLoad(&_module, _fname);
|
|
330
347
|
|
|
348
|
+
RB_GC_GUARD(fname);
|
|
331
349
|
check_status(status);
|
|
350
|
+
cumo_cuda_handle_set_add(&modules, (size_t)_module);
|
|
332
351
|
return SIZET2NUM((size_t)_module);
|
|
333
352
|
}
|
|
334
353
|
|
|
@@ -350,14 +369,17 @@ static VALUE
|
|
|
350
369
|
rb_cuModuleLoadData(VALUE self, VALUE image)
|
|
351
370
|
{
|
|
352
371
|
CUmodule _module;
|
|
353
|
-
|
|
372
|
+
// A cubin is binary, so the image is not required to be NUL-free.
|
|
373
|
+
const void* _image = (void*)StringValuePtr(image);
|
|
354
374
|
CUresult status;
|
|
355
375
|
|
|
356
376
|
struct cuModuleLoadDataParam param = {&_module, _image};
|
|
357
377
|
status = (CUresult)rb_thread_call_without_gvl(cuModuleLoadData_without_gvl_cb, ¶m, NULL, NULL);
|
|
358
378
|
//status = cuModuleLoadData(&_module, _image);
|
|
359
379
|
|
|
380
|
+
RB_GC_GUARD(image);
|
|
360
381
|
check_status(status);
|
|
382
|
+
cumo_cuda_handle_set_add(&modules, (size_t)_module);
|
|
361
383
|
return SIZET2NUM((size_t)_module);
|
|
362
384
|
}
|
|
363
385
|
|
|
@@ -377,7 +399,7 @@ cuModuleUnload_without_gvl_cb(void *param)
|
|
|
377
399
|
static VALUE
|
|
378
400
|
rb_cuModuleUnload(VALUE self, VALUE hmod)
|
|
379
401
|
{
|
|
380
|
-
CUmodule _hmod = (CUmodule)
|
|
402
|
+
CUmodule _hmod = (CUmodule)cumo_cuda_handle_take(&modules, hmod, "CUmodule");
|
|
381
403
|
CUresult status;
|
|
382
404
|
|
|
383
405
|
struct cuModuleUnloadParam param = {_hmod};
|
|
@@ -399,6 +421,9 @@ Init_cumo_cuda_driver()
|
|
|
399
421
|
mDriver = rb_define_module_under(mCUDA, "Driver");
|
|
400
422
|
eDriverError = rb_define_class_under(mCUDA, "DriverError", rb_eStandardError);
|
|
401
423
|
|
|
424
|
+
cumo_cuda_handle_set_init(&link_states);
|
|
425
|
+
cumo_cuda_handle_set_init(&modules);
|
|
426
|
+
|
|
402
427
|
rb_define_singleton_method(mDriver, "cuCtxGetCurrent", rb_cuCtxGetCurrent, 0);
|
|
403
428
|
rb_define_singleton_method(mDriver, "cuLinkAddData", rb_cuLinkAddData, 4);
|
|
404
429
|
rb_define_singleton_method(mDriver, "cuLinkAddFile", rb_cuLinkAddFile, 3);
|
|
@@ -50,16 +50,21 @@ cumo_cuda_runtime_malloc(size_t size)
|
|
|
50
50
|
void
|
|
51
51
|
cumo_cuda_runtime_free(char *ptr)
|
|
52
52
|
{
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
53
|
+
// Always offer the pointer to the pool first, whatever memory_pool_enabled
|
|
54
|
+
// says now: MemoryPool.enable/disable is public, so the state can differ
|
|
55
|
+
// from what it was at allocation time. Handing a pooled chunk to cudaFree
|
|
56
|
+
// releases memory the pool still hands out, and fails outright for a chunk
|
|
57
|
+
// which is not at the head of its buffer.
|
|
58
|
+
try {
|
|
59
|
+
if (pool.Free(reinterpret_cast<intptr_t>(ptr))) {
|
|
60
|
+
return;
|
|
59
61
|
}
|
|
60
|
-
}
|
|
61
|
-
cumo_cuda_runtime_check_status(
|
|
62
|
+
} catch (const cumo::internal::CUDARuntimeError& e) {
|
|
63
|
+
cumo_cuda_runtime_check_status(e.status());
|
|
64
|
+
return;
|
|
62
65
|
}
|
|
66
|
+
// No pool owns it, so it came straight from cudaMallocManaged.
|
|
67
|
+
cumo_cuda_runtime_check_status(cudaFree((void*)ptr));
|
|
63
68
|
}
|
|
64
69
|
|
|
65
70
|
/*
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
#include "memory_pool_impl.hpp"
|
|
2
2
|
|
|
3
|
+
#include <cstdio>
|
|
4
|
+
|
|
3
5
|
#include <ruby.h>
|
|
4
6
|
|
|
5
7
|
namespace cumo {
|
|
@@ -25,9 +27,15 @@ Memory::~Memory() {
|
|
|
25
27
|
cudaError_t status = cudaFree(ptr_);
|
|
26
28
|
// CUDA driver may shut down before freeing memory inside memory pool.
|
|
27
29
|
// It is okay to simply ignore because CUDA driver automatically frees memory.
|
|
28
|
-
if (status
|
|
29
|
-
|
|
30
|
+
if (status == cudaSuccess || status == cudaErrorCudartUnloading) {
|
|
31
|
+
return;
|
|
30
32
|
}
|
|
33
|
+
// A destructor is implicitly noexcept, so throwing here would call
|
|
34
|
+
// std::terminate and abort the process. Report the failure instead:
|
|
35
|
+
// cudaFree only fails once the context is already unusable, and the
|
|
36
|
+
// next runtime call reports the same status to the caller anyway.
|
|
37
|
+
std::fprintf(stderr, "cumo: failed to free %zu bytes of device memory: %s\n",
|
|
38
|
+
size_, cudaGetErrorString(status));
|
|
31
39
|
}
|
|
32
40
|
}
|
|
33
41
|
|
|
@@ -63,14 +71,14 @@ void Merge(std::shared_ptr<Chunk>& self, std::shared_ptr<Chunk> remaining) {
|
|
|
63
71
|
|
|
64
72
|
void SingleDeviceMemoryPool::AppendToFreeList(size_t size, std::shared_ptr<Chunk>& chunk, cudaStream_t stream_ptr) {
|
|
65
73
|
assert(chunk != nullptr && !chunk->in_use());
|
|
66
|
-
|
|
74
|
+
size_t bin_index = GetBinIndex(size);
|
|
67
75
|
|
|
68
76
|
std::lock_guard<std::recursive_mutex> lock{mutex_};
|
|
69
77
|
|
|
70
78
|
Arena& arena = GetArena(stream_ptr);
|
|
71
79
|
ArenaIndexMap& arena_index_map = GetArenaIndexMap(stream_ptr);
|
|
72
|
-
|
|
73
|
-
|
|
80
|
+
size_t arena_index = std::lower_bound(arena_index_map.begin(), arena_index_map.end(), bin_index) - arena_index_map.begin();
|
|
81
|
+
size_t length = arena_index_map.size();
|
|
74
82
|
if (arena_index >= length || arena_index_map.at(arena_index) != bin_index) {
|
|
75
83
|
arena_index_map.insert(arena_index_map.begin() + arena_index, bin_index);
|
|
76
84
|
arena.insert(arena.begin() + arena_index, FreeList{});
|
|
@@ -81,7 +89,7 @@ void SingleDeviceMemoryPool::AppendToFreeList(size_t size, std::shared_ptr<Chunk
|
|
|
81
89
|
|
|
82
90
|
bool SingleDeviceMemoryPool::RemoveFromFreeList(size_t size, std::shared_ptr<Chunk>& chunk, cudaStream_t stream_ptr) {
|
|
83
91
|
assert(chunk != nullptr && !chunk->in_use());
|
|
84
|
-
|
|
92
|
+
size_t bin_index = GetBinIndex(size);
|
|
85
93
|
|
|
86
94
|
std::lock_guard<std::recursive_mutex> lock{mutex_};
|
|
87
95
|
|
|
@@ -90,20 +98,32 @@ bool SingleDeviceMemoryPool::RemoveFromFreeList(size_t size, std::shared_ptr<Chu
|
|
|
90
98
|
if (arena_index_map.size() == 0) {
|
|
91
99
|
return false;
|
|
92
100
|
}
|
|
93
|
-
|
|
94
|
-
if (
|
|
101
|
+
size_t arena_index = std::lower_bound(arena_index_map.begin(), arena_index_map.end(), bin_index) - arena_index_map.begin();
|
|
102
|
+
if (arena_index == arena_index_map.size()) {
|
|
95
103
|
// Bin does not exist for the given chunk size.
|
|
96
104
|
return false;
|
|
97
105
|
}
|
|
98
106
|
if (arena_index_map.at(arena_index) != bin_index) {
|
|
99
107
|
return false;
|
|
100
108
|
}
|
|
101
|
-
assert(arena.size() >
|
|
109
|
+
assert(arena.size() > arena_index);
|
|
102
110
|
FreeList& free_list = arena[arena_index];
|
|
103
111
|
return EraseFromFreeList(free_list, chunk);
|
|
104
112
|
}
|
|
105
113
|
|
|
106
114
|
intptr_t SingleDeviceMemoryPool::Malloc(size_t size, cudaStream_t stream_ptr) {
|
|
115
|
+
if (size == 0) {
|
|
116
|
+
// A zero-sized chunk would share its address with the chunk it was
|
|
117
|
+
// split from, and aliased addresses break the `in_use_` bookkeeping.
|
|
118
|
+
// cudaMalloc returns a null pointer for a zero-sized request as well.
|
|
119
|
+
return 0;
|
|
120
|
+
}
|
|
121
|
+
if (size > kMaxAllocationSize) {
|
|
122
|
+
// Rounding up would wrap around in size_t and the pool would then hand
|
|
123
|
+
// out a chunk far smaller than requested. Such a request can never be
|
|
124
|
+
// satisfied, so report it as out of memory instead.
|
|
125
|
+
throw OutOfMemoryError(size, GetTotalBytes());
|
|
126
|
+
}
|
|
107
127
|
size = GetRoundedSize(size);
|
|
108
128
|
std::shared_ptr<Chunk> chunk = nullptr;
|
|
109
129
|
|
|
@@ -112,9 +132,9 @@ intptr_t SingleDeviceMemoryPool::Malloc(size_t size, cudaStream_t stream_ptr) {
|
|
|
112
132
|
|
|
113
133
|
// find best-fit, or a smallest larger allocation
|
|
114
134
|
Arena& arena = GetArena(stream_ptr);
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
for (
|
|
135
|
+
size_t arena_index = GetArenaIndex(size, stream_ptr);
|
|
136
|
+
size_t arena_length = arena.size();
|
|
137
|
+
for (size_t i = arena_index; i < arena_length; ++i) {
|
|
118
138
|
FreeList& free_list = arena[i];
|
|
119
139
|
if (free_list.empty()) {
|
|
120
140
|
continue;
|
|
@@ -123,15 +143,22 @@ intptr_t SingleDeviceMemoryPool::Malloc(size_t size, cudaStream_t stream_ptr) {
|
|
|
123
143
|
// TODO(sonots): compact_index
|
|
124
144
|
break;
|
|
125
145
|
}
|
|
126
|
-
}
|
|
127
146
|
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
147
|
+
// Splitting rewrites the prev/next pointers of the neighbouring
|
|
148
|
+
// chunks, which other threads reach through the free lists, so it has
|
|
149
|
+
// to happen under the same lock as the search above.
|
|
150
|
+
if (chunk != nullptr) {
|
|
151
|
+
std::shared_ptr<Chunk> remaining = Split(chunk, size);
|
|
152
|
+
if (remaining != nullptr) {
|
|
153
|
+
AppendToFreeList(remaining->size(), remaining, stream_ptr);
|
|
154
|
+
}
|
|
132
155
|
}
|
|
133
|
-
}
|
|
134
|
-
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
if (chunk == nullptr) {
|
|
159
|
+
// cudaMalloc if a cache is not found. This stays outside the lock: it
|
|
160
|
+
// is slow, and a chunk of a fresh allocation has no neighbours for
|
|
161
|
+
// another thread to reach it through.
|
|
135
162
|
std::shared_ptr<Memory> mem = nullptr;
|
|
136
163
|
try {
|
|
137
164
|
mem = std::make_shared<Memory>(size);
|
|
@@ -166,19 +193,45 @@ intptr_t SingleDeviceMemoryPool::Malloc(size_t size, cudaStream_t stream_ptr) {
|
|
|
166
193
|
return chunk->ptr();
|
|
167
194
|
}
|
|
168
195
|
|
|
169
|
-
|
|
196
|
+
bool SingleDeviceMemoryPool::Free(intptr_t ptr) {
|
|
170
197
|
std::shared_ptr<Chunk> chunk = nullptr;
|
|
171
198
|
|
|
172
|
-
|
|
173
|
-
|
|
199
|
+
// The whole body runs under the lock. Walking prev/next and merging is a
|
|
200
|
+
// read-modify-write of the chunk graph, which is shared: a neighbour can
|
|
201
|
+
// be split by a concurrent Malloc while this reads its prev/next.
|
|
202
|
+
//
|
|
203
|
+
// The lost merge is the part that shows: two threads freeing neighbouring
|
|
204
|
+
// chunks each look for the other in the free list before the other has
|
|
205
|
+
// appended itself, so both merges fail and the two chunks stay split for
|
|
206
|
+
// good. Fragmentation accumulates from there.
|
|
207
|
+
//
|
|
208
|
+
// mutex_ is recursive, so the RemoveFromFreeList/AppendToFreeList calls
|
|
209
|
+
// below can take it again.
|
|
210
|
+
std::lock_guard<std::recursive_mutex> lock{mutex_};
|
|
174
211
|
|
|
175
|
-
|
|
176
|
-
//
|
|
177
|
-
|
|
212
|
+
{
|
|
213
|
+
// find rather than operator[], which would insert an empty entry for
|
|
214
|
+
// every pointer this pool does not own.
|
|
215
|
+
auto it = in_use_.find(ptr);
|
|
216
|
+
if (it == in_use_.end()) {
|
|
217
|
+
return false;
|
|
218
|
+
}
|
|
219
|
+
chunk = it->second;
|
|
220
|
+
in_use_.erase(it);
|
|
221
|
+
if (!chunk) {
|
|
222
|
+
return false;
|
|
223
|
+
}
|
|
178
224
|
chunk->set_in_use(false);
|
|
179
|
-
in_use_.erase(ptr);
|
|
180
225
|
}
|
|
181
226
|
|
|
227
|
+
// A chunk belongs to the arena of the stream it was allocated on. That is
|
|
228
|
+
// what makes reuse safe: the next allocation from that arena is queued on
|
|
229
|
+
// the same stream as the work that last used the chunk, so the two are
|
|
230
|
+
// ordered. Returning it to another stream's arena hands it to work with no
|
|
231
|
+
// such ordering. Malloc and Merge each state the invariant in an assert,
|
|
232
|
+
// but ruby.h defines NDEBUG, so neither ever runs.
|
|
233
|
+
const cudaStream_t stream_ptr = chunk->stream_ptr();
|
|
234
|
+
|
|
182
235
|
if (chunk->next() != nullptr && !chunk->next()->in_use()) {
|
|
183
236
|
if (RemoveFromFreeList(chunk->next()->size(), chunk->next(), stream_ptr)) {
|
|
184
237
|
Merge(chunk, chunk->next());
|
|
@@ -191,6 +244,7 @@ void SingleDeviceMemoryPool::Free(intptr_t ptr, cudaStream_t stream_ptr) {
|
|
|
191
244
|
}
|
|
192
245
|
}
|
|
193
246
|
AppendToFreeList(chunk->size(), chunk, stream_ptr);
|
|
247
|
+
return true;
|
|
194
248
|
}
|
|
195
249
|
|
|
196
250
|
void SingleDeviceMemoryPool::CompactIndex(cudaStream_t stream_ptr, bool free) {
|