cumo 0.5.6 → 0.5.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +25 -0
- data/Rakefile +11 -2
- data/ext/cumo/cuda/driver.c +17 -7
- data/ext/cumo/cuda/memory_pool.cpp +0 -1
- data/ext/cumo/cuda/memory_pool_impl.cpp +37 -11
- data/ext/cumo/cuda/memory_pool_impl.hpp +59 -30
- data/ext/cumo/cuda/memory_pool_impl_test.cpp +143 -0
- data/ext/cumo/cuda/nvrtc.c +10 -4
- data/ext/cumo/include/cumo/cuda/handle.h +85 -0
- data/ext/cumo/include/cumo/types/robj_macro.h +0 -7
- data/ext/cumo/include/cumo.h +1 -1
- data/ext/cumo/narray/data.c +13 -1
- data/ext/cumo/narray/gen/spec.rb +1 -0
- data/ext/cumo/narray/gen/tmpl/alloc_func.c +4 -0
- data/ext/cumo/narray/gen/tmpl/format.c +8 -0
- data/ext/cumo/narray/index.c +45 -15
- data/ext/cumo/narray/narray.c +21 -8
- data/lib/cumo/cuda/nvrtc_program.rb +3 -1
- data/lib/cumo/narray/extra.rb +42 -1
- data/test/cuda/driver_test.rb +27 -0
- data/test/cuda/nvrtc_test.rb +14 -0
- data/test/narray_test.rb +239 -0
- metadata +3 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 3ddc7b756681afdf9fccd45652c422d2f67accfd1cf238d1fd05f6827fb5029d
|
|
4
|
+
data.tar.gz: 47848503fd8a388b7dfee79d2da0e627032fd4d8c8717645da70d8f09770f599
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: f59ddc27c8f96a5eceec7fb4ae6665d1fae6dd20ef039a83563c0e0f5a76780fd2ab9b5ebbe06e116a3f01cd59c7493eee06dc6c1166a0bb82ba546c43331f03
|
|
7
|
+
data.tar.gz: 16543c06d74f79089799442abc30741ffb2a26556da953410023b19711fd25e15adb05548f9664c79bf8430a3be9c8f15c0813dccfae4b5e918ffc55a5fc68c0
|
data/CHANGELOG.md
CHANGED
|
@@ -1,3 +1,28 @@
|
|
|
1
|
+
# 0.5.7 (2026/08/13)
|
|
2
|
+
|
|
3
|
+
Breaking changes:
|
|
4
|
+
|
|
5
|
+
* `reshape` raises `ArgumentError` instead of `RangeError` for a dimension above `INT_MAX` (PR #183)
|
|
6
|
+
* `parse` accepts only numeric literals and `true`/`false`/`nil`, so `parse("3/4")` raises `ArgumentError` instead of returning `[[0]]` (PR #190)
|
|
7
|
+
|
|
8
|
+
Fixes:
|
|
9
|
+
|
|
10
|
+
* Fix segfaults from CUDA handles the process did not create, and from destroying one twice (PR #192)
|
|
11
|
+
* Fix process abort from freeing NArray data with the wrong allocator, including a raise out of GC for `Cumo::RObject` (PR #191)
|
|
12
|
+
* Fix arbitrary code execution in `parse`, which ran `eval` on every token (PR #190)
|
|
13
|
+
* Fix stack buffer overflow in `Cumo::RObject#format` with an element longer than 47 characters (PR #189)
|
|
14
|
+
* Fix a freed chunk returning to the arena of the wrong stream (PR #187)
|
|
15
|
+
* Fix `Malloc` searching the free list with the default stream's index (PR #186)
|
|
16
|
+
* Fix data race on the memory pool chunk graph and on the per-device pool table (PR #185)
|
|
17
|
+
* Fix a SIGFPE on fractional range steps and an out-of-bounds read on newaxis (PR #184)
|
|
18
|
+
* Fix a SIGFPE and a silently corrupt shape from unchecked `reshape` arguments (PR #183)
|
|
19
|
+
* Fix segfault when `store_binary` receives a non-String (PR #182)
|
|
20
|
+
* Fix missing range check on NArray indices, which segfaulted on a negative one (PR #181)
|
|
21
|
+
|
|
22
|
+
Changes:
|
|
23
|
+
|
|
24
|
+
* Compile the memory pool test harness in CI, and run it from `rake test` (PR #188)
|
|
25
|
+
|
|
1
26
|
# 0.5.6 (2026/08/09)
|
|
2
27
|
|
|
3
28
|
Fixes:
|
data/Rakefile
CHANGED
|
@@ -12,10 +12,19 @@ end
|
|
|
12
12
|
require "rake/extensiontask"
|
|
13
13
|
Rake::ExtensionTask.new("cumo")
|
|
14
14
|
|
|
15
|
-
|
|
16
|
-
|
|
15
|
+
desc 'Build the C++ memory pool test harness without running it (running it needs a GPU)'
|
|
16
|
+
task :build_ctest do
|
|
17
|
+
sh 'cd ext/cumo && ruby extconf.rb && make build-ctest'
|
|
17
18
|
end
|
|
18
19
|
|
|
20
|
+
task :ctest => :build_ctest do
|
|
21
|
+
sh 'cd ext/cumo && make run-ctest'
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
# Both need a GPU, so CI runs neither. Tie them together so the one place
|
|
25
|
+
# they do run does not skip half of them.
|
|
26
|
+
task :test => :ctest
|
|
27
|
+
|
|
19
28
|
task :docs do
|
|
20
29
|
dir = "ext/cumo"
|
|
21
30
|
srcs = %w[array.c data.c index.c math.c narray.c rand.c struct.c].map { |s| File.join(dir, "narray", s) }
|
data/ext/cumo/cuda/driver.c
CHANGED
|
@@ -3,12 +3,16 @@
|
|
|
3
3
|
#include <cuda.h>
|
|
4
4
|
#include <cuda_runtime.h>
|
|
5
5
|
#include "cumo/cuda/driver.h"
|
|
6
|
+
#include "cumo/cuda/handle.h"
|
|
6
7
|
|
|
7
8
|
VALUE cumo_cuda_eDriverError;
|
|
8
9
|
VALUE cumo_cuda_mDriver;
|
|
9
10
|
#define eDriverError cumo_cuda_eDriverError
|
|
10
11
|
#define mDriver cumo_cuda_mDriver
|
|
11
12
|
|
|
13
|
+
static cumo_cuda_handle_set_t link_states;
|
|
14
|
+
static cumo_cuda_handle_set_t modules;
|
|
15
|
+
|
|
12
16
|
static void
|
|
13
17
|
check_status(CUresult status)
|
|
14
18
|
{
|
|
@@ -100,13 +104,13 @@ cuLinkAddData_without_gvl_cb(void *param)
|
|
|
100
104
|
static VALUE
|
|
101
105
|
rb_cuLinkAddData(VALUE self, VALUE state, VALUE type, VALUE data, VALUE name)
|
|
102
106
|
{
|
|
103
|
-
CUlinkState _state = (CUlinkState)NUM2SIZET(state);
|
|
104
107
|
CUjitInputType _type = (CUjitInputType)NUM2INT(type);
|
|
105
108
|
// The image may be a cubin, so it is taken by length and allowed to hold
|
|
106
109
|
// NUL bytes; the name is a plain C string cuLinkAddData reports errors with.
|
|
107
110
|
void* _data = (void *)StringValuePtr(data);
|
|
108
111
|
size_t _size = RSTRING_LEN(data);
|
|
109
112
|
const char* _name = StringValueCStr(name);
|
|
113
|
+
CUlinkState _state = (CUlinkState)cumo_cuda_handle_get(&link_states, state, "CUlinkState");
|
|
110
114
|
CUresult status;
|
|
111
115
|
|
|
112
116
|
struct cuLinkAddDataParam param = {_state, _type, _data, _size, _name, 0, (CUjit_option*)0, (void**)0};
|
|
@@ -141,9 +145,9 @@ cuLinkAddFile_without_gvl_cb(void *param)
|
|
|
141
145
|
static VALUE
|
|
142
146
|
rb_cuLinkAddFile(VALUE self, VALUE state, VALUE type, VALUE path)
|
|
143
147
|
{
|
|
144
|
-
CUlinkState _state = (CUlinkState)NUM2SIZET(state);
|
|
145
148
|
CUjitInputType _type = (CUjitInputType)NUM2INT(type);
|
|
146
149
|
const char* _path = StringValueCStr(path);
|
|
150
|
+
CUlinkState _state = (CUlinkState)cumo_cuda_handle_get(&link_states, state, "CUlinkState");
|
|
147
151
|
CUresult status;
|
|
148
152
|
|
|
149
153
|
struct cuLinkAddFileParam param = {_state, _type, _path, 0, (CUjit_option*)0, (void **)0};
|
|
@@ -173,7 +177,7 @@ cuLinkComplete_without_gvl_cb(void *param)
|
|
|
173
177
|
static VALUE
|
|
174
178
|
rb_cuLinkComplete(VALUE self, VALUE state)
|
|
175
179
|
{
|
|
176
|
-
CUlinkState _state = (CUlinkState)
|
|
180
|
+
CUlinkState _state = (CUlinkState)cumo_cuda_handle_get(&link_states, state, "CUlinkState");
|
|
177
181
|
void* _cubinOut;
|
|
178
182
|
size_t _sizeOut;
|
|
179
183
|
CUresult status;
|
|
@@ -214,6 +218,7 @@ rb_cuLinkCreate(VALUE self)
|
|
|
214
218
|
//status = cuLinkCreate(0, (CUjit_option*)0, (void**)0, &state);
|
|
215
219
|
|
|
216
220
|
check_status(status);
|
|
221
|
+
cumo_cuda_handle_set_add(&link_states, (size_t)state);
|
|
217
222
|
return SIZET2NUM((size_t)state);
|
|
218
223
|
}
|
|
219
224
|
|
|
@@ -233,7 +238,7 @@ cuLinkDestroy_without_gvl_cb(void *param)
|
|
|
233
238
|
static VALUE
|
|
234
239
|
rb_cuLinkDestroy(VALUE self, VALUE state)
|
|
235
240
|
{
|
|
236
|
-
CUlinkState _state = (CUlinkState)
|
|
241
|
+
CUlinkState _state = (CUlinkState)cumo_cuda_handle_take(&link_states, state, "CUlinkState");
|
|
237
242
|
CUresult status;
|
|
238
243
|
|
|
239
244
|
struct cuLinkDestroyParam param = {_state};
|
|
@@ -263,8 +268,8 @@ static VALUE
|
|
|
263
268
|
rb_cuModuleGetFunction(VALUE self, VALUE hmod, VALUE name)
|
|
264
269
|
{
|
|
265
270
|
CUfunction _hfunc;
|
|
266
|
-
CUmodule _hmod = (CUmodule)NUM2SIZET(hmod);
|
|
267
271
|
const char* _name = StringValueCStr(name);
|
|
272
|
+
CUmodule _hmod = (CUmodule)cumo_cuda_handle_get(&modules, hmod, "CUmodule");
|
|
268
273
|
CUresult status;
|
|
269
274
|
|
|
270
275
|
struct cuModuleGetFunctionParam param = {&_hfunc, _hmod, _name};
|
|
@@ -297,8 +302,8 @@ rb_cuModuleGetGlobal(VALUE self, VALUE hmod, VALUE name)
|
|
|
297
302
|
{
|
|
298
303
|
CUdeviceptr _dptr;
|
|
299
304
|
size_t _bytes;
|
|
300
|
-
CUmodule _hmod = (CUmodule)NUM2SIZET(hmod);
|
|
301
305
|
const char* _name = StringValueCStr(name);
|
|
306
|
+
CUmodule _hmod = (CUmodule)cumo_cuda_handle_get(&modules, hmod, "CUmodule");
|
|
302
307
|
CUresult status;
|
|
303
308
|
VALUE ret;
|
|
304
309
|
|
|
@@ -342,6 +347,7 @@ rb_cuModuleLoad(VALUE self, VALUE fname)
|
|
|
342
347
|
|
|
343
348
|
RB_GC_GUARD(fname);
|
|
344
349
|
check_status(status);
|
|
350
|
+
cumo_cuda_handle_set_add(&modules, (size_t)_module);
|
|
345
351
|
return SIZET2NUM((size_t)_module);
|
|
346
352
|
}
|
|
347
353
|
|
|
@@ -373,6 +379,7 @@ rb_cuModuleLoadData(VALUE self, VALUE image)
|
|
|
373
379
|
|
|
374
380
|
RB_GC_GUARD(image);
|
|
375
381
|
check_status(status);
|
|
382
|
+
cumo_cuda_handle_set_add(&modules, (size_t)_module);
|
|
376
383
|
return SIZET2NUM((size_t)_module);
|
|
377
384
|
}
|
|
378
385
|
|
|
@@ -392,7 +399,7 @@ cuModuleUnload_without_gvl_cb(void *param)
|
|
|
392
399
|
static VALUE
|
|
393
400
|
rb_cuModuleUnload(VALUE self, VALUE hmod)
|
|
394
401
|
{
|
|
395
|
-
CUmodule _hmod = (CUmodule)
|
|
402
|
+
CUmodule _hmod = (CUmodule)cumo_cuda_handle_take(&modules, hmod, "CUmodule");
|
|
396
403
|
CUresult status;
|
|
397
404
|
|
|
398
405
|
struct cuModuleUnloadParam param = {_hmod};
|
|
@@ -414,6 +421,9 @@ Init_cumo_cuda_driver()
|
|
|
414
421
|
mDriver = rb_define_module_under(mCUDA, "Driver");
|
|
415
422
|
eDriverError = rb_define_class_under(mCUDA, "DriverError", rb_eStandardError);
|
|
416
423
|
|
|
424
|
+
cumo_cuda_handle_set_init(&link_states);
|
|
425
|
+
cumo_cuda_handle_set_init(&modules);
|
|
426
|
+
|
|
417
427
|
rb_define_singleton_method(mDriver, "cuCtxGetCurrent", rb_cuCtxGetCurrent, 0);
|
|
418
428
|
rb_define_singleton_method(mDriver, "cuLinkAddData", rb_cuLinkAddData, 4);
|
|
419
429
|
rb_define_singleton_method(mDriver, "cuLinkAddFile", rb_cuLinkAddFile, 3);
|
|
@@ -56,7 +56,6 @@ cumo_cuda_runtime_free(char *ptr)
|
|
|
56
56
|
// releases memory the pool still hands out, and fails outright for a chunk
|
|
57
57
|
// which is not at the head of its buffer.
|
|
58
58
|
try {
|
|
59
|
-
// TODO(sonots): Get current CUDA stream and pass it
|
|
60
59
|
if (pool.Free(reinterpret_cast<intptr_t>(ptr))) {
|
|
61
60
|
return;
|
|
62
61
|
}
|
|
@@ -132,7 +132,7 @@ intptr_t SingleDeviceMemoryPool::Malloc(size_t size, cudaStream_t stream_ptr) {
|
|
|
132
132
|
|
|
133
133
|
// find best-fit, or a smallest larger allocation
|
|
134
134
|
Arena& arena = GetArena(stream_ptr);
|
|
135
|
-
size_t arena_index = GetArenaIndex(size);
|
|
135
|
+
size_t arena_index = GetArenaIndex(size, stream_ptr);
|
|
136
136
|
size_t arena_length = arena.size();
|
|
137
137
|
for (size_t i = arena_index; i < arena_length; ++i) {
|
|
138
138
|
FreeList& free_list = arena[i];
|
|
@@ -143,15 +143,22 @@ intptr_t SingleDeviceMemoryPool::Malloc(size_t size, cudaStream_t stream_ptr) {
|
|
|
143
143
|
// TODO(sonots): compact_index
|
|
144
144
|
break;
|
|
145
145
|
}
|
|
146
|
-
}
|
|
147
146
|
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
147
|
+
// Splitting rewrites the prev/next pointers of the neighbouring
|
|
148
|
+
// chunks, which other threads reach through the free lists, so it has
|
|
149
|
+
// to happen under the same lock as the search above.
|
|
150
|
+
if (chunk != nullptr) {
|
|
151
|
+
std::shared_ptr<Chunk> remaining = Split(chunk, size);
|
|
152
|
+
if (remaining != nullptr) {
|
|
153
|
+
AppendToFreeList(remaining->size(), remaining, stream_ptr);
|
|
154
|
+
}
|
|
152
155
|
}
|
|
153
|
-
}
|
|
154
|
-
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
if (chunk == nullptr) {
|
|
159
|
+
// cudaMalloc if a cache is not found. This stays outside the lock: it
|
|
160
|
+
// is slow, and a chunk of a fresh allocation has no neighbours for
|
|
161
|
+
// another thread to reach it through.
|
|
155
162
|
std::shared_ptr<Memory> mem = nullptr;
|
|
156
163
|
try {
|
|
157
164
|
mem = std::make_shared<Memory>(size);
|
|
@@ -186,12 +193,23 @@ intptr_t SingleDeviceMemoryPool::Malloc(size_t size, cudaStream_t stream_ptr) {
|
|
|
186
193
|
return chunk->ptr();
|
|
187
194
|
}
|
|
188
195
|
|
|
189
|
-
bool SingleDeviceMemoryPool::Free(intptr_t ptr
|
|
196
|
+
bool SingleDeviceMemoryPool::Free(intptr_t ptr) {
|
|
190
197
|
std::shared_ptr<Chunk> chunk = nullptr;
|
|
191
198
|
|
|
192
|
-
|
|
193
|
-
|
|
199
|
+
// The whole body runs under the lock. Walking prev/next and merging is a
|
|
200
|
+
// read-modify-write of the chunk graph, which is shared: a neighbour can
|
|
201
|
+
// be split by a concurrent Malloc while this reads its prev/next.
|
|
202
|
+
//
|
|
203
|
+
// The lost merge is the part that shows: two threads freeing neighbouring
|
|
204
|
+
// chunks each look for the other in the free list before the other has
|
|
205
|
+
// appended itself, so both merges fail and the two chunks stay split for
|
|
206
|
+
// good. Fragmentation accumulates from there.
|
|
207
|
+
//
|
|
208
|
+
// mutex_ is recursive, so the RemoveFromFreeList/AppendToFreeList calls
|
|
209
|
+
// below can take it again.
|
|
210
|
+
std::lock_guard<std::recursive_mutex> lock{mutex_};
|
|
194
211
|
|
|
212
|
+
{
|
|
195
213
|
// find rather than operator[], which would insert an empty entry for
|
|
196
214
|
// every pointer this pool does not own.
|
|
197
215
|
auto it = in_use_.find(ptr);
|
|
@@ -206,6 +224,14 @@ bool SingleDeviceMemoryPool::Free(intptr_t ptr, cudaStream_t stream_ptr) {
|
|
|
206
224
|
chunk->set_in_use(false);
|
|
207
225
|
}
|
|
208
226
|
|
|
227
|
+
// A chunk belongs to the arena of the stream it was allocated on. That is
|
|
228
|
+
// what makes reuse safe: the next allocation from that arena is queued on
|
|
229
|
+
// the same stream as the work that last used the chunk, so the two are
|
|
230
|
+
// ordered. Returning it to another stream's arena hands it to work with no
|
|
231
|
+
// such ordering. Malloc and Merge each state the invariant in an assert,
|
|
232
|
+
// but ruby.h defines NDEBUG, so neither ever runs.
|
|
233
|
+
const cudaStream_t stream_ptr = chunk->stream_ptr();
|
|
234
|
+
|
|
209
235
|
if (chunk->next() != nullptr && !chunk->next()->in_use()) {
|
|
210
236
|
if (RemoveFromFreeList(chunk->next()->size(), chunk->next(), stream_ptr)) {
|
|
211
237
|
Merge(chunk, chunk->next());
|
|
@@ -174,7 +174,10 @@ public:
|
|
|
174
174
|
|
|
175
175
|
// Returns false if this pool did not allocate the pointer, in which case
|
|
176
176
|
// nothing was freed.
|
|
177
|
-
|
|
177
|
+
//
|
|
178
|
+
// There is no stream parameter: the chunk records the stream it was
|
|
179
|
+
// allocated on, and that is the only arena it may be returned to.
|
|
180
|
+
bool Free(intptr_t ptr);
|
|
178
181
|
|
|
179
182
|
// Free all **non-split** chunks in all arenas
|
|
180
183
|
void FreeAllBlocks();
|
|
@@ -216,7 +219,12 @@ public:
|
|
|
216
219
|
return (size - 1) / kRoundSize;
|
|
217
220
|
}
|
|
218
221
|
|
|
219
|
-
|
|
222
|
+
// Returns where a chunk of `size` belongs in the arena of `stream_ptr`.
|
|
223
|
+
//
|
|
224
|
+
// The stream is not optional: an index taken from one stream's index map
|
|
225
|
+
// means nothing in another stream's arena, and defaulting it silently
|
|
226
|
+
// started the free list search at the wrong bin.
|
|
227
|
+
size_t GetArenaIndex(size_t size, cudaStream_t stream_ptr) {
|
|
220
228
|
size_t bin_index = GetBinIndex(size);
|
|
221
229
|
ArenaIndexMap& arena_index_map = GetArenaIndexMap(stream_ptr);
|
|
222
230
|
return std::lower_bound(arena_index_map.begin(), arena_index_map.end(), bin_index) - arena_index_map.begin();
|
|
@@ -267,14 +275,14 @@ public:
|
|
|
267
275
|
return true;
|
|
268
276
|
}
|
|
269
277
|
|
|
270
|
-
void AppendToFreeList(size_t size, std::shared_ptr<Chunk>& chunk, cudaStream_t stream_ptr
|
|
278
|
+
void AppendToFreeList(size_t size, std::shared_ptr<Chunk>& chunk, cudaStream_t stream_ptr);
|
|
271
279
|
|
|
272
280
|
// Removes the chunk from the free list.
|
|
273
281
|
//
|
|
274
282
|
// @return true if the chunk can successfully be removed from
|
|
275
283
|
// the free list. false` otherwise (e.g., the chunk could not
|
|
276
284
|
// be found in the free list as the chunk is allocated.)
|
|
277
|
-
bool RemoveFromFreeList(size_t size, std::shared_ptr<Chunk>& chunk, cudaStream_t stream_ptr
|
|
285
|
+
bool RemoveFromFreeList(size_t size, std::shared_ptr<Chunk>& chunk, cudaStream_t stream_ptr);
|
|
278
286
|
|
|
279
287
|
void CompactIndex(cudaStream_t stream_ptr, bool free);
|
|
280
288
|
};
|
|
@@ -304,6 +312,20 @@ private:
|
|
|
304
312
|
}
|
|
305
313
|
|
|
306
314
|
std::unordered_map<int, SingleDeviceMemoryPool> pools_;
|
|
315
|
+
std::mutex pools_mutex_;
|
|
316
|
+
|
|
317
|
+
// Returns the pool of the current device, creating it on first use.
|
|
318
|
+
//
|
|
319
|
+
// The lock only has to cover the lookup: pools_ is never erased from
|
|
320
|
+
// before destruction, and unordered_map keeps references to its elements
|
|
321
|
+
// valid across a rehash, so the returned reference outlives the lock.
|
|
322
|
+
// Without it, the insert operator[] performs on first use could rehash the
|
|
323
|
+
// map underneath another thread walking it.
|
|
324
|
+
SingleDeviceMemoryPool& GetPool() {
|
|
325
|
+
int id = device_id(); // a CUDA call, so keep it out of the lock
|
|
326
|
+
std::lock_guard<std::mutex> lock{pools_mutex_};
|
|
327
|
+
return pools_[id]; // find or create
|
|
328
|
+
}
|
|
307
329
|
|
|
308
330
|
public:
|
|
309
331
|
MemoryPool() {}
|
|
@@ -318,35 +340,48 @@ public:
|
|
|
318
340
|
// Returns:
|
|
319
341
|
// intptr_t: Pointer address to the allocated buffer.
|
|
320
342
|
intptr_t Malloc(size_t size, cudaStream_t stream_ptr = 0) {
|
|
321
|
-
|
|
322
|
-
return mp.Malloc(size, stream_ptr);
|
|
343
|
+
return GetPool().Malloc(size, stream_ptr);
|
|
323
344
|
}
|
|
324
345
|
|
|
325
346
|
// Frees the memory, to the pool
|
|
326
347
|
//
|
|
327
348
|
// Args:
|
|
328
349
|
// ptr (intptr_t): Pointer of the memory buffer
|
|
329
|
-
// stream_ptr (cudaStream_t): Return the memory to the arena of given stream
|
|
330
350
|
// Returns:
|
|
331
351
|
// bool: false if no pool allocated the pointer, in which case nothing
|
|
332
352
|
// was freed and the caller has to free it by its own means.
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
353
|
+
//
|
|
354
|
+
// The stream is not a parameter: the chunk records the stream it was
|
|
355
|
+
// allocated on, and that is the only arena it may be returned to.
|
|
356
|
+
bool Free(intptr_t ptr) {
|
|
337
357
|
int current_device_id = device_id();
|
|
338
|
-
|
|
339
|
-
|
|
358
|
+
|
|
359
|
+
// Take the pointers out under the lock and release it before freeing:
|
|
360
|
+
// an element of pools_ stays put once inserted, so the pointers remain
|
|
361
|
+
// valid, and this keeps a Free of one device's pool from blocking a
|
|
362
|
+
// Malloc on another. `others` does not allocate in the usual
|
|
363
|
+
// single-device case, where it stays empty.
|
|
364
|
+
SingleDeviceMemoryPool* current = nullptr;
|
|
365
|
+
std::vector<SingleDeviceMemoryPool*> others;
|
|
366
|
+
{
|
|
367
|
+
std::lock_guard<std::mutex> lock{pools_mutex_};
|
|
368
|
+
for (auto& entry : pools_) {
|
|
369
|
+
if (entry.first == current_device_id) {
|
|
370
|
+
current = &entry.second;
|
|
371
|
+
} else {
|
|
372
|
+
others.emplace_back(&entry.second);
|
|
373
|
+
}
|
|
374
|
+
}
|
|
375
|
+
}
|
|
376
|
+
|
|
377
|
+
if (current != nullptr && current->Free(ptr)) {
|
|
340
378
|
return true;
|
|
341
379
|
}
|
|
342
380
|
// The current device may have been switched since the allocation.
|
|
343
381
|
// cudaMallocManaged hands out addresses which are unique across the
|
|
344
382
|
// host and every device, so the pointer identifies its pool on its own.
|
|
345
|
-
for (
|
|
346
|
-
if (
|
|
347
|
-
continue;
|
|
348
|
-
}
|
|
349
|
-
if (entry.second.Free(ptr, stream_ptr)) {
|
|
383
|
+
for (SingleDeviceMemoryPool* mp : others) {
|
|
384
|
+
if (mp->Free(ptr)) {
|
|
350
385
|
return true;
|
|
351
386
|
}
|
|
352
387
|
}
|
|
@@ -355,8 +390,7 @@ public:
|
|
|
355
390
|
|
|
356
391
|
// Free all **non-split** chunks in all arenas
|
|
357
392
|
void FreeAllBlocks() {
|
|
358
|
-
|
|
359
|
-
return mp.FreeAllBlocks();
|
|
393
|
+
return GetPool().FreeAllBlocks();
|
|
360
394
|
}
|
|
361
395
|
|
|
362
396
|
// Free all **non-split** chunks in specified arena
|
|
@@ -364,8 +398,7 @@ public:
|
|
|
364
398
|
// Args:
|
|
365
399
|
// stream_ptr (cudaStream_t): Release free blocks in the arena of given stream
|
|
366
400
|
void FreeAllBlocks(cudaStream_t stream_ptr) {
|
|
367
|
-
|
|
368
|
-
return mp.FreeAllBlocks(stream_ptr);
|
|
401
|
+
return GetPool().FreeAllBlocks(stream_ptr);
|
|
369
402
|
}
|
|
370
403
|
|
|
371
404
|
// Count the total number of free blocks.
|
|
@@ -373,8 +406,7 @@ public:
|
|
|
373
406
|
// Returns:
|
|
374
407
|
// size_t: The total number of free blocks.
|
|
375
408
|
size_t GetNumFreeBlocks() {
|
|
376
|
-
|
|
377
|
-
return mp.GetNumFreeBlocks();
|
|
409
|
+
return GetPool().GetNumFreeBlocks();
|
|
378
410
|
}
|
|
379
411
|
|
|
380
412
|
// Get the total number of bytes used.
|
|
@@ -382,8 +414,7 @@ public:
|
|
|
382
414
|
// Returns:
|
|
383
415
|
// size_t: The total number of bytes used.
|
|
384
416
|
size_t GetUsedBytes() {
|
|
385
|
-
|
|
386
|
-
return mp.GetUsedBytes();
|
|
417
|
+
return GetPool().GetUsedBytes();
|
|
387
418
|
}
|
|
388
419
|
|
|
389
420
|
// Get the total number of bytes acquired but not used in the pool.
|
|
@@ -391,8 +422,7 @@ public:
|
|
|
391
422
|
// Returns:
|
|
392
423
|
// size_t: The total number of bytes acquired but not used in the pool.
|
|
393
424
|
size_t GetFreeBytes() {
|
|
394
|
-
|
|
395
|
-
return mp.GetFreeBytes();
|
|
425
|
+
return GetPool().GetFreeBytes();
|
|
396
426
|
}
|
|
397
427
|
|
|
398
428
|
// Get the total number of bytes acquired in the pool.
|
|
@@ -400,8 +430,7 @@ public:
|
|
|
400
430
|
// Returns:
|
|
401
431
|
// size_t: The total number of bytes acquired in the pool.
|
|
402
432
|
size_t GetTotalBytes() {
|
|
403
|
-
|
|
404
|
-
return mp.GetTotalBytes();
|
|
433
|
+
return GetPool().GetTotalBytes();
|
|
405
434
|
}
|
|
406
435
|
};
|
|
407
436
|
|
|
@@ -1,8 +1,12 @@
|
|
|
1
1
|
#include "memory_pool_impl.hpp"
|
|
2
2
|
|
|
3
|
+
#include <atomic>
|
|
3
4
|
#include <cassert>
|
|
5
|
+
#include <cstring>
|
|
4
6
|
#include <memory>
|
|
5
7
|
#include <iostream>
|
|
8
|
+
#include <thread>
|
|
9
|
+
#include <vector>
|
|
6
10
|
|
|
7
11
|
// TODO(sonots): Use googletest?
|
|
8
12
|
// TODO(sonots): Provide clean way to build this test outside extconf.rb
|
|
@@ -135,6 +139,8 @@ public:
|
|
|
135
139
|
TearDown(); SetUp(); TestGetRoundedSize();
|
|
136
140
|
TearDown(); SetUp(); TestGetBinIndex();
|
|
137
141
|
TearDown(); SetUp(); TestGetArenaIndexWithHugeSize();
|
|
142
|
+
TearDown(); SetUp(); TestMallocOnAnotherStream();
|
|
143
|
+
TearDown(); SetUp(); TestFreeReturnsChunkToItsOwnStream();
|
|
138
144
|
TearDown(); SetUp(); TestAppendToFreeList();
|
|
139
145
|
TearDown(); SetUp(); TestRemoveFromFreeList();
|
|
140
146
|
TearDown(); SetUp(); TestMalloc();
|
|
@@ -183,6 +189,53 @@ public:
|
|
|
183
189
|
assert(pool_->GetArenaIndex(kMaxAllocationSize, stream_ptr_) == 1);
|
|
184
190
|
}
|
|
185
191
|
|
|
192
|
+
// Malloc used to take the starting arena index from stream 0's index map
|
|
193
|
+
// and then index the requested stream's arena with it. The two index maps
|
|
194
|
+
// are unrelated, so the search could start below the requested size and
|
|
195
|
+
// hand out a chunk too small for it.
|
|
196
|
+
void TestMallocOnAnotherStream() {
|
|
197
|
+
cudaStream_t other = reinterpret_cast<cudaStream_t>(1);
|
|
198
|
+
|
|
199
|
+
auto small_mem = std::make_shared<Memory>(kRoundSize);
|
|
200
|
+
auto small = std::make_shared<Chunk>(small_mem, 0, small_mem->size(), other);
|
|
201
|
+
pool_->AppendToFreeList(small->size(), small, other);
|
|
202
|
+
|
|
203
|
+
auto big_mem = std::make_shared<Memory>(kRoundSize * 4);
|
|
204
|
+
auto big = std::make_shared<Chunk>(big_mem, 0, big_mem->size(), other);
|
|
205
|
+
pool_->AppendToFreeList(big->size(), big, other);
|
|
206
|
+
|
|
207
|
+
// stream 0 has no bins at all here, so its index is 0 for any size.
|
|
208
|
+
assert(pool_->GetArenaIndex(kRoundSize * 4, stream_ptr_) == 0);
|
|
209
|
+
assert(pool_->GetArenaIndex(kRoundSize * 4, other) == 1);
|
|
210
|
+
|
|
211
|
+
auto p = pool_->Malloc(kRoundSize * 4, other);
|
|
212
|
+
assert(p == big->ptr());
|
|
213
|
+
assert(pool_->GetUsedBytes() == kRoundSize * 4);
|
|
214
|
+
// The small chunk is untouched, and no chunk was split to a size its
|
|
215
|
+
// buffer cannot back -- which used to underflow GetFreeBytes.
|
|
216
|
+
assert(pool_->GetFreeBytes() == kRoundSize);
|
|
217
|
+
assert(small->size() == kRoundSize);
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
// Free used to append the chunk to the arena of the stream it was passed,
|
|
221
|
+
// and cumo_cuda_runtime_free passes none, so anything allocated on another
|
|
222
|
+
// stream was returned to stream 0's arena -- where a later Malloc on
|
|
223
|
+
// stream 0 would reuse memory it never ordered against.
|
|
224
|
+
void TestFreeReturnsChunkToItsOwnStream() {
|
|
225
|
+
cudaStream_t other = reinterpret_cast<cudaStream_t>(1);
|
|
226
|
+
|
|
227
|
+
auto p = pool_->Malloc(kRoundSize, other);
|
|
228
|
+
pool_->Free(p);
|
|
229
|
+
assert(pool_->GetNumFreeBlocks() == 1);
|
|
230
|
+
assert(!pool_->HasArena(stream_ptr_));
|
|
231
|
+
|
|
232
|
+
// The owning stream gets its own chunk back rather than cudaMalloc'ing
|
|
233
|
+
// a second one, which is the same thing seen from the other side.
|
|
234
|
+
auto q = pool_->Malloc(kRoundSize, other);
|
|
235
|
+
assert(q == p);
|
|
236
|
+
assert(pool_->GetTotalBytes() == kRoundSize);
|
|
237
|
+
}
|
|
238
|
+
|
|
186
239
|
void TestAppendToFreeList() {
|
|
187
240
|
Arena& arena = pool_->GetArena(stream_ptr_);
|
|
188
241
|
ArenaIndexMap& arena_index_map = pool_->GetArenaIndexMap(stream_ptr_);
|
|
@@ -620,6 +673,95 @@ public:
|
|
|
620
673
|
}
|
|
621
674
|
};
|
|
622
675
|
|
|
676
|
+
// The pool is reachable from several threads at once: cumo.c declares the
|
|
677
|
+
// extension Ractor-safe, and a Ractor does not hold the GVL that serializes
|
|
678
|
+
// ordinary Ruby threads.
|
|
679
|
+
class TestConcurrency {
|
|
680
|
+
private:
|
|
681
|
+
static const int kThreads = 8;
|
|
682
|
+
static const int kRounds = 400;
|
|
683
|
+
static const int kPerThread = 8;
|
|
684
|
+
|
|
685
|
+
public:
|
|
686
|
+
void Run() {
|
|
687
|
+
TestConcurrentSplitAndMerge();
|
|
688
|
+
TestConcurrentPools();
|
|
689
|
+
}
|
|
690
|
+
|
|
691
|
+
// Chunks are only neighbours when they were Split from a common
|
|
692
|
+
// allocation, so prime the pool with one large free chunk: every Malloc
|
|
693
|
+
// below splits off it, and each thread then frees chunks sitting next to
|
|
694
|
+
// chunks other threads still hold. That is the state Free walks over.
|
|
695
|
+
void TestConcurrentSplitAndMerge() {
|
|
696
|
+
SingleDeviceMemoryPool pool;
|
|
697
|
+
const size_t total = size_t{kRoundSize} * kThreads * kPerThread * 2;
|
|
698
|
+
intptr_t big = pool.Malloc(total);
|
|
699
|
+
pool.Free(big);
|
|
700
|
+
assert(pool.GetNumFreeBlocks() == 1);
|
|
701
|
+
|
|
702
|
+
std::atomic<int> corrupted{0};
|
|
703
|
+
std::vector<std::thread> threads;
|
|
704
|
+
for (int t = 0; t < kThreads; ++t) {
|
|
705
|
+
threads.emplace_back([&pool, &corrupted, t] {
|
|
706
|
+
std::vector<intptr_t> ptrs(kPerThread);
|
|
707
|
+
for (int round = 0; round < kRounds; ++round) {
|
|
708
|
+
for (int k = 0; k < kPerThread; ++k) {
|
|
709
|
+
ptrs[k] = pool.Malloc(kRoundSize);
|
|
710
|
+
// Stamp the chunk with a value no other thread uses.
|
|
711
|
+
// Merging across a chunk which is still in use hands
|
|
712
|
+
// the same bytes out twice, and the second holder's
|
|
713
|
+
// stamp overwrites the first one's.
|
|
714
|
+
std::memset(reinterpret_cast<void*>(ptrs[k]),
|
|
715
|
+
Stamp(t, k), kRoundSize);
|
|
716
|
+
}
|
|
717
|
+
for (int k = 0; k < kPerThread; ++k) {
|
|
718
|
+
auto* p = reinterpret_cast<unsigned char*>(ptrs[k]);
|
|
719
|
+
for (size_t i = 0; i < kRoundSize; ++i) {
|
|
720
|
+
if (p[i] != Stamp(t, k)) {
|
|
721
|
+
++corrupted;
|
|
722
|
+
break;
|
|
723
|
+
}
|
|
724
|
+
}
|
|
725
|
+
pool.Free(ptrs[k]);
|
|
726
|
+
}
|
|
727
|
+
}
|
|
728
|
+
});
|
|
729
|
+
}
|
|
730
|
+
for (auto& thread : threads) thread.join();
|
|
731
|
+
|
|
732
|
+
assert(corrupted == 0);
|
|
733
|
+
assert(pool.GetUsedBytes() == 0);
|
|
734
|
+
// Free merges a chunk with any free neighbour, so once every chunk is
|
|
735
|
+
// freed they all collapse back into the single block they came from.
|
|
736
|
+
// A lost merge leaves the pool fragmented forever.
|
|
737
|
+
assert(pool.GetNumFreeBlocks() == 1);
|
|
738
|
+
assert(pool.GetFreeBytes() == total);
|
|
739
|
+
}
|
|
740
|
+
|
|
741
|
+
// MemoryPool::pools_ is inserted into on first use of a device.
|
|
742
|
+
void TestConcurrentPools() {
|
|
743
|
+
MemoryPool pool;
|
|
744
|
+
std::vector<std::thread> threads;
|
|
745
|
+
for (int t = 0; t < kThreads; ++t) {
|
|
746
|
+
threads.emplace_back([&pool] {
|
|
747
|
+
for (int round = 0; round < kRounds; ++round) {
|
|
748
|
+
intptr_t p = pool.Malloc(kRoundSize);
|
|
749
|
+
assert(p != 0);
|
|
750
|
+
pool.Free(p);
|
|
751
|
+
}
|
|
752
|
+
});
|
|
753
|
+
}
|
|
754
|
+
for (auto& thread : threads) thread.join();
|
|
755
|
+
|
|
756
|
+
assert(pool.GetUsedBytes() == 0);
|
|
757
|
+
}
|
|
758
|
+
|
|
759
|
+
private:
|
|
760
|
+
static unsigned char Stamp(int t, int k) {
|
|
761
|
+
return static_cast<unsigned char>(t * kPerThread + k + 1);
|
|
762
|
+
}
|
|
763
|
+
};
|
|
764
|
+
|
|
623
765
|
// Resets the CUDA device, so it has to run after every other test.
|
|
624
766
|
class TestMemoryDestructor {
|
|
625
767
|
public:
|
|
@@ -650,6 +792,7 @@ int main() {
|
|
|
650
792
|
cumo::internal::TestChunk{}.Run();
|
|
651
793
|
cumo::internal::TestSingleDeviceMemoryPool{}.Run();
|
|
652
794
|
cumo::internal::TestMemoryPool{}.Run();
|
|
795
|
+
cumo::internal::TestConcurrency{}.Run();
|
|
653
796
|
cumo::internal::TestMemoryDestructor{}.Run();
|
|
654
797
|
return 0;
|
|
655
798
|
}
|