cumo 0.5.5 → 0.5.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +27 -0
- data/ext/cumo/cuda/driver.c +23 -8
- data/ext/cumo/cuda/memory_pool.cpp +14 -8
- data/ext/cumo/cuda/memory_pool_impl.cpp +45 -17
- data/ext/cumo/cuda/memory_pool_impl.hpp +48 -8
- data/ext/cumo/cuda/memory_pool_impl_test.cpp +102 -1
- data/ext/cumo/cuda/nvrtc.c +91 -17
- data/ext/cumo/depend.erb +1 -1
- data/ext/cumo/extconf.rb +1 -0
- data/ext/cumo/include/cumo/types/float_macro.h +6 -1
- data/ext/cumo/include/cumo/types/robj_macro.h +1 -1
- data/ext/cumo/include/cumo/types/robject.h +1 -1
- data/ext/cumo/include/cumo.h +1 -1
- data/ext/cumo/narray/data.c +14 -10
- data/ext/cumo/narray/gen/tmpl/alloc_func.c +3 -5
- data/ext/cumo/narray/gen/tmpl/qsort.c +2 -5
- data/ext/cumo/narray/gen/tmpl/store_array.c +26 -8
- data/ext/cumo/narray/index.c +46 -27
- data/ext/cumo/narray/narray.c +7 -5
- data/test/cuda/driver_test.rb +65 -0
- data/test/cuda/memory_pool_test.rb +10 -0
- data/test/cuda/nvrtc_test.rb +54 -0
- data/test/narray_test.rb +255 -0
- metadata +4 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 324cdfec0cf163d40b3b3b635fa69ecf516aa12e15a0bf614cbc34d3fe61a4c7
|
|
4
|
+
data.tar.gz: 3d0f6c8b66a9b51eca9fe6912c1a4e7c3534a028ed4c8d740467ae8d6e215824
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 9c82a985c471260943c4716630d6db803706ac50cccd5e1fc8a0a2bb02d80172ec38743f5c5f286467e97814843b8fbe1d6ec1f98e75081364fd9a5c0a4da832
|
|
7
|
+
data.tar.gz: d3ed8c3e8844d51676c4011139faa509de8cdb39ef325d84b4b8b41f6725fb63b9893fae8dbced5e546810ff4eb570f60a347ef248323f805bffa9fde60faab1
|
data/CHANGELOG.md
CHANGED
|
@@ -1,3 +1,30 @@
|
|
|
1
|
+
# 0.5.6 (2026/08/09)
|
|
2
|
+
|
|
3
|
+
Fixes:
|
|
4
|
+
|
|
5
|
+
* Fix out-of-tree build failing to create the narray/types object directory
|
|
6
|
+
* Fix segfaults from unchecked String arguments in Driver (#179)
|
|
7
|
+
* Fix out-of-bounds read of include_names in nvrtcCreateProgram (#178)
|
|
8
|
+
* Fix store into a strided view reading a released staging buffer (#177)
|
|
9
|
+
* Fix pinned host memory leak when indexing by an array (#176)
|
|
10
|
+
* Fix segfault in Module#get_global_var reading device memory as a host pointer (#174)
|
|
11
|
+
* Fix freeing pooled memory with cudaFree after disabling the pool (#173)
|
|
12
|
+
* Fix run-ctest failing to start the test binary (#172)
|
|
13
|
+
* Fix process abort when freeing device memory fails (#171)
|
|
14
|
+
* Fix memory pool handing out a tiny chunk for a huge allocation (#170)
|
|
15
|
+
* Fix out-of-bounds read of a shorter sub-narray in store (#169)
|
|
16
|
+
* Fix out-of-bounds device write in store when a sub-narray is too long (#168)
|
|
17
|
+
* Backport: fix na_flatten_dim for multi-dimensional empty arrays (#167)
|
|
18
|
+
* Backport: make qsort loop condition explicit to prevent incorrect optimization (#166)
|
|
19
|
+
* Backport: free previously allocated shape in cumo_na_alloc_shape (#165)
|
|
20
|
+
* Backport: free shape on deallocation regardless of size (#164)
|
|
21
|
+
* Backport: convert max to double in rand method of Cumo::RObject (#163)
|
|
22
|
+
* Backport: use an inline function to prevent double use of macro argument (#162)
|
|
23
|
+
* Fix out-of-bounds stridx access in at() when :new is given (#161)
|
|
24
|
+
* Backport: prevent out-of-bounds access to stridx when orig_dim exceeds ndim (#160)
|
|
25
|
+
* Backport: prevent negative array index in na_get_strides_nadata when ndim is zero (#159)
|
|
26
|
+
* Backport: use correct array index for mod result in divmod macro for Cumo::RObject (#158)
|
|
27
|
+
|
|
1
28
|
# 0.5.5 (2026/07/11)
|
|
2
29
|
|
|
3
30
|
Fixes:
|
data/ext/cumo/cuda/driver.c
CHANGED
|
@@ -102,15 +102,19 @@ rb_cuLinkAddData(VALUE self, VALUE state, VALUE type, VALUE data, VALUE name)
|
|
|
102
102
|
{
|
|
103
103
|
CUlinkState _state = (CUlinkState)NUM2SIZET(state);
|
|
104
104
|
CUjitInputType _type = (CUjitInputType)NUM2INT(type);
|
|
105
|
-
|
|
105
|
+
// The image may be a cubin, so it is taken by length and allowed to hold
|
|
106
|
+
// NUL bytes; the name is a plain C string cuLinkAddData reports errors with.
|
|
107
|
+
void* _data = (void *)StringValuePtr(data);
|
|
106
108
|
size_t _size = RSTRING_LEN(data);
|
|
107
|
-
const char* _name =
|
|
109
|
+
const char* _name = StringValueCStr(name);
|
|
108
110
|
CUresult status;
|
|
109
111
|
|
|
110
112
|
struct cuLinkAddDataParam param = {_state, _type, _data, _size, _name, 0, (CUjit_option*)0, (void**)0};
|
|
111
113
|
status = (CUresult)rb_thread_call_without_gvl(cuLinkAddData_without_gvl_cb, ¶m, NULL, NULL);
|
|
112
114
|
//status = cuLinkAddData(_state, _type, _data, _size, _name, 0, (CUjit_option*)0, (void**)0);
|
|
113
115
|
|
|
116
|
+
RB_GC_GUARD(data);
|
|
117
|
+
RB_GC_GUARD(name);
|
|
114
118
|
check_status(status);
|
|
115
119
|
return Qnil;
|
|
116
120
|
}
|
|
@@ -139,13 +143,14 @@ rb_cuLinkAddFile(VALUE self, VALUE state, VALUE type, VALUE path)
|
|
|
139
143
|
{
|
|
140
144
|
CUlinkState _state = (CUlinkState)NUM2SIZET(state);
|
|
141
145
|
CUjitInputType _type = (CUjitInputType)NUM2INT(type);
|
|
142
|
-
const char* _path =
|
|
146
|
+
const char* _path = StringValueCStr(path);
|
|
143
147
|
CUresult status;
|
|
144
148
|
|
|
145
149
|
struct cuLinkAddFileParam param = {_state, _type, _path, 0, (CUjit_option*)0, (void **)0};
|
|
146
150
|
status = (CUresult)rb_thread_call_without_gvl(cuLinkAddFile_without_gvl_cb, ¶m, NULL, NULL);
|
|
147
151
|
//status = cuLinkAddFile(_state, _type, _path, 0, (CUjit_option*)0, (void **)0);
|
|
148
152
|
|
|
153
|
+
RB_GC_GUARD(path);
|
|
149
154
|
check_status(status);
|
|
150
155
|
return Qnil;
|
|
151
156
|
}
|
|
@@ -259,13 +264,14 @@ rb_cuModuleGetFunction(VALUE self, VALUE hmod, VALUE name)
|
|
|
259
264
|
{
|
|
260
265
|
CUfunction _hfunc;
|
|
261
266
|
CUmodule _hmod = (CUmodule)NUM2SIZET(hmod);
|
|
262
|
-
const char* _name =
|
|
267
|
+
const char* _name = StringValueCStr(name);
|
|
263
268
|
CUresult status;
|
|
264
269
|
|
|
265
270
|
struct cuModuleGetFunctionParam param = {&_hfunc, _hmod, _name};
|
|
266
271
|
status = (CUresult)rb_thread_call_without_gvl(cuModuleGetFunction_without_gvl_cb, ¶m, NULL, NULL);
|
|
267
272
|
//status = cuModuleGetFunction(&_hfunc, _hmod, _name);
|
|
268
273
|
|
|
274
|
+
RB_GC_GUARD(name);
|
|
269
275
|
check_status(status);
|
|
270
276
|
return SIZET2NUM((size_t)_hfunc);
|
|
271
277
|
}
|
|
@@ -292,15 +298,21 @@ rb_cuModuleGetGlobal(VALUE self, VALUE hmod, VALUE name)
|
|
|
292
298
|
CUdeviceptr _dptr;
|
|
293
299
|
size_t _bytes;
|
|
294
300
|
CUmodule _hmod = (CUmodule)NUM2SIZET(hmod);
|
|
295
|
-
const char* _name =
|
|
301
|
+
const char* _name = StringValueCStr(name);
|
|
296
302
|
CUresult status;
|
|
303
|
+
VALUE ret;
|
|
297
304
|
|
|
298
305
|
struct cuModuleGetGlobalParam param = {&_dptr, &_bytes, _hmod, _name};
|
|
299
306
|
status = (CUresult)rb_thread_call_without_gvl(cuModuleGetGlobal_without_gvl_cb, ¶m, NULL, NULL);
|
|
300
307
|
//status = cuModuleGetGlobal(&_dptr, &_bytes, _hmod, _name);
|
|
301
308
|
|
|
309
|
+
RB_GC_GUARD(name);
|
|
302
310
|
check_status(status);
|
|
303
|
-
|
|
311
|
+
|
|
312
|
+
// _dptr addresses device memory, which the host cannot read directly.
|
|
313
|
+
ret = rb_str_new(NULL, (long)_bytes);
|
|
314
|
+
check_status(cuMemcpyDtoH(RSTRING_PTR(ret), _dptr, _bytes));
|
|
315
|
+
return ret;
|
|
304
316
|
}
|
|
305
317
|
|
|
306
318
|
struct cuModuleLoadParam {
|
|
@@ -321,13 +333,14 @@ static VALUE
|
|
|
321
333
|
rb_cuModuleLoad(VALUE self, VALUE fname)
|
|
322
334
|
{
|
|
323
335
|
CUmodule _module;
|
|
324
|
-
const char* _fname =
|
|
336
|
+
const char* _fname = StringValueCStr(fname);
|
|
325
337
|
CUresult status;
|
|
326
338
|
|
|
327
339
|
struct cuModuleLoadParam param = {&_module, _fname};
|
|
328
340
|
status = (CUresult)rb_thread_call_without_gvl(cuModuleLoad_without_gvl_cb, ¶m, NULL, NULL);
|
|
329
341
|
//status = cuModuleLoad(&_module, _fname);
|
|
330
342
|
|
|
343
|
+
RB_GC_GUARD(fname);
|
|
331
344
|
check_status(status);
|
|
332
345
|
return SIZET2NUM((size_t)_module);
|
|
333
346
|
}
|
|
@@ -350,13 +363,15 @@ static VALUE
|
|
|
350
363
|
rb_cuModuleLoadData(VALUE self, VALUE image)
|
|
351
364
|
{
|
|
352
365
|
CUmodule _module;
|
|
353
|
-
|
|
366
|
+
// A cubin is binary, so the image is not required to be NUL-free.
|
|
367
|
+
const void* _image = (void*)StringValuePtr(image);
|
|
354
368
|
CUresult status;
|
|
355
369
|
|
|
356
370
|
struct cuModuleLoadDataParam param = {&_module, _image};
|
|
357
371
|
status = (CUresult)rb_thread_call_without_gvl(cuModuleLoadData_without_gvl_cb, ¶m, NULL, NULL);
|
|
358
372
|
//status = cuModuleLoadData(&_module, _image);
|
|
359
373
|
|
|
374
|
+
RB_GC_GUARD(image);
|
|
360
375
|
check_status(status);
|
|
361
376
|
return SIZET2NUM((size_t)_module);
|
|
362
377
|
}
|
|
@@ -50,16 +50,22 @@ cumo_cuda_runtime_malloc(size_t size)
|
|
|
50
50
|
void
|
|
51
51
|
cumo_cuda_runtime_free(char *ptr)
|
|
52
52
|
{
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
53
|
+
// Always offer the pointer to the pool first, whatever memory_pool_enabled
|
|
54
|
+
// says now: MemoryPool.enable/disable is public, so the state can differ
|
|
55
|
+
// from what it was at allocation time. Handing a pooled chunk to cudaFree
|
|
56
|
+
// releases memory the pool still hands out, and fails outright for a chunk
|
|
57
|
+
// which is not at the head of its buffer.
|
|
58
|
+
try {
|
|
59
|
+
// TODO(sonots): Get current CUDA stream and pass it
|
|
60
|
+
if (pool.Free(reinterpret_cast<intptr_t>(ptr))) {
|
|
61
|
+
return;
|
|
59
62
|
}
|
|
60
|
-
}
|
|
61
|
-
cumo_cuda_runtime_check_status(
|
|
63
|
+
} catch (const cumo::internal::CUDARuntimeError& e) {
|
|
64
|
+
cumo_cuda_runtime_check_status(e.status());
|
|
65
|
+
return;
|
|
62
66
|
}
|
|
67
|
+
// No pool owns it, so it came straight from cudaMallocManaged.
|
|
68
|
+
cumo_cuda_runtime_check_status(cudaFree((void*)ptr));
|
|
63
69
|
}
|
|
64
70
|
|
|
65
71
|
/*
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
#include "memory_pool_impl.hpp"
|
|
2
2
|
|
|
3
|
+
#include <cstdio>
|
|
4
|
+
|
|
3
5
|
#include <ruby.h>
|
|
4
6
|
|
|
5
7
|
namespace cumo {
|
|
@@ -25,9 +27,15 @@ Memory::~Memory() {
|
|
|
25
27
|
cudaError_t status = cudaFree(ptr_);
|
|
26
28
|
// CUDA driver may shut down before freeing memory inside memory pool.
|
|
27
29
|
// It is okay to simply ignore because CUDA driver automatically frees memory.
|
|
28
|
-
if (status
|
|
29
|
-
|
|
30
|
+
if (status == cudaSuccess || status == cudaErrorCudartUnloading) {
|
|
31
|
+
return;
|
|
30
32
|
}
|
|
33
|
+
// A destructor is implicitly noexcept, so throwing here would call
|
|
34
|
+
// std::terminate and abort the process. Report the failure instead:
|
|
35
|
+
// cudaFree only fails once the context is already unusable, and the
|
|
36
|
+
// next runtime call reports the same status to the caller anyway.
|
|
37
|
+
std::fprintf(stderr, "cumo: failed to free %zu bytes of device memory: %s\n",
|
|
38
|
+
size_, cudaGetErrorString(status));
|
|
31
39
|
}
|
|
32
40
|
}
|
|
33
41
|
|
|
@@ -63,14 +71,14 @@ void Merge(std::shared_ptr<Chunk>& self, std::shared_ptr<Chunk> remaining) {
|
|
|
63
71
|
|
|
64
72
|
void SingleDeviceMemoryPool::AppendToFreeList(size_t size, std::shared_ptr<Chunk>& chunk, cudaStream_t stream_ptr) {
|
|
65
73
|
assert(chunk != nullptr && !chunk->in_use());
|
|
66
|
-
|
|
74
|
+
size_t bin_index = GetBinIndex(size);
|
|
67
75
|
|
|
68
76
|
std::lock_guard<std::recursive_mutex> lock{mutex_};
|
|
69
77
|
|
|
70
78
|
Arena& arena = GetArena(stream_ptr);
|
|
71
79
|
ArenaIndexMap& arena_index_map = GetArenaIndexMap(stream_ptr);
|
|
72
|
-
|
|
73
|
-
|
|
80
|
+
size_t arena_index = std::lower_bound(arena_index_map.begin(), arena_index_map.end(), bin_index) - arena_index_map.begin();
|
|
81
|
+
size_t length = arena_index_map.size();
|
|
74
82
|
if (arena_index >= length || arena_index_map.at(arena_index) != bin_index) {
|
|
75
83
|
arena_index_map.insert(arena_index_map.begin() + arena_index, bin_index);
|
|
76
84
|
arena.insert(arena.begin() + arena_index, FreeList{});
|
|
@@ -81,7 +89,7 @@ void SingleDeviceMemoryPool::AppendToFreeList(size_t size, std::shared_ptr<Chunk
|
|
|
81
89
|
|
|
82
90
|
bool SingleDeviceMemoryPool::RemoveFromFreeList(size_t size, std::shared_ptr<Chunk>& chunk, cudaStream_t stream_ptr) {
|
|
83
91
|
assert(chunk != nullptr && !chunk->in_use());
|
|
84
|
-
|
|
92
|
+
size_t bin_index = GetBinIndex(size);
|
|
85
93
|
|
|
86
94
|
std::lock_guard<std::recursive_mutex> lock{mutex_};
|
|
87
95
|
|
|
@@ -90,20 +98,32 @@ bool SingleDeviceMemoryPool::RemoveFromFreeList(size_t size, std::shared_ptr<Chu
|
|
|
90
98
|
if (arena_index_map.size() == 0) {
|
|
91
99
|
return false;
|
|
92
100
|
}
|
|
93
|
-
|
|
94
|
-
if (
|
|
101
|
+
size_t arena_index = std::lower_bound(arena_index_map.begin(), arena_index_map.end(), bin_index) - arena_index_map.begin();
|
|
102
|
+
if (arena_index == arena_index_map.size()) {
|
|
95
103
|
// Bin does not exist for the given chunk size.
|
|
96
104
|
return false;
|
|
97
105
|
}
|
|
98
106
|
if (arena_index_map.at(arena_index) != bin_index) {
|
|
99
107
|
return false;
|
|
100
108
|
}
|
|
101
|
-
assert(arena.size() >
|
|
109
|
+
assert(arena.size() > arena_index);
|
|
102
110
|
FreeList& free_list = arena[arena_index];
|
|
103
111
|
return EraseFromFreeList(free_list, chunk);
|
|
104
112
|
}
|
|
105
113
|
|
|
106
114
|
intptr_t SingleDeviceMemoryPool::Malloc(size_t size, cudaStream_t stream_ptr) {
|
|
115
|
+
if (size == 0) {
|
|
116
|
+
// A zero-sized chunk would share its address with the chunk it was
|
|
117
|
+
// split from, and aliased addresses break the `in_use_` bookkeeping.
|
|
118
|
+
// cudaMalloc returns a null pointer for a zero-sized request as well.
|
|
119
|
+
return 0;
|
|
120
|
+
}
|
|
121
|
+
if (size > kMaxAllocationSize) {
|
|
122
|
+
// Rounding up would wrap around in size_t and the pool would then hand
|
|
123
|
+
// out a chunk far smaller than requested. Such a request can never be
|
|
124
|
+
// satisfied, so report it as out of memory instead.
|
|
125
|
+
throw OutOfMemoryError(size, GetTotalBytes());
|
|
126
|
+
}
|
|
107
127
|
size = GetRoundedSize(size);
|
|
108
128
|
std::shared_ptr<Chunk> chunk = nullptr;
|
|
109
129
|
|
|
@@ -112,9 +132,9 @@ intptr_t SingleDeviceMemoryPool::Malloc(size_t size, cudaStream_t stream_ptr) {
|
|
|
112
132
|
|
|
113
133
|
// find best-fit, or a smallest larger allocation
|
|
114
134
|
Arena& arena = GetArena(stream_ptr);
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
for (
|
|
135
|
+
size_t arena_index = GetArenaIndex(size);
|
|
136
|
+
size_t arena_length = arena.size();
|
|
137
|
+
for (size_t i = arena_index; i < arena_length; ++i) {
|
|
118
138
|
FreeList& free_list = arena[i];
|
|
119
139
|
if (free_list.empty()) {
|
|
120
140
|
continue;
|
|
@@ -166,17 +186,24 @@ intptr_t SingleDeviceMemoryPool::Malloc(size_t size, cudaStream_t stream_ptr) {
|
|
|
166
186
|
return chunk->ptr();
|
|
167
187
|
}
|
|
168
188
|
|
|
169
|
-
|
|
189
|
+
bool SingleDeviceMemoryPool::Free(intptr_t ptr, cudaStream_t stream_ptr) {
|
|
170
190
|
std::shared_ptr<Chunk> chunk = nullptr;
|
|
171
191
|
|
|
172
192
|
{
|
|
173
193
|
std::lock_guard<std::recursive_mutex> lock{mutex_};
|
|
174
194
|
|
|
175
|
-
|
|
176
|
-
//
|
|
177
|
-
|
|
195
|
+
// find rather than operator[], which would insert an empty entry for
|
|
196
|
+
// every pointer this pool does not own.
|
|
197
|
+
auto it = in_use_.find(ptr);
|
|
198
|
+
if (it == in_use_.end()) {
|
|
199
|
+
return false;
|
|
200
|
+
}
|
|
201
|
+
chunk = it->second;
|
|
202
|
+
in_use_.erase(it);
|
|
203
|
+
if (!chunk) {
|
|
204
|
+
return false;
|
|
205
|
+
}
|
|
178
206
|
chunk->set_in_use(false);
|
|
179
|
-
in_use_.erase(ptr);
|
|
180
207
|
}
|
|
181
208
|
|
|
182
209
|
if (chunk->next() != nullptr && !chunk->next()->in_use()) {
|
|
@@ -191,6 +218,7 @@ void SingleDeviceMemoryPool::Free(intptr_t ptr, cudaStream_t stream_ptr) {
|
|
|
191
218
|
}
|
|
192
219
|
}
|
|
193
220
|
AppendToFreeList(chunk->size(), chunk, stream_ptr);
|
|
221
|
+
return true;
|
|
194
222
|
}
|
|
195
223
|
|
|
196
224
|
void SingleDeviceMemoryPool::CompactIndex(cudaStream_t stream_ptr, bool free) {
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
|
|
4
4
|
#include <algorithm>
|
|
5
5
|
#include <cassert>
|
|
6
|
+
#include <limits>
|
|
6
7
|
#include <memory>
|
|
7
8
|
#include <mutex>
|
|
8
9
|
#include <stdexcept>
|
|
@@ -21,6 +22,10 @@ namespace internal {
|
|
|
21
22
|
// cf. https://gist.github.com/sonots/41daaa6432b1c8b27ef782cd14064269
|
|
22
23
|
constexpr int kRoundSize = 512; // bytes
|
|
23
24
|
|
|
25
|
+
// The largest size which can be rounded up to a multiple of kRoundSize without
|
|
26
|
+
// wrapping around in size_t. Larger requests can never be satisfied anyway.
|
|
27
|
+
constexpr size_t kMaxAllocationSize = std::numeric_limits<size_t>::max() - (kRoundSize - 1);
|
|
28
|
+
|
|
24
29
|
class CUDARuntimeError : public std::runtime_error {
|
|
25
30
|
private:
|
|
26
31
|
cudaError_t status_;
|
|
@@ -143,7 +148,7 @@ public:
|
|
|
143
148
|
|
|
144
149
|
using FreeList = std::vector<std::shared_ptr<Chunk>>; // list of free chunk
|
|
145
150
|
using Arena = std::vector<FreeList>; // free_list w.r.t arena index
|
|
146
|
-
using ArenaIndexMap = std::vector<
|
|
151
|
+
using ArenaIndexMap = std::vector<size_t>; // arena index <=> bin size index
|
|
147
152
|
|
|
148
153
|
// Memory pool implementation for single device.
|
|
149
154
|
// - The allocator attempts to find the smallest cached block that will fit
|
|
@@ -167,7 +172,9 @@ public:
|
|
|
167
172
|
|
|
168
173
|
intptr_t Malloc(size_t size, cudaStream_t stream_ptr = 0);
|
|
169
174
|
|
|
170
|
-
|
|
175
|
+
// Returns false if this pool did not allocate the pointer, in which case
|
|
176
|
+
// nothing was freed.
|
|
177
|
+
bool Free(intptr_t ptr, cudaStream_t stream_ptr = 0);
|
|
171
178
|
|
|
172
179
|
// Free all **non-split** chunks in all arenas
|
|
173
180
|
void FreeAllBlocks();
|
|
@@ -188,17 +195,29 @@ public:
|
|
|
188
195
|
// private:
|
|
189
196
|
|
|
190
197
|
// Rounds up the memory size to fit memory alignment of cudaMalloc.
|
|
198
|
+
//
|
|
199
|
+
// `size` must not exceed kMaxAllocationSize. Otherwise the addition below
|
|
200
|
+
// wraps around in size_t, and the largest sizes would round down to 0.
|
|
191
201
|
size_t GetRoundedSize(size_t size) {
|
|
202
|
+
assert(size <= kMaxAllocationSize);
|
|
192
203
|
return ((size + kRoundSize - 1) / kRoundSize) * kRoundSize;
|
|
193
204
|
}
|
|
194
205
|
|
|
195
206
|
// Get bin index regarding the memory size
|
|
196
|
-
|
|
207
|
+
//
|
|
208
|
+
// The index is a size_t rather than an int because the bin index of a huge
|
|
209
|
+
// size does not fit in an int. A truncated (possibly negative) index makes
|
|
210
|
+
// the free list search below start at the wrong bin and hand out a chunk
|
|
211
|
+
// smaller than requested.
|
|
212
|
+
//
|
|
213
|
+
// `size` must be positive; a zero-sized allocation has no bin.
|
|
214
|
+
size_t GetBinIndex(size_t size) {
|
|
215
|
+
assert(size > 0);
|
|
197
216
|
return (size - 1) / kRoundSize;
|
|
198
217
|
}
|
|
199
218
|
|
|
200
|
-
|
|
201
|
-
|
|
219
|
+
size_t GetArenaIndex(size_t size, cudaStream_t stream_ptr = 0) {
|
|
220
|
+
size_t bin_index = GetBinIndex(size);
|
|
202
221
|
ArenaIndexMap& arena_index_map = GetArenaIndexMap(stream_ptr);
|
|
203
222
|
return std::lower_bound(arena_index_map.begin(), arena_index_map.end(), bin_index) - arena_index_map.begin();
|
|
204
223
|
}
|
|
@@ -308,9 +327,30 @@ public:
|
|
|
308
327
|
// Args:
|
|
309
328
|
// ptr (intptr_t): Pointer of the memory buffer
|
|
310
329
|
// stream_ptr (cudaStream_t): Return the memory to the arena of given stream
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
330
|
+
// Returns:
|
|
331
|
+
// bool: false if no pool allocated the pointer, in which case nothing
|
|
332
|
+
// was freed and the caller has to free it by its own means.
|
|
333
|
+
bool Free(intptr_t ptr, cudaStream_t stream_ptr = 0) {
|
|
334
|
+
if (pools_.empty()) { // nothing has ever been allocated from a pool
|
|
335
|
+
return false;
|
|
336
|
+
}
|
|
337
|
+
int current_device_id = device_id();
|
|
338
|
+
auto it = pools_.find(current_device_id);
|
|
339
|
+
if (it != pools_.end() && it->second.Free(ptr, stream_ptr)) {
|
|
340
|
+
return true;
|
|
341
|
+
}
|
|
342
|
+
// The current device may have been switched since the allocation.
|
|
343
|
+
// cudaMallocManaged hands out addresses which are unique across the
|
|
344
|
+
// host and every device, so the pointer identifies its pool on its own.
|
|
345
|
+
for (auto& entry : pools_) {
|
|
346
|
+
if (entry.first == current_device_id) {
|
|
347
|
+
continue;
|
|
348
|
+
}
|
|
349
|
+
if (entry.second.Free(ptr, stream_ptr)) {
|
|
350
|
+
return true;
|
|
351
|
+
}
|
|
352
|
+
}
|
|
353
|
+
return false;
|
|
314
354
|
}
|
|
315
355
|
|
|
316
356
|
// Free all **non-split** chunks in all arenas
|
|
@@ -134,12 +134,15 @@ public:
|
|
|
134
134
|
void Run() {
|
|
135
135
|
TearDown(); SetUp(); TestGetRoundedSize();
|
|
136
136
|
TearDown(); SetUp(); TestGetBinIndex();
|
|
137
|
+
TearDown(); SetUp(); TestGetArenaIndexWithHugeSize();
|
|
137
138
|
TearDown(); SetUp(); TestAppendToFreeList();
|
|
138
139
|
TearDown(); SetUp(); TestRemoveFromFreeList();
|
|
139
140
|
TearDown(); SetUp(); TestMalloc();
|
|
140
141
|
TearDown(); SetUp(); TestMallocWithZero();
|
|
142
|
+
TearDown(); SetUp(); TestMallocWithHugeSize();
|
|
141
143
|
TearDown(); SetUp(); TestFree();
|
|
142
144
|
TearDown(); SetUp(); TestFreeDoubly();
|
|
145
|
+
TearDown(); SetUp(); TestFreeReportsOwnership();
|
|
143
146
|
TearDown(); SetUp(); TestMallocSplit();
|
|
144
147
|
TearDown(); SetUp(); TestFreeMerge();
|
|
145
148
|
TearDown(); SetUp(); TestFreeDifferentSize();
|
|
@@ -156,12 +159,28 @@ public:
|
|
|
156
159
|
assert(pool_->GetRoundedSize(kRoundSize - 1) == kRoundSize);
|
|
157
160
|
assert(pool_->GetRoundedSize(kRoundSize) == kRoundSize);
|
|
158
161
|
assert(pool_->GetRoundedSize(kRoundSize + 1) == kRoundSize * 2);
|
|
162
|
+
// the largest roundable size must not wrap around to 0
|
|
163
|
+
assert(pool_->GetRoundedSize(kMaxAllocationSize) == kMaxAllocationSize);
|
|
159
164
|
}
|
|
160
165
|
|
|
161
166
|
void TestGetBinIndex() {
|
|
162
167
|
assert(pool_->GetBinIndex(kRoundSize - 1) == 0);
|
|
163
168
|
assert(pool_->GetBinIndex(kRoundSize) == 0);
|
|
164
169
|
assert(pool_->GetBinIndex(kRoundSize + 1) == 1);
|
|
170
|
+
// the bin index of a huge size does not fit in an int
|
|
171
|
+
assert(pool_->GetBinIndex(kMaxAllocationSize) == (kMaxAllocationSize - 1) / kRoundSize);
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
void TestGetArenaIndexWithHugeSize() {
|
|
175
|
+
auto mem = std::make_shared<Memory>(kRoundSize * 4);
|
|
176
|
+
auto chunk = std::make_shared<Chunk>(mem, 0, mem->size(), stream_ptr_);
|
|
177
|
+
pool_->AppendToFreeList(chunk->size(), chunk, stream_ptr_);
|
|
178
|
+
|
|
179
|
+
assert(pool_->GetArenaIndex(kRoundSize * 4, stream_ptr_) == 0);
|
|
180
|
+
// A bin index which does not fit in an int must not fold back to a
|
|
181
|
+
// smaller arena index, or the free list search would pick this chunk.
|
|
182
|
+
assert(pool_->GetArenaIndex(size_t(1) << 41, stream_ptr_) == 1);
|
|
183
|
+
assert(pool_->GetArenaIndex(kMaxAllocationSize, stream_ptr_) == 1);
|
|
165
184
|
}
|
|
166
185
|
|
|
167
186
|
void TestAppendToFreeList() {
|
|
@@ -292,7 +311,38 @@ public:
|
|
|
292
311
|
}
|
|
293
312
|
|
|
294
313
|
void TestMallocWithZero() {
|
|
295
|
-
pool_->Malloc(
|
|
314
|
+
intptr_t p1 = pool_->Malloc(kRoundSize * 4);
|
|
315
|
+
pool_->Free(p1);
|
|
316
|
+
|
|
317
|
+
assert(pool_->Malloc(0) == 0); // actually, cuda returns 0
|
|
318
|
+
// A zero-sized request must not take a cached chunk, or two live
|
|
319
|
+
// allocations would share one address.
|
|
320
|
+
assert(pool_->GetUsedBytes() == 0);
|
|
321
|
+
assert(pool_->GetFreeBytes() == kRoundSize * 4);
|
|
322
|
+
assert(pool_->Malloc(kRoundSize * 4) == p1);
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
void TestMallocWithHugeSize() {
|
|
326
|
+
intptr_t p1 = pool_->Malloc(kRoundSize * 4);
|
|
327
|
+
pool_->Free(p1);
|
|
328
|
+
|
|
329
|
+
// Sizes which cannot be rounded up without wrapping around in size_t
|
|
330
|
+
// must be rejected instead of being rounded down to 0.
|
|
331
|
+
assert(RaisesOutOfMemory(kMaxAllocationSize + 1));
|
|
332
|
+
assert(RaisesOutOfMemory(std::numeric_limits<size_t>::max()));
|
|
333
|
+
|
|
334
|
+
assert(pool_->GetUsedBytes() == 0);
|
|
335
|
+
assert(pool_->GetFreeBytes() == kRoundSize * 4);
|
|
336
|
+
assert(pool_->Malloc(kRoundSize * 4) == p1);
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
bool RaisesOutOfMemory(size_t size) {
|
|
340
|
+
try {
|
|
341
|
+
pool_->Malloc(size);
|
|
342
|
+
} catch (const OutOfMemoryError&) {
|
|
343
|
+
return true;
|
|
344
|
+
}
|
|
345
|
+
return false;
|
|
296
346
|
}
|
|
297
347
|
|
|
298
348
|
void TestFree() {
|
|
@@ -308,6 +358,33 @@ public:
|
|
|
308
358
|
// pool_->Free(p1); // will abort
|
|
309
359
|
}
|
|
310
360
|
|
|
361
|
+
// Free reports whether the pool owns the pointer, so that a caller which
|
|
362
|
+
// allocated outside the pool can fall back to cudaFree without releasing
|
|
363
|
+
// memory the pool still hands out.
|
|
364
|
+
void TestFreeReportsOwnership() {
|
|
365
|
+
intptr_t p = pool_->Malloc(kRoundSize * 4);
|
|
366
|
+
assert(pool_->Free(p));
|
|
367
|
+
assert(!pool_->Free(p)); // already returned to the free list
|
|
368
|
+
|
|
369
|
+
assert(!pool_->Free(0));
|
|
370
|
+
assert(!pool_->Free(p + 1));
|
|
371
|
+
|
|
372
|
+
// a buffer allocated outside the pool, as happens while the pool is
|
|
373
|
+
// disabled, stays the caller's to free
|
|
374
|
+
void* raw = nullptr;
|
|
375
|
+
CheckStatus(cudaMallocManaged(&raw, kRoundSize, cudaMemAttachGlobal));
|
|
376
|
+
assert(!pool_->Free(reinterpret_cast<intptr_t>(raw)));
|
|
377
|
+
CheckStatus(cudaFree(raw));
|
|
378
|
+
|
|
379
|
+
// a chunk which is not at the head of its buffer is owned just as much,
|
|
380
|
+
// and is exactly the one cudaFree cannot free
|
|
381
|
+
intptr_t head = pool_->Malloc(kRoundSize * 2);
|
|
382
|
+
intptr_t tail = pool_->Malloc(kRoundSize * 2);
|
|
383
|
+
assert(head != tail);
|
|
384
|
+
assert(pool_->Free(tail));
|
|
385
|
+
assert(pool_->Free(head));
|
|
386
|
+
}
|
|
387
|
+
|
|
311
388
|
void TestMallocSplit() {
|
|
312
389
|
intptr_t p = pool_->Malloc(kRoundSize * 4);
|
|
313
390
|
pool_->Free(p);
|
|
@@ -543,6 +620,29 @@ public:
|
|
|
543
620
|
}
|
|
544
621
|
};
|
|
545
622
|
|
|
623
|
+
// Resets the CUDA device, so it has to run after every other test.
|
|
624
|
+
class TestMemoryDestructor {
|
|
625
|
+
public:
|
|
626
|
+
void Run() {
|
|
627
|
+
TestFailingFreeDoesNotAbort();
|
|
628
|
+
}
|
|
629
|
+
|
|
630
|
+
// A destructor is implicitly noexcept, so a throwing CheckStatus here used
|
|
631
|
+
// to call std::terminate and abort the whole process.
|
|
632
|
+
void TestFailingFreeDoesNotAbort() {
|
|
633
|
+
auto mem = std::make_shared<Memory>(kRoundSize);
|
|
634
|
+
// Destroying the primary context invalidates the pointer, so the
|
|
635
|
+
// cudaFree in ~Memory fails with cudaErrorInvalidValue.
|
|
636
|
+
CheckStatus(cudaDeviceReset());
|
|
637
|
+
|
|
638
|
+
std::cerr << "(the cumo warning below is expected)" << std::endl;
|
|
639
|
+
mem.reset();
|
|
640
|
+
// Reaching this line is the assertion: the destructor reported the
|
|
641
|
+
// failure instead of terminating the process.
|
|
642
|
+
cudaGetLastError(); // clear the error left by the failed cudaFree
|
|
643
|
+
}
|
|
644
|
+
};
|
|
645
|
+
|
|
546
646
|
} // namespace internal
|
|
547
647
|
} // namespace cumo
|
|
548
648
|
|
|
@@ -550,5 +650,6 @@ int main() {
|
|
|
550
650
|
cumo::internal::TestChunk{}.Run();
|
|
551
651
|
cumo::internal::TestSingleDeviceMemoryPool{}.Run();
|
|
552
652
|
cumo::internal::TestMemoryPool{}.Run();
|
|
653
|
+
cumo::internal::TestMemoryDestructor{}.Run();
|
|
553
654
|
return 0;
|
|
554
655
|
}
|