mwrap 2.3.0 → 3.0.0.pre1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/COPYING +617 -282
- data/Documentation/.gitignore +2 -0
- data/Documentation/GNUmakefile +63 -0
- data/Documentation/mwrap.1 +242 -0
- data/Documentation/mwrap.pod +123 -0
- data/MANIFEST +13 -1
- data/README +25 -17
- data/Rakefile +10 -2
- data/VERSION-GEN +1 -1
- data/ext/mwrap/check.h +23 -0
- data/ext/mwrap/dlmalloc_c.h +6294 -0
- data/ext/mwrap/extconf.rb +3 -7
- data/ext/mwrap/gcc.h +13 -0
- data/ext/mwrap/httpd.h +1367 -0
- data/ext/mwrap/mwrap.c +44 -1151
- data/ext/mwrap/mwrap_core.h +1095 -0
- data/ext/mwrap/mymalloc.h +299 -0
- data/ext/mwrap/picohttpparser.h +92 -0
- data/ext/mwrap/picohttpparser_c.h +670 -0
- data/lib/mwrap/version.rb +1 -1
- data/lib/mwrap_rack.rb +14 -58
- data/mwrap.gemspec +10 -3
- data/t/httpd.t +191 -0
- data/t/test_common.perl +54 -0
- data/test/test_mwrap.rb +34 -50
- metadata +21 -7
data/ext/mwrap/mwrap.c
CHANGED
@@ -2,864 +2,14 @@
|
|
2
2
|
* Copyright (C) mwrap hackers <mwrap-public@80x24.org>
|
3
3
|
* License: GPL-2.0+ <https://www.gnu.org/licenses/gpl-2.0.txt>
|
4
4
|
*/
|
5
|
-
#define
|
6
|
-
#include
|
7
|
-
#include <ruby/thread.h>
|
8
|
-
#include <ruby/io.h>
|
9
|
-
#include <execinfo.h>
|
10
|
-
#include <stdio.h>
|
11
|
-
#include <stdlib.h>
|
12
|
-
#include <string.h>
|
13
|
-
#include <dlfcn.h>
|
14
|
-
#include <assert.h>
|
15
|
-
#include <errno.h>
|
16
|
-
#include <sys/types.h>
|
17
|
-
#include <sys/stat.h>
|
18
|
-
#include <fcntl.h>
|
19
|
-
#include <pthread.h>
|
20
|
-
#include <urcu-bp.h>
|
21
|
-
#include <urcu/rculfhash.h>
|
22
|
-
#include <urcu/rculist.h>
|
23
|
-
#include "jhash.h"
|
24
|
-
|
25
|
-
#if __STDC_VERSION__ >= 201112
|
26
|
-
# define MWRAP_TSD _Thread_local
|
27
|
-
#elif defined(__GNUC__)
|
28
|
-
# define MWRAP_TSD __thread
|
29
|
-
#else
|
30
|
-
# error _Thread_local nor __thread supported
|
31
|
-
#endif
|
5
|
+
#define MWRAP_RUBY 1
|
6
|
+
#include "mwrap_core.h"
|
32
7
|
|
33
8
|
static ID id_uminus;
|
34
|
-
|
35
|
-
|
36
|
-
#ifdef HAVE_RUBY_RACTOR_H /* Ruby 3.0+ */
|
37
|
-
extern MWRAP_TSD void * __attribute__((weak)) ruby_current_ec;
|
38
|
-
#else /* Ruby 2.6-2.7 */
|
39
|
-
extern void * __attribute__((weak)) ruby_current_execution_context_ptr;
|
40
|
-
# define ruby_current_ec ruby_current_execution_context_ptr
|
41
|
-
#endif
|
42
|
-
extern void * __attribute__((weak)) ruby_current_vm_ptr; /* for rb_gc_count */
|
43
|
-
extern size_t __attribute__((weak)) rb_gc_count(void);
|
9
|
+
extern VALUE __attribute__((weak)) rb_stderr;
|
44
10
|
extern VALUE __attribute__((weak)) rb_cObject;
|
45
11
|
extern VALUE __attribute__((weak)) rb_eTypeError;
|
46
12
|
extern VALUE __attribute__((weak)) rb_yield(VALUE);
|
47
|
-
int __attribute__((weak)) ruby_thread_has_gvl_p(void);
|
48
|
-
|
49
|
-
static size_t total_bytes_inc, total_bytes_dec;
|
50
|
-
|
51
|
-
/* true for glibc/dlmalloc/ptmalloc, not sure about jemalloc */
|
52
|
-
#define ASSUMED_MALLOC_ALIGNMENT (sizeof(void *) * 2)
|
53
|
-
|
54
|
-
/* match values in Ruby gc.c */
|
55
|
-
#define HEAP_PAGE_ALIGN_LOG 14
|
56
|
-
enum {
|
57
|
-
HEAP_PAGE_ALIGN = (1UL << HEAP_PAGE_ALIGN_LOG)
|
58
|
-
#ifndef HEAP_PAGE_SIZE /* Ruby 2.6-2.7 only */
|
59
|
-
,
|
60
|
-
REQUIRED_SIZE_BY_MALLOC = (sizeof(size_t) * 5),
|
61
|
-
HEAP_PAGE_SIZE = (HEAP_PAGE_ALIGN - REQUIRED_SIZE_BY_MALLOC)
|
62
|
-
#endif
|
63
|
-
};
|
64
|
-
|
65
|
-
#define IS_HEAP_PAGE_BODY ((struct src_loc *)-1)
|
66
|
-
|
67
|
-
#ifdef __FreeBSD__
|
68
|
-
void *__malloc(size_t);
|
69
|
-
void __free(void *);
|
70
|
-
# define real_malloc __malloc
|
71
|
-
# define real_free __free
|
72
|
-
#else
|
73
|
-
static void *(*real_malloc)(size_t);
|
74
|
-
static void (*real_free)(void *);
|
75
|
-
static int resolving_malloc;
|
76
|
-
#endif /* !FreeBSD */
|
77
|
-
|
78
|
-
/*
|
79
|
-
* we need to fake an OOM condition while dlsym is running,
|
80
|
-
* as that calls calloc under glibc, but we don't have the
|
81
|
-
* symbol for the jemalloc calloc, yet
|
82
|
-
*/
|
83
|
-
# define RETURN_IF_NOT_READY() do { \
|
84
|
-
if (!real_malloc) { \
|
85
|
-
errno = ENOMEM; \
|
86
|
-
return NULL; \
|
87
|
-
} \
|
88
|
-
} while (0)
|
89
|
-
|
90
|
-
static MWRAP_TSD size_t locating;
|
91
|
-
static size_t generation;
|
92
|
-
static size_t page_size;
|
93
|
-
static struct cds_lfht *totals;
|
94
|
-
union padded_mutex {
|
95
|
-
pthread_mutex_t mtx;
|
96
|
-
char pad[64];
|
97
|
-
};
|
98
|
-
|
99
|
-
/* a round-robin pool of mutexes */
|
100
|
-
#define MUTEX_NR (1 << 6)
|
101
|
-
#define MUTEX_MASK (MUTEX_NR - 1)
|
102
|
-
#ifdef __FreeBSD__
|
103
|
-
# define STATIC_MTX_INIT_OK (0)
|
104
|
-
#else /* only tested on Linux + glibc */
|
105
|
-
# define STATIC_MTX_INIT_OK (1)
|
106
|
-
#endif
|
107
|
-
static size_t mutex_i;
|
108
|
-
static union padded_mutex mutexes[MUTEX_NR] = {
|
109
|
-
#if STATIC_MTX_INIT_OK
|
110
|
-
[0 ... (MUTEX_NR-1)].mtx = PTHREAD_MUTEX_INITIALIZER
|
111
|
-
#endif
|
112
|
-
};
|
113
|
-
|
114
|
-
#define ACC_INIT(name) { .nr=0, .min=INT64_MAX, .max=-1, .m2=0, .mean=0 }
|
115
|
-
struct acc {
|
116
|
-
uint64_t nr;
|
117
|
-
int64_t min;
|
118
|
-
int64_t max;
|
119
|
-
double m2;
|
120
|
-
double mean;
|
121
|
-
};
|
122
|
-
|
123
|
-
/* for tracking 16K-aligned heap page bodies (protected by GVL) */
|
124
|
-
struct {
|
125
|
-
pthread_mutex_t lock;
|
126
|
-
struct cds_list_head bodies;
|
127
|
-
struct cds_list_head freed;
|
128
|
-
|
129
|
-
struct acc alive;
|
130
|
-
struct acc reborn;
|
131
|
-
} hpb_stats = {
|
132
|
-
#if STATIC_MTX_INIT_OK
|
133
|
-
.lock = PTHREAD_MUTEX_INITIALIZER,
|
134
|
-
#endif
|
135
|
-
.bodies = CDS_LIST_HEAD_INIT(hpb_stats.bodies),
|
136
|
-
.freed = CDS_LIST_HEAD_INIT(hpb_stats.freed),
|
137
|
-
.alive = ACC_INIT(hpb_stats.alive),
|
138
|
-
.reborn = ACC_INIT(hpb_stats.reborn)
|
139
|
-
};
|
140
|
-
|
141
|
-
static pthread_mutex_t *mutex_assign(void)
|
142
|
-
{
|
143
|
-
return &mutexes[uatomic_add_return(&mutex_i, 1) & MUTEX_MASK].mtx;
|
144
|
-
}
|
145
|
-
|
146
|
-
static struct cds_lfht *
|
147
|
-
lfht_new(void)
|
148
|
-
{
|
149
|
-
return cds_lfht_new(16384, 1, 0, CDS_LFHT_AUTO_RESIZE, 0);
|
150
|
-
}
|
151
|
-
|
152
|
-
__attribute__((constructor)) static void resolve_malloc(void)
|
153
|
-
{
|
154
|
-
int err;
|
155
|
-
++locating;
|
156
|
-
|
157
|
-
/*
|
158
|
-
* PTHREAD_MUTEX_INITIALIZER on FreeBSD means lazy initialization,
|
159
|
-
* which happens at pthread_mutex_lock, and that calls calloc
|
160
|
-
*/
|
161
|
-
if (!STATIC_MTX_INIT_OK) {
|
162
|
-
size_t i;
|
163
|
-
|
164
|
-
for (i = 0; i < MUTEX_NR; i++) {
|
165
|
-
err = pthread_mutex_init(&mutexes[i].mtx, 0);
|
166
|
-
if (err) {
|
167
|
-
fprintf(stderr, "error: %s\n", strerror(err));
|
168
|
-
_exit(1);
|
169
|
-
}
|
170
|
-
}
|
171
|
-
err = pthread_mutex_init(&hpb_stats.lock, 0);
|
172
|
-
if (err) {
|
173
|
-
fprintf(stderr, "error: %s\n", strerror(err));
|
174
|
-
_exit(1);
|
175
|
-
}
|
176
|
-
/* initialize mutexes used by urcu-bp */
|
177
|
-
rcu_read_lock();
|
178
|
-
rcu_read_unlock();
|
179
|
-
#ifndef __FreeBSD__
|
180
|
-
} else {
|
181
|
-
if (!real_malloc) {
|
182
|
-
resolving_malloc = 1;
|
183
|
-
real_malloc = dlsym(RTLD_NEXT, "malloc");
|
184
|
-
}
|
185
|
-
real_free = dlsym(RTLD_NEXT, "free");
|
186
|
-
if (!real_malloc || !real_free) {
|
187
|
-
fprintf(stderr, "missing malloc/aligned_alloc/free\n"
|
188
|
-
"\t%p %p\n", real_malloc, real_free);
|
189
|
-
_exit(1);
|
190
|
-
}
|
191
|
-
#endif /* !__FreeBSD__ */
|
192
|
-
}
|
193
|
-
CMM_STORE_SHARED(totals, lfht_new());
|
194
|
-
if (!CMM_LOAD_SHARED(totals))
|
195
|
-
fprintf(stderr, "failed to allocate totals table\n");
|
196
|
-
|
197
|
-
err = pthread_atfork(call_rcu_before_fork,
|
198
|
-
call_rcu_after_fork_parent,
|
199
|
-
call_rcu_after_fork_child);
|
200
|
-
if (err)
|
201
|
-
fprintf(stderr, "pthread_atfork failed: %s\n", strerror(err));
|
202
|
-
page_size = sysconf(_SC_PAGESIZE);
|
203
|
-
--locating;
|
204
|
-
}
|
205
|
-
|
206
|
-
#ifdef NDEBUG
|
207
|
-
#define QUIET_CC_WARNING(var) (void)var;
|
208
|
-
#else
|
209
|
-
#define QUIET_CC_WARNING(var)
|
210
|
-
#endif
|
211
|
-
|
212
|
-
static void
|
213
|
-
mutex_lock(pthread_mutex_t *m)
|
214
|
-
{
|
215
|
-
int err = pthread_mutex_lock(m);
|
216
|
-
assert(err == 0);
|
217
|
-
QUIET_CC_WARNING(err)
|
218
|
-
}
|
219
|
-
|
220
|
-
static void
|
221
|
-
mutex_unlock(pthread_mutex_t *m)
|
222
|
-
{
|
223
|
-
int err = pthread_mutex_unlock(m);
|
224
|
-
assert(err == 0);
|
225
|
-
QUIET_CC_WARNING(err)
|
226
|
-
}
|
227
|
-
|
228
|
-
#ifndef HAVE_MEMPCPY
|
229
|
-
static void *
|
230
|
-
my_mempcpy(void *dest, const void *src, size_t n)
|
231
|
-
{
|
232
|
-
return (char *)memcpy(dest, src, n) + n;
|
233
|
-
}
|
234
|
-
#define mempcpy(dst,src,n) my_mempcpy(dst,src,n)
|
235
|
-
#endif
|
236
|
-
|
237
|
-
/* stolen from glibc: */
|
238
|
-
#define RETURN_ADDRESS(nr) \
|
239
|
-
(uintptr_t)(__builtin_extract_return_addr(__builtin_return_address(nr)))
|
240
|
-
|
241
|
-
#define INT2STR_MAX (sizeof(int) == 4 ? 10 : 19)
|
242
|
-
static char *int2str(int num, char *dst, size_t * size)
|
243
|
-
{
|
244
|
-
if (num <= 9) {
|
245
|
-
*size -= 1;
|
246
|
-
*dst++ = (char)(num + '0');
|
247
|
-
return dst;
|
248
|
-
} else {
|
249
|
-
char buf[INT2STR_MAX];
|
250
|
-
char *end = buf + sizeof(buf);
|
251
|
-
char *p = end;
|
252
|
-
size_t adj;
|
253
|
-
|
254
|
-
do {
|
255
|
-
*size -= 1;
|
256
|
-
*--p = (char)((num % 10) + '0');
|
257
|
-
num /= 10;
|
258
|
-
} while (num && *size);
|
259
|
-
|
260
|
-
if (!num) {
|
261
|
-
adj = end - p;
|
262
|
-
return mempcpy(dst, p, adj);
|
263
|
-
}
|
264
|
-
}
|
265
|
-
return NULL;
|
266
|
-
}
|
267
|
-
|
268
|
-
/*
|
269
|
-
* rb_source_location_cstr relies on GET_EC(), and it's possible
|
270
|
-
* to have a native thread but no EC during the early and late
|
271
|
-
* (teardown) phases of the Ruby process
|
272
|
-
*/
|
273
|
-
static int has_ec_p(void)
|
274
|
-
{
|
275
|
-
return ruby_thread_has_gvl_p && ruby_thread_has_gvl_p() &&
|
276
|
-
ruby_current_vm_ptr && ruby_current_ec;
|
277
|
-
}
|
278
|
-
|
279
|
-
/* allocated via real_malloc/real_free */
|
280
|
-
struct src_loc {
|
281
|
-
pthread_mutex_t *mtx;
|
282
|
-
size_t total;
|
283
|
-
size_t allocations;
|
284
|
-
size_t frees;
|
285
|
-
size_t age_total; /* (age_total / frees) => mean age at free */
|
286
|
-
size_t max_lifespan;
|
287
|
-
struct cds_lfht_node hnode;
|
288
|
-
struct cds_list_head allocs; /* <=> alloc_hdr.node */
|
289
|
-
uint32_t hval;
|
290
|
-
uint32_t capa;
|
291
|
-
char k[];
|
292
|
-
};
|
293
|
-
|
294
|
-
/* every allocation has this in the header, maintain alignment with malloc */
|
295
|
-
struct alloc_hdr {
|
296
|
-
struct cds_list_head anode; /* <=> src_loc.allocs */
|
297
|
-
union {
|
298
|
-
struct {
|
299
|
-
size_t gen; /* rb_gc_count() */
|
300
|
-
struct src_loc *loc;
|
301
|
-
} live;
|
302
|
-
struct rcu_head dead;
|
303
|
-
struct {
|
304
|
-
size_t at; /* rb_gc_count() */
|
305
|
-
} hpb_freed;
|
306
|
-
} as;
|
307
|
-
void *real; /* what to call real_free on */
|
308
|
-
size_t size;
|
309
|
-
};
|
310
|
-
|
311
|
-
static char kbuf[PATH_MAX + INT2STR_MAX + sizeof(struct alloc_hdr) + 2];
|
312
|
-
|
313
|
-
static struct alloc_hdr *ptr2hdr(void *p)
|
314
|
-
{
|
315
|
-
return (struct alloc_hdr *)((uintptr_t)p - sizeof(struct alloc_hdr));
|
316
|
-
}
|
317
|
-
|
318
|
-
static void *hdr2ptr(struct alloc_hdr *h)
|
319
|
-
{
|
320
|
-
return (void *)((uintptr_t)h + sizeof(struct alloc_hdr));
|
321
|
-
}
|
322
|
-
|
323
|
-
static int loc_is_addr(const struct src_loc *l)
|
324
|
-
{
|
325
|
-
return l->capa == 0;
|
326
|
-
}
|
327
|
-
|
328
|
-
static size_t loc_size(const struct src_loc *l)
|
329
|
-
{
|
330
|
-
return loc_is_addr(l) ? sizeof(uintptr_t) : l->capa;
|
331
|
-
}
|
332
|
-
|
333
|
-
static int loc_eq(struct cds_lfht_node *node, const void *key)
|
334
|
-
{
|
335
|
-
const struct src_loc *existing;
|
336
|
-
const struct src_loc *k = key;
|
337
|
-
|
338
|
-
existing = caa_container_of(node, struct src_loc, hnode);
|
339
|
-
|
340
|
-
return (k->hval == existing->hval &&
|
341
|
-
k->capa == existing->capa &&
|
342
|
-
memcmp(k->k, existing->k, loc_size(k)) == 0);
|
343
|
-
}
|
344
|
-
|
345
|
-
/* note: not atomic */
|
346
|
-
static void
|
347
|
-
acc_add(struct acc *acc, size_t val)
|
348
|
-
{
|
349
|
-
double delta = val - acc->mean;
|
350
|
-
uint64_t nr = ++acc->nr;
|
351
|
-
|
352
|
-
/* just don't divide-by-zero if we ever hit this (unlikely :P) */
|
353
|
-
if (nr)
|
354
|
-
acc->mean += delta / nr;
|
355
|
-
|
356
|
-
acc->m2 += delta * (val - acc->mean);
|
357
|
-
if ((int64_t)val < acc->min)
|
358
|
-
acc->min = (int64_t)val;
|
359
|
-
if ((int64_t)val > acc->max)
|
360
|
-
acc->max = (int64_t)val;
|
361
|
-
}
|
362
|
-
|
363
|
-
#if SIZEOF_LONG == 8
|
364
|
-
# define INT64toNUM(x) LONG2NUM((long)x)
|
365
|
-
#elif defined(HAVE_LONG_LONG) && SIZEOF_LONG_LONG == 8
|
366
|
-
# define INT64toNUM(x) LL2NUM((LONG_LONG)x)
|
367
|
-
#endif
|
368
|
-
|
369
|
-
static VALUE
|
370
|
-
acc_max(const struct acc *acc)
|
371
|
-
{
|
372
|
-
return INT64toNUM(acc->max);
|
373
|
-
}
|
374
|
-
|
375
|
-
static VALUE
|
376
|
-
acc_min(const struct acc *acc)
|
377
|
-
{
|
378
|
-
return acc->min == INT64_MAX ? INT2FIX(-1) : INT64toNUM(acc->min);
|
379
|
-
}
|
380
|
-
|
381
|
-
static VALUE
|
382
|
-
acc_mean(const struct acc *acc)
|
383
|
-
{
|
384
|
-
return DBL2NUM(acc->nr ? acc->mean : HUGE_VAL);
|
385
|
-
}
|
386
|
-
|
387
|
-
static double
|
388
|
-
acc_stddev_dbl(const struct acc *acc)
|
389
|
-
{
|
390
|
-
if (acc->nr > 1) {
|
391
|
-
double variance = acc->m2 / (acc->nr - 1);
|
392
|
-
return sqrt(variance);
|
393
|
-
}
|
394
|
-
return 0.0;
|
395
|
-
}
|
396
|
-
|
397
|
-
static VALUE
|
398
|
-
acc_stddev(const struct acc *acc)
|
399
|
-
{
|
400
|
-
return DBL2NUM(acc_stddev_dbl(acc));
|
401
|
-
}
|
402
|
-
|
403
|
-
static struct src_loc *totals_add_rcu(const struct src_loc *k)
|
404
|
-
{
|
405
|
-
struct cds_lfht_iter iter;
|
406
|
-
struct cds_lfht_node *cur;
|
407
|
-
struct src_loc *l = 0;
|
408
|
-
struct cds_lfht *t;
|
409
|
-
|
410
|
-
again:
|
411
|
-
t = CMM_LOAD_SHARED(totals);
|
412
|
-
if (!t) goto out_unlock;
|
413
|
-
cds_lfht_lookup(t, k->hval, loc_eq, k, &iter);
|
414
|
-
cur = cds_lfht_iter_get_node(&iter);
|
415
|
-
if (cur) {
|
416
|
-
l = caa_container_of(cur, struct src_loc, hnode);
|
417
|
-
uatomic_add(&l->total, k->total);
|
418
|
-
uatomic_add(&l->allocations, 1);
|
419
|
-
} else {
|
420
|
-
size_t n = loc_size(k);
|
421
|
-
l = real_malloc(sizeof(*l) + n);
|
422
|
-
if (!l) goto out_unlock;
|
423
|
-
memcpy(l, k, sizeof(*l) + n);
|
424
|
-
l->mtx = mutex_assign();
|
425
|
-
l->age_total = 0;
|
426
|
-
l->max_lifespan = 0;
|
427
|
-
l->frees = 0;
|
428
|
-
l->allocations = 1;
|
429
|
-
CDS_INIT_LIST_HEAD(&l->allocs);
|
430
|
-
cur = cds_lfht_add_unique(t, k->hval, loc_eq, l, &l->hnode);
|
431
|
-
if (cur != &l->hnode) { /* lost race */
|
432
|
-
rcu_read_unlock();
|
433
|
-
real_free(l);
|
434
|
-
rcu_read_lock();
|
435
|
-
goto again;
|
436
|
-
}
|
437
|
-
}
|
438
|
-
out_unlock:
|
439
|
-
return l;
|
440
|
-
}
|
441
|
-
|
442
|
-
static void update_stats_rcu_unlock(const struct src_loc *l)
|
443
|
-
{
|
444
|
-
if (caa_likely(l)) rcu_read_unlock();
|
445
|
-
}
|
446
|
-
|
447
|
-
static struct src_loc *update_stats_rcu_lock(size_t size, uintptr_t caller)
|
448
|
-
{
|
449
|
-
struct src_loc *k, *ret = 0;
|
450
|
-
static const size_t xlen = sizeof(caller);
|
451
|
-
char *dst;
|
452
|
-
|
453
|
-
if (caa_unlikely(!CMM_LOAD_SHARED(totals))) return 0;
|
454
|
-
if (locating++) goto out; /* do not recurse into another *alloc */
|
455
|
-
|
456
|
-
uatomic_add(&total_bytes_inc, size);
|
457
|
-
|
458
|
-
rcu_read_lock();
|
459
|
-
if (has_ec_p()) {
|
460
|
-
int line;
|
461
|
-
const char *ptr = rb_source_location_cstr(&line);
|
462
|
-
size_t len;
|
463
|
-
size_t int_size = INT2STR_MAX;
|
464
|
-
|
465
|
-
generation = rb_gc_count();
|
466
|
-
|
467
|
-
if (!ptr) goto unknown;
|
468
|
-
|
469
|
-
/* avoid vsnprintf or anything which could call malloc here: */
|
470
|
-
len = strlen(ptr);
|
471
|
-
k = (void *)kbuf;
|
472
|
-
k->total = size;
|
473
|
-
dst = mempcpy(k->k, ptr, len);
|
474
|
-
*dst++ = ':';
|
475
|
-
dst = int2str(line, dst, &int_size);
|
476
|
-
if (dst) {
|
477
|
-
*dst = 0; /* terminate string */
|
478
|
-
k->capa = (uint32_t)(dst - k->k + 1);
|
479
|
-
k->hval = jhash(k->k, k->capa, 0xdeadbeef);
|
480
|
-
ret = totals_add_rcu(k);
|
481
|
-
} else {
|
482
|
-
rb_bug("bad math making key from location %s:%d\n",
|
483
|
-
ptr, line);
|
484
|
-
}
|
485
|
-
} else {
|
486
|
-
unknown:
|
487
|
-
k = alloca(sizeof(*k) + xlen);
|
488
|
-
k->total = size;
|
489
|
-
memcpy(k->k, &caller, xlen);
|
490
|
-
k->capa = 0;
|
491
|
-
k->hval = jhash(k->k, xlen, 0xdeadbeef);
|
492
|
-
ret = totals_add_rcu(k);
|
493
|
-
}
|
494
|
-
out:
|
495
|
-
--locating;
|
496
|
-
return ret;
|
497
|
-
}
|
498
|
-
|
499
|
-
size_t malloc_usable_size(void *p)
|
500
|
-
{
|
501
|
-
return ptr2hdr(p)->size;
|
502
|
-
}
|
503
|
-
|
504
|
-
static void
|
505
|
-
free_hdr_rcu(struct rcu_head *dead)
|
506
|
-
{
|
507
|
-
struct alloc_hdr *h = caa_container_of(dead, struct alloc_hdr, as.dead);
|
508
|
-
real_free(h->real);
|
509
|
-
}
|
510
|
-
|
511
|
-
void free(void *p)
|
512
|
-
{
|
513
|
-
if (p) {
|
514
|
-
struct alloc_hdr *h = ptr2hdr(p);
|
515
|
-
struct src_loc *l = h->as.live.loc;
|
516
|
-
|
517
|
-
if (!real_free) return; /* oh well, leak a little */
|
518
|
-
if (l && l != IS_HEAP_PAGE_BODY) {
|
519
|
-
size_t age = generation - h->as.live.gen;
|
520
|
-
|
521
|
-
uatomic_add(&total_bytes_dec, h->size);
|
522
|
-
uatomic_set(&h->size, 0);
|
523
|
-
uatomic_add(&l->frees, 1);
|
524
|
-
uatomic_add(&l->age_total, age);
|
525
|
-
|
526
|
-
mutex_lock(l->mtx);
|
527
|
-
cds_list_del_rcu(&h->anode);
|
528
|
-
if (age > l->max_lifespan)
|
529
|
-
l->max_lifespan = age;
|
530
|
-
mutex_unlock(l->mtx);
|
531
|
-
|
532
|
-
call_rcu(&h->as.dead, free_hdr_rcu);
|
533
|
-
} else if (l == IS_HEAP_PAGE_BODY) {
|
534
|
-
size_t gen = generation;
|
535
|
-
size_t age = gen - h->as.live.gen;
|
536
|
-
|
537
|
-
h->as.hpb_freed.at = gen;
|
538
|
-
|
539
|
-
mutex_lock(&hpb_stats.lock);
|
540
|
-
acc_add(&hpb_stats.alive, age);
|
541
|
-
|
542
|
-
/* hpb_stats.bodies => hpb_stats.freed */
|
543
|
-
cds_list_move(&h->anode, &hpb_stats.freed);
|
544
|
-
|
545
|
-
mutex_unlock(&hpb_stats.lock);
|
546
|
-
} else {
|
547
|
-
real_free(h->real);
|
548
|
-
}
|
549
|
-
}
|
550
|
-
}
|
551
|
-
|
552
|
-
static void
|
553
|
-
alloc_insert_rcu(struct src_loc *l, struct alloc_hdr *h, size_t size, void *real)
|
554
|
-
{
|
555
|
-
/* we need src_loc to remain alive for the duration of this call */
|
556
|
-
if (!h) return;
|
557
|
-
h->size = size;
|
558
|
-
h->real = real;
|
559
|
-
h->as.live.loc = l;
|
560
|
-
h->as.live.gen = generation;
|
561
|
-
if (l) {
|
562
|
-
mutex_lock(l->mtx);
|
563
|
-
cds_list_add_rcu(&h->anode, &l->allocs);
|
564
|
-
mutex_unlock(l->mtx);
|
565
|
-
}
|
566
|
-
}
|
567
|
-
|
568
|
-
static size_t size_align(size_t size, size_t alignment)
|
569
|
-
{
|
570
|
-
return ((size + (alignment - 1)) & ~(alignment - 1));
|
571
|
-
}
|
572
|
-
|
573
|
-
static bool ptr_is_aligned(const void *ptr, size_t alignment)
|
574
|
-
{
|
575
|
-
return ((uintptr_t)ptr & (alignment - 1)) == 0;
|
576
|
-
}
|
577
|
-
|
578
|
-
static void *ptr_align(void *ptr, size_t alignment)
|
579
|
-
{
|
580
|
-
return (void *)(((uintptr_t)ptr + (alignment - 1)) & ~(alignment - 1));
|
581
|
-
}
|
582
|
-
|
583
|
-
static bool is_power_of_two(size_t n) { return (n & (n - 1)) == 0; }
|
584
|
-
|
585
|
-
static int
|
586
|
-
internal_memalign(void **pp, size_t alignment, size_t size, uintptr_t caller)
|
587
|
-
{
|
588
|
-
struct src_loc *l;
|
589
|
-
struct alloc_hdr *h;
|
590
|
-
void *real;
|
591
|
-
size_t asize;
|
592
|
-
size_t d = alignment / sizeof(void*);
|
593
|
-
size_t r = alignment % sizeof(void*);
|
594
|
-
|
595
|
-
if (!real_malloc) return ENOMEM;
|
596
|
-
|
597
|
-
if (r != 0 || d == 0 || !is_power_of_two(d))
|
598
|
-
return EINVAL;
|
599
|
-
|
600
|
-
if (alignment <= ASSUMED_MALLOC_ALIGNMENT) {
|
601
|
-
void *p = malloc(size);
|
602
|
-
if (!p) return ENOMEM;
|
603
|
-
*pp = p;
|
604
|
-
return 0;
|
605
|
-
}
|
606
|
-
for (; alignment < sizeof(struct alloc_hdr); alignment *= 2)
|
607
|
-
; /* double alignment until >= sizeof(struct alloc_hdr) */
|
608
|
-
if (__builtin_add_overflow(size, alignment, &asize) ||
|
609
|
-
__builtin_add_overflow(asize, sizeof(struct alloc_hdr), &asize))
|
610
|
-
return ENOMEM;
|
611
|
-
|
612
|
-
|
613
|
-
if (alignment == HEAP_PAGE_ALIGN && size == HEAP_PAGE_SIZE) {
|
614
|
-
if (has_ec_p()) generation = rb_gc_count();
|
615
|
-
l = IS_HEAP_PAGE_BODY;
|
616
|
-
} else {
|
617
|
-
l = update_stats_rcu_lock(size, caller);
|
618
|
-
}
|
619
|
-
|
620
|
-
if (l == IS_HEAP_PAGE_BODY) {
|
621
|
-
void *p;
|
622
|
-
size_t gen = generation;
|
623
|
-
|
624
|
-
mutex_lock(&hpb_stats.lock);
|
625
|
-
|
626
|
-
/* reuse existing entry */
|
627
|
-
if (!cds_list_empty(&hpb_stats.freed)) {
|
628
|
-
size_t deathspan;
|
629
|
-
|
630
|
-
h = cds_list_first_entry(&hpb_stats.freed,
|
631
|
-
struct alloc_hdr, anode);
|
632
|
-
/* hpb_stats.freed => hpb_stats.bodies */
|
633
|
-
cds_list_move(&h->anode, &hpb_stats.bodies);
|
634
|
-
assert(h->size == size);
|
635
|
-
assert(h->real);
|
636
|
-
real = h->real;
|
637
|
-
p = hdr2ptr(h);
|
638
|
-
assert(ptr_is_aligned(p, alignment));
|
639
|
-
|
640
|
-
deathspan = gen - h->as.hpb_freed.at;
|
641
|
-
acc_add(&hpb_stats.reborn, deathspan);
|
642
|
-
}
|
643
|
-
else {
|
644
|
-
real = real_malloc(asize);
|
645
|
-
if (!real) return ENOMEM;
|
646
|
-
|
647
|
-
p = hdr2ptr(real);
|
648
|
-
if (!ptr_is_aligned(p, alignment))
|
649
|
-
p = ptr_align(p, alignment);
|
650
|
-
h = ptr2hdr(p);
|
651
|
-
h->size = size;
|
652
|
-
h->real = real;
|
653
|
-
cds_list_add(&h->anode, &hpb_stats.bodies);
|
654
|
-
}
|
655
|
-
mutex_unlock(&hpb_stats.lock);
|
656
|
-
h->as.live.loc = l;
|
657
|
-
h->as.live.gen = gen;
|
658
|
-
*pp = p;
|
659
|
-
}
|
660
|
-
else {
|
661
|
-
real = real_malloc(asize);
|
662
|
-
if (real) {
|
663
|
-
void *p = hdr2ptr(real);
|
664
|
-
if (!ptr_is_aligned(p, alignment))
|
665
|
-
p = ptr_align(p, alignment);
|
666
|
-
h = ptr2hdr(p);
|
667
|
-
alloc_insert_rcu(l, h, size, real);
|
668
|
-
*pp = p;
|
669
|
-
}
|
670
|
-
update_stats_rcu_unlock(l);
|
671
|
-
}
|
672
|
-
|
673
|
-
return real ? 0 : ENOMEM;
|
674
|
-
}
|
675
|
-
|
676
|
-
static void *
|
677
|
-
memalign_result(int err, void *p)
|
678
|
-
{
|
679
|
-
if (caa_unlikely(err))
|
680
|
-
errno = err;
|
681
|
-
return p;
|
682
|
-
}
|
683
|
-
|
684
|
-
void *memalign(size_t alignment, size_t size)
|
685
|
-
{
|
686
|
-
void *p = NULL;
|
687
|
-
int err = internal_memalign(&p, alignment, size, RETURN_ADDRESS(0));
|
688
|
-
return memalign_result(err, p);
|
689
|
-
}
|
690
|
-
|
691
|
-
int posix_memalign(void **p, size_t alignment, size_t size)
|
692
|
-
{
|
693
|
-
return internal_memalign(p, alignment, size, RETURN_ADDRESS(0));
|
694
|
-
}
|
695
|
-
|
696
|
-
void *aligned_alloc(size_t, size_t) __attribute__((alias("memalign")));
|
697
|
-
void cfree(void *) __attribute__((alias("free")));
|
698
|
-
|
699
|
-
void *valloc(size_t size)
|
700
|
-
{
|
701
|
-
void *p = NULL;
|
702
|
-
int err = internal_memalign(&p, page_size, size, RETURN_ADDRESS(0));
|
703
|
-
return memalign_result(err, p);
|
704
|
-
}
|
705
|
-
|
706
|
-
#if __GNUC__ < 7
|
707
|
-
# define add_overflow_p(a,b) __extension__({ \
|
708
|
-
__typeof__(a) _c; \
|
709
|
-
__builtin_add_overflow(a,b,&_c); \
|
710
|
-
})
|
711
|
-
#else
|
712
|
-
# define add_overflow_p(a,b) \
|
713
|
-
__builtin_add_overflow_p((a),(b),(__typeof__(a+b))0)
|
714
|
-
#endif
|
715
|
-
|
716
|
-
void *pvalloc(size_t size)
|
717
|
-
{
|
718
|
-
size_t alignment = page_size;
|
719
|
-
void *p = NULL;
|
720
|
-
int err;
|
721
|
-
|
722
|
-
if (add_overflow_p(size, alignment)) {
|
723
|
-
errno = ENOMEM;
|
724
|
-
return 0;
|
725
|
-
}
|
726
|
-
size = size_align(size, alignment);
|
727
|
-
err = internal_memalign(&p, alignment, size, RETURN_ADDRESS(0));
|
728
|
-
return memalign_result(err, p);
|
729
|
-
}
|
730
|
-
|
731
|
-
void *malloc(size_t size)
|
732
|
-
{
|
733
|
-
struct src_loc *l;
|
734
|
-
struct alloc_hdr *h;
|
735
|
-
size_t asize;
|
736
|
-
void *p;
|
737
|
-
|
738
|
-
if (__builtin_add_overflow(size, sizeof(struct alloc_hdr), &asize))
|
739
|
-
goto enomem;
|
740
|
-
|
741
|
-
/*
|
742
|
-
* Needed for C++ global declarations using "new",
|
743
|
-
* which happens before our constructor
|
744
|
-
*/
|
745
|
-
#ifndef __FreeBSD__
|
746
|
-
if (!real_malloc) {
|
747
|
-
if (resolving_malloc) goto enomem;
|
748
|
-
resolving_malloc = 1;
|
749
|
-
real_malloc = dlsym(RTLD_NEXT, "malloc");
|
750
|
-
}
|
751
|
-
#endif
|
752
|
-
l = update_stats_rcu_lock(size, RETURN_ADDRESS(0));
|
753
|
-
p = h = real_malloc(asize);
|
754
|
-
if (h) {
|
755
|
-
alloc_insert_rcu(l, h, size, h);
|
756
|
-
p = hdr2ptr(h);
|
757
|
-
}
|
758
|
-
update_stats_rcu_unlock(l);
|
759
|
-
if (caa_unlikely(!p)) errno = ENOMEM;
|
760
|
-
return p;
|
761
|
-
enomem:
|
762
|
-
errno = ENOMEM;
|
763
|
-
return 0;
|
764
|
-
}
|
765
|
-
|
766
|
-
void *calloc(size_t nmemb, size_t size)
|
767
|
-
{
|
768
|
-
void *p;
|
769
|
-
struct src_loc *l;
|
770
|
-
struct alloc_hdr *h;
|
771
|
-
size_t asize;
|
772
|
-
|
773
|
-
if (__builtin_mul_overflow(size, nmemb, &size)) {
|
774
|
-
errno = ENOMEM;
|
775
|
-
return 0;
|
776
|
-
}
|
777
|
-
if (__builtin_add_overflow(size, sizeof(struct alloc_hdr), &asize)) {
|
778
|
-
errno = ENOMEM;
|
779
|
-
return 0;
|
780
|
-
}
|
781
|
-
RETURN_IF_NOT_READY();
|
782
|
-
l = update_stats_rcu_lock(size, RETURN_ADDRESS(0));
|
783
|
-
p = h = real_malloc(asize);
|
784
|
-
if (p) {
|
785
|
-
alloc_insert_rcu(l, h, size, h);
|
786
|
-
p = hdr2ptr(h);
|
787
|
-
memset(p, 0, size);
|
788
|
-
}
|
789
|
-
update_stats_rcu_unlock(l);
|
790
|
-
if (caa_unlikely(!p)) errno = ENOMEM;
|
791
|
-
return p;
|
792
|
-
}
|
793
|
-
|
794
|
-
void *realloc(void *ptr, size_t size)
|
795
|
-
{
|
796
|
-
void *p;
|
797
|
-
struct src_loc *l;
|
798
|
-
struct alloc_hdr *h;
|
799
|
-
size_t asize;
|
800
|
-
|
801
|
-
if (!size) {
|
802
|
-
free(ptr);
|
803
|
-
return 0;
|
804
|
-
}
|
805
|
-
if (__builtin_add_overflow(size, sizeof(struct alloc_hdr), &asize)) {
|
806
|
-
errno = ENOMEM;
|
807
|
-
return 0;
|
808
|
-
}
|
809
|
-
RETURN_IF_NOT_READY();
|
810
|
-
|
811
|
-
l = update_stats_rcu_lock(size, RETURN_ADDRESS(0));
|
812
|
-
p = h = real_malloc(asize);
|
813
|
-
if (p) {
|
814
|
-
alloc_insert_rcu(l, h, size, h);
|
815
|
-
p = hdr2ptr(h);
|
816
|
-
}
|
817
|
-
update_stats_rcu_unlock(l);
|
818
|
-
|
819
|
-
if (ptr && p) {
|
820
|
-
struct alloc_hdr *old = ptr2hdr(ptr);
|
821
|
-
memcpy(p, ptr, old->size < size ? old->size : size);
|
822
|
-
free(ptr);
|
823
|
-
}
|
824
|
-
if (caa_unlikely(!p)) errno = ENOMEM;
|
825
|
-
return p;
|
826
|
-
}
|
827
|
-
|
828
|
-
struct dump_arg {
|
829
|
-
FILE *fp;
|
830
|
-
size_t min;
|
831
|
-
};
|
832
|
-
|
833
|
-
static void *dump_to_file(void *x)
|
834
|
-
{
|
835
|
-
struct dump_arg *a = x;
|
836
|
-
struct cds_lfht_iter iter;
|
837
|
-
struct src_loc *l;
|
838
|
-
struct cds_lfht *t;
|
839
|
-
|
840
|
-
++locating;
|
841
|
-
rcu_read_lock();
|
842
|
-
t = CMM_LOAD_SHARED(totals);
|
843
|
-
if (!t)
|
844
|
-
goto out_unlock;
|
845
|
-
cds_lfht_for_each_entry(t, &iter, l, hnode) {
|
846
|
-
const void *p = l->k;
|
847
|
-
char **s = 0;
|
848
|
-
if (l->total <= a->min) continue;
|
849
|
-
|
850
|
-
if (loc_is_addr(l)) {
|
851
|
-
s = backtrace_symbols(p, 1);
|
852
|
-
p = s[0];
|
853
|
-
}
|
854
|
-
fprintf(a->fp, "%16zu %12zu %s\n",
|
855
|
-
l->total, l->allocations, (const char *)p);
|
856
|
-
if (s) free(s);
|
857
|
-
}
|
858
|
-
out_unlock:
|
859
|
-
rcu_read_unlock();
|
860
|
-
--locating;
|
861
|
-
return 0;
|
862
|
-
}
|
863
13
|
|
864
14
|
/*
|
865
15
|
* call-seq:
|
@@ -874,7 +24,7 @@ out_unlock:
|
|
874
24
|
*
|
875
25
|
* total_size call_count location
|
876
26
|
*/
|
877
|
-
static VALUE mwrap_dump(int argc, VALUE *
|
27
|
+
static VALUE mwrap_dump(int argc, VALUE *argv, VALUE mod)
|
878
28
|
{
|
879
29
|
VALUE io, min;
|
880
30
|
struct dump_arg a;
|
@@ -884,7 +34,7 @@ static VALUE mwrap_dump(int argc, VALUE * argv, VALUE mod)
|
|
884
34
|
|
885
35
|
if (NIL_P(io))
|
886
36
|
/* library may be linked w/o Ruby */
|
887
|
-
io =
|
37
|
+
io = rb_stderr;
|
888
38
|
|
889
39
|
a.min = NIL_P(min) ? 0 : NUM2SIZET(min);
|
890
40
|
io = rb_io_get_io(io);
|
@@ -892,7 +42,7 @@ static VALUE mwrap_dump(int argc, VALUE * argv, VALUE mod)
|
|
892
42
|
GetOpenFile(io, fptr);
|
893
43
|
a.fp = rb_io_stdio_file(fptr);
|
894
44
|
|
895
|
-
rb_thread_call_without_gvl(dump_to_file, &a, 0, 0);
|
45
|
+
rb_thread_call_without_gvl((void *(*)(void *))dump_to_file, &a, 0, 0);
|
896
46
|
RB_GC_GUARD(io);
|
897
47
|
return Qnil;
|
898
48
|
}
|
@@ -900,24 +50,8 @@ static VALUE mwrap_dump(int argc, VALUE * argv, VALUE mod)
|
|
900
50
|
/* The whole operation is not remotely atomic... */
|
901
51
|
static void *totals_reset(void *ign)
|
902
52
|
{
|
903
|
-
|
904
|
-
|
905
|
-
struct src_loc *l;
|
906
|
-
|
907
|
-
uatomic_set(&total_bytes_inc, 0);
|
908
|
-
uatomic_set(&total_bytes_dec, 0);
|
909
|
-
|
910
|
-
rcu_read_lock();
|
911
|
-
t = CMM_LOAD_SHARED(totals);
|
912
|
-
cds_lfht_for_each_entry(t, &iter, l, hnode) {
|
913
|
-
uatomic_set(&l->total, 0);
|
914
|
-
uatomic_set(&l->allocations, 0);
|
915
|
-
uatomic_set(&l->frees, 0);
|
916
|
-
uatomic_set(&l->age_total, 0);
|
917
|
-
uatomic_set(&l->max_lifespan, 0);
|
918
|
-
}
|
919
|
-
rcu_read_unlock();
|
920
|
-
return 0;
|
53
|
+
mwrap_reset();
|
54
|
+
return NULL;
|
921
55
|
}
|
922
56
|
|
923
57
|
/*
|
@@ -929,18 +63,12 @@ static void *totals_reset(void *ign)
|
|
929
63
|
* This resets all statistics. This is not an atomic operation
|
930
64
|
* as other threads (outside of GVL) may increment counters.
|
931
65
|
*/
|
932
|
-
static VALUE
|
66
|
+
static VALUE reset_m(VALUE mod)
|
933
67
|
{
|
934
68
|
rb_thread_call_without_gvl(totals_reset, 0, 0, 0);
|
935
69
|
return Qnil;
|
936
70
|
}
|
937
71
|
|
938
|
-
/* :nodoc: */
|
939
|
-
static VALUE mwrap_clear(VALUE mod)
|
940
|
-
{
|
941
|
-
return mwrap_reset(mod);
|
942
|
-
}
|
943
|
-
|
944
72
|
static VALUE rcu_unlock_ensure(VALUE ignored)
|
945
73
|
{
|
946
74
|
rcu_read_unlock();
|
@@ -948,21 +76,31 @@ static VALUE rcu_unlock_ensure(VALUE ignored)
|
|
948
76
|
return Qfalse;
|
949
77
|
}
|
950
78
|
|
951
|
-
static VALUE location_string(struct src_loc *l)
|
79
|
+
static VALUE location_string(const struct src_loc *l)
|
952
80
|
{
|
953
|
-
VALUE
|
81
|
+
VALUE tmp = rb_str_new(NULL, 0);
|
954
82
|
|
955
|
-
if (
|
956
|
-
|
957
|
-
|
958
|
-
|
83
|
+
if (l->f) {
|
84
|
+
rb_str_cat(tmp, l->f->fn, l->f->fn_len);
|
85
|
+
if (l->lineno == U24_MAX)
|
86
|
+
rb_str_cat_cstr(tmp, ":-");
|
87
|
+
else
|
88
|
+
rb_str_catf(tmp, ":%u", l->lineno);
|
959
89
|
}
|
960
|
-
|
961
|
-
|
90
|
+
if (l->bt_len) {
|
91
|
+
AUTO_FREE char **s = bt_syms(l->bt, l->bt_len);
|
92
|
+
|
93
|
+
if (s) {
|
94
|
+
if (l->f)
|
95
|
+
rb_str_cat_cstr(tmp, "\n");
|
96
|
+
rb_str_cat_cstr(tmp, s[0]);
|
97
|
+
for (uint32_t i = 1; i < l->bt_len; ++i)
|
98
|
+
rb_str_catf(tmp, "\n%s", s[i]);
|
99
|
+
}
|
962
100
|
}
|
963
101
|
|
964
102
|
/* deduplicate and try to free up some memory */
|
965
|
-
ret = rb_funcall(tmp, id_uminus, 0);
|
103
|
+
VALUE ret = rb_funcall(tmp, id_uminus, 0);
|
966
104
|
if (!OBJ_FROZEN_RAW(tmp))
|
967
105
|
rb_str_resize(tmp, 0);
|
968
106
|
|
@@ -1034,17 +172,6 @@ static const rb_data_type_t src_loc_type = {
|
|
1034
172
|
|
1035
173
|
static VALUE cSrcLoc;
|
1036
174
|
|
1037
|
-
static int
|
1038
|
-
extract_addr(const char *str, size_t len, void **p)
|
1039
|
-
{
|
1040
|
-
const char *c;
|
1041
|
-
#if defined(__GLIBC__)
|
1042
|
-
return ((c = memrchr(str, '[', len)) && sscanf(c, "[%p]", p));
|
1043
|
-
#else /* tested FreeBSD */
|
1044
|
-
return ((c = strstr(str, "0x")) && sscanf(c, "%p", p));
|
1045
|
-
#endif
|
1046
|
-
}
|
1047
|
-
|
1048
175
|
/*
|
1049
176
|
* call-seq:
|
1050
177
|
* Mwrap[location] -> Mwrap::SourceLocation
|
@@ -1057,41 +184,11 @@ extract_addr(const char *str, size_t len, void **p)
|
|
1057
184
|
static VALUE mwrap_aref(VALUE mod, VALUE loc)
|
1058
185
|
{
|
1059
186
|
const char *str = StringValueCStr(loc);
|
1060
|
-
|
1061
|
-
|
1062
|
-
|
1063
|
-
struct cds_lfht_iter iter;
|
1064
|
-
struct cds_lfht_node *cur;
|
1065
|
-
struct cds_lfht *t;
|
1066
|
-
struct src_loc *l;
|
1067
|
-
VALUE val = Qnil;
|
1068
|
-
|
1069
|
-
if (extract_addr(str, len, (void **)&p)) {
|
1070
|
-
k = (void *)kbuf;
|
1071
|
-
memcpy(k->k, &p, sizeof(p));
|
1072
|
-
k->capa = 0;
|
1073
|
-
k->hval = jhash(k->k, sizeof(p), 0xdeadbeef);
|
1074
|
-
} else {
|
1075
|
-
k = (void *)kbuf;
|
1076
|
-
memcpy(k->k, str, len + 1);
|
1077
|
-
k->capa = len + 1;
|
1078
|
-
k->hval = jhash(k->k, k->capa, 0xdeadbeef);
|
1079
|
-
}
|
187
|
+
long len = RSTRING_LEN(loc);
|
188
|
+
assert(len >= 0);
|
189
|
+
struct src_loc *l = mwrap_get(str, (size_t)len);
|
1080
190
|
|
1081
|
-
|
1082
|
-
|
1083
|
-
t = CMM_LOAD_SHARED(totals);
|
1084
|
-
if (!t) return val;
|
1085
|
-
rcu_read_lock();
|
1086
|
-
|
1087
|
-
cds_lfht_lookup(t, k->hval, loc_eq, k, &iter);
|
1088
|
-
cur = cds_lfht_iter_get_node(&iter);
|
1089
|
-
if (cur) {
|
1090
|
-
l = caa_container_of(cur, struct src_loc, hnode);
|
1091
|
-
val = TypedData_Wrap_Struct(cSrcLoc, &src_loc_type, l);
|
1092
|
-
}
|
1093
|
-
rcu_read_unlock();
|
1094
|
-
return val;
|
191
|
+
return l ? TypedData_Wrap_Struct(cSrcLoc, &src_loc_type, l) : Qnil;
|
1095
192
|
}
|
1096
193
|
|
1097
194
|
static VALUE src_loc_each_i(VALUE p)
|
@@ -1115,7 +212,7 @@ static VALUE src_loc_each_i(VALUE p)
|
|
1115
212
|
return Qfalse;
|
1116
213
|
}
|
1117
214
|
|
1118
|
-
static struct src_loc *
|
215
|
+
static struct src_loc *src_loc_of(VALUE self)
|
1119
216
|
{
|
1120
217
|
struct src_loc *l;
|
1121
218
|
TypedData_Get_Struct(self, struct src_loc, &src_loc_type, l);
|
@@ -1137,7 +234,7 @@ static struct src_loc *src_loc_get(VALUE self)
|
|
1137
234
|
*/
|
1138
235
|
static VALUE src_loc_each(VALUE self)
|
1139
236
|
{
|
1140
|
-
struct src_loc *l =
|
237
|
+
struct src_loc *l = src_loc_of(self);
|
1141
238
|
|
1142
239
|
assert(locating == 0 && "forgot to clear locating");
|
1143
240
|
++locating;
|
@@ -1152,7 +249,7 @@ static VALUE src_loc_each(VALUE self)
|
|
1152
249
|
*/
|
1153
250
|
static VALUE src_loc_mean_lifespan(VALUE self)
|
1154
251
|
{
|
1155
|
-
struct src_loc *l =
|
252
|
+
struct src_loc *l = src_loc_of(self);
|
1156
253
|
size_t tot, frees;
|
1157
254
|
|
1158
255
|
frees = uatomic_read(&l->frees);
|
@@ -1163,19 +260,19 @@ static VALUE src_loc_mean_lifespan(VALUE self)
|
|
1163
260
|
/* The number of frees made from this location */
|
1164
261
|
static VALUE src_loc_frees(VALUE self)
|
1165
262
|
{
|
1166
|
-
return SIZET2NUM(uatomic_read(&
|
263
|
+
return SIZET2NUM(uatomic_read(&src_loc_of(self)->frees));
|
1167
264
|
}
|
1168
265
|
|
1169
266
|
/* The number of allocations made from this location */
|
1170
267
|
static VALUE src_loc_allocations(VALUE self)
|
1171
268
|
{
|
1172
|
-
return SIZET2NUM(uatomic_read(&
|
269
|
+
return SIZET2NUM(uatomic_read(&src_loc_of(self)->allocations));
|
1173
270
|
}
|
1174
271
|
|
1175
272
|
/* The total number of bytes allocated from this location */
|
1176
273
|
static VALUE src_loc_total(VALUE self)
|
1177
274
|
{
|
1178
|
-
return SIZET2NUM(uatomic_read(&
|
275
|
+
return SIZET2NUM(uatomic_read(&src_loc_of(self)->total));
|
1179
276
|
}
|
1180
277
|
|
1181
278
|
/*
|
@@ -1184,7 +281,7 @@ static VALUE src_loc_total(VALUE self)
|
|
1184
281
|
*/
|
1185
282
|
static VALUE src_loc_max_lifespan(VALUE self)
|
1186
283
|
{
|
1187
|
-
return SIZET2NUM(uatomic_read(&
|
284
|
+
return SIZET2NUM(uatomic_read(&src_loc_of(self)->max_lifespan));
|
1188
285
|
}
|
1189
286
|
|
1190
287
|
/*
|
@@ -1192,7 +289,7 @@ static VALUE src_loc_max_lifespan(VALUE self)
|
|
1192
289
|
*/
|
1193
290
|
static VALUE src_loc_name(VALUE self)
|
1194
291
|
{
|
1195
|
-
struct src_loc *l =
|
292
|
+
struct src_loc *l = src_loc_of(self);
|
1196
293
|
VALUE ret;
|
1197
294
|
|
1198
295
|
++locating;
|
@@ -1239,73 +336,6 @@ static VALUE total_dec(VALUE mod)
|
|
1239
336
|
return SIZET2NUM(total_bytes_dec);
|
1240
337
|
}
|
1241
338
|
|
1242
|
-
static VALUE hpb_each_yield(VALUE ignore)
|
1243
|
-
{
|
1244
|
-
struct alloc_hdr *h, *next;
|
1245
|
-
|
1246
|
-
cds_list_for_each_entry_safe(h, next, &hpb_stats.bodies, anode) {
|
1247
|
-
VALUE v[2]; /* [ generation, address ] */
|
1248
|
-
void *addr = hdr2ptr(h);
|
1249
|
-
assert(ptr_is_aligned(addr, HEAP_PAGE_ALIGN));
|
1250
|
-
v[0] = LONG2NUM((long)addr);
|
1251
|
-
v[1] = SIZET2NUM(h->as.live.gen);
|
1252
|
-
rb_yield_values2(2, v);
|
1253
|
-
}
|
1254
|
-
return Qnil;
|
1255
|
-
}
|
1256
|
-
|
1257
|
-
/*
|
1258
|
-
* call-seq:
|
1259
|
-
*
|
1260
|
-
* Mwrap::HeapPageBody.each { |gen, addr| } -> Integer
|
1261
|
-
*
|
1262
|
-
* Yields the generation (GC.count) the heap page body was created
|
1263
|
-
* and address of the heap page body as an Integer. Returns the
|
1264
|
-
* number of allocated pages as an Integer. This return value should
|
1265
|
-
* match the result of GC.stat(:heap_allocated_pages)
|
1266
|
-
*/
|
1267
|
-
static VALUE hpb_each(VALUE mod)
|
1268
|
-
{
|
1269
|
-
++locating;
|
1270
|
-
return rb_ensure(hpb_each_yield, Qfalse, reset_locating, 0);
|
1271
|
-
}
|
1272
|
-
|
1273
|
-
/*
|
1274
|
-
* call-seq:
|
1275
|
-
*
|
1276
|
-
* Mwrap::HeapPageBody.stat -> Hash
|
1277
|
-
* Mwrap::HeapPageBody.stat(hash) -> hash
|
1278
|
-
*
|
1279
|
-
* The maximum lifespan of a heap page body in the Ruby VM.
|
1280
|
-
* This may be Infinity if no heap page bodies were ever freed.
|
1281
|
-
*/
|
1282
|
-
static VALUE hpb_stat(int argc, VALUE *argv, VALUE hpb)
|
1283
|
-
{
|
1284
|
-
VALUE h;
|
1285
|
-
|
1286
|
-
rb_scan_args(argc, argv, "01", &h);
|
1287
|
-
if (NIL_P(h))
|
1288
|
-
h = rb_hash_new();
|
1289
|
-
else if (!RB_TYPE_P(h, T_HASH))
|
1290
|
-
rb_raise(rb_eTypeError, "not a hash %+"PRIsVALUE, h);
|
1291
|
-
|
1292
|
-
++locating;
|
1293
|
-
#define S(x) ID2SYM(rb_intern(#x))
|
1294
|
-
rb_hash_aset(h, S(lifespan_max), acc_max(&hpb_stats.alive));
|
1295
|
-
rb_hash_aset(h, S(lifespan_min), acc_min(&hpb_stats.alive));
|
1296
|
-
rb_hash_aset(h, S(lifespan_mean), acc_mean(&hpb_stats.alive));
|
1297
|
-
rb_hash_aset(h, S(lifespan_stddev), acc_stddev(&hpb_stats.alive));
|
1298
|
-
rb_hash_aset(h, S(deathspan_max), acc_max(&hpb_stats.reborn));
|
1299
|
-
rb_hash_aset(h, S(deathspan_min), acc_min(&hpb_stats.reborn));
|
1300
|
-
rb_hash_aset(h, S(deathspan_mean), acc_mean(&hpb_stats.reborn));
|
1301
|
-
rb_hash_aset(h, S(deathspan_stddev), acc_stddev(&hpb_stats.reborn));
|
1302
|
-
rb_hash_aset(h, S(resurrects), SIZET2NUM(hpb_stats.reborn.nr));
|
1303
|
-
#undef S
|
1304
|
-
--locating;
|
1305
|
-
|
1306
|
-
return h;
|
1307
|
-
}
|
1308
|
-
|
1309
339
|
/*
|
1310
340
|
* Document-module: Mwrap
|
1311
341
|
*
|
@@ -1324,19 +354,13 @@ static VALUE hpb_stat(int argc, VALUE *argv, VALUE hpb)
|
|
1324
354
|
* * dump_fd: a writable FD to dump to
|
1325
355
|
* * dump_path: a path to dump to, the file is opened in O_APPEND mode
|
1326
356
|
* * dump_min: the minimum allocation size (total) to dump
|
1327
|
-
* * dump_heap: mask of heap_page_body statistics to dump
|
1328
357
|
*
|
1329
358
|
* If both `dump_fd' and `dump_path' are specified, dump_path takes
|
1330
359
|
* precedence.
|
1331
|
-
*
|
1332
|
-
* dump_heap bitmask
|
1333
|
-
* * 0x01 - summary stats (same info as HeapPageBody.stat)
|
1334
|
-
* * 0x02 - all live heaps (similar to HeapPageBody.each)
|
1335
|
-
* * 0x04 - skip non-heap_page_body-related output
|
1336
360
|
*/
|
1337
361
|
void Init_mwrap(void)
|
1338
362
|
{
|
1339
|
-
VALUE mod
|
363
|
+
VALUE mod;
|
1340
364
|
|
1341
365
|
++locating;
|
1342
366
|
mod = rb_define_module("Mwrap");
|
@@ -1350,9 +374,10 @@ void Init_mwrap(void)
|
|
1350
374
|
* This class is only available since mwrap 2.0.0+.
|
1351
375
|
*/
|
1352
376
|
cSrcLoc = rb_define_class_under(mod, "SourceLocation", rb_cObject);
|
377
|
+
rb_undef_alloc_func(cSrcLoc);
|
1353
378
|
rb_define_singleton_method(mod, "dump", mwrap_dump, -1);
|
1354
|
-
rb_define_singleton_method(mod, "reset",
|
1355
|
-
rb_define_singleton_method(mod, "clear",
|
379
|
+
rb_define_singleton_method(mod, "reset", reset_m, 0);
|
380
|
+
rb_define_singleton_method(mod, "clear", reset_m, 0);
|
1356
381
|
rb_define_singleton_method(mod, "each", mwrap_each, -1);
|
1357
382
|
rb_define_singleton_method(mod, "[]", mwrap_aref, 1);
|
1358
383
|
rb_define_singleton_method(mod, "quiet", mwrap_quiet, 0);
|
@@ -1368,137 +393,5 @@ void Init_mwrap(void)
|
|
1368
393
|
rb_define_method(cSrcLoc, "max_lifespan", src_loc_max_lifespan, 0);
|
1369
394
|
rb_define_method(cSrcLoc, "name", src_loc_name, 0);
|
1370
395
|
|
1371
|
-
/*
|
1372
|
-
* Information about "struct heap_page_body" allocations from
|
1373
|
-
* Ruby gc.c. This can be useful for tracking fragmentation
|
1374
|
-
* from posix_memalign(3) use in mainline Ruby:
|
1375
|
-
*
|
1376
|
-
* https://sourceware.org/bugzilla/show_bug.cgi?id=14581
|
1377
|
-
*
|
1378
|
-
* These statistics are never reset by Mwrap.reset or
|
1379
|
-
* any other method. They only make sense in the context
|
1380
|
-
* of an entire program lifetime.
|
1381
|
-
*/
|
1382
|
-
hpb = rb_define_class_under(mod, "HeapPageBody", rb_cObject);
|
1383
|
-
rb_define_singleton_method(hpb, "stat", hpb_stat, -1);
|
1384
|
-
rb_define_singleton_method(hpb, "each", hpb_each, 0);
|
1385
|
-
|
1386
|
-
--locating;
|
1387
|
-
}
|
1388
|
-
|
1389
|
-
enum {
|
1390
|
-
DUMP_HPB_STATS = 0x1,
|
1391
|
-
DUMP_HPB_EACH = 0x2,
|
1392
|
-
DUMP_HPB_EXCL = 0x4,
|
1393
|
-
};
|
1394
|
-
|
1395
|
-
static void dump_hpb(FILE *fp, unsigned flags)
|
1396
|
-
{
|
1397
|
-
if (flags & DUMP_HPB_STATS) {
|
1398
|
-
fprintf(fp,
|
1399
|
-
"lifespan_max: %"PRId64"\n"
|
1400
|
-
"lifespan_min:%s%"PRId64"\n"
|
1401
|
-
"lifespan_mean: %0.3f\n"
|
1402
|
-
"lifespan_stddev: %0.3f\n"
|
1403
|
-
"deathspan_max: %"PRId64"\n"
|
1404
|
-
"deathspan_min:%s%"PRId64"\n"
|
1405
|
-
"deathspan_mean: %0.3f\n"
|
1406
|
-
"deathspan_stddev: %0.3f\n"
|
1407
|
-
"gc_count: %zu\n",
|
1408
|
-
hpb_stats.alive.max,
|
1409
|
-
hpb_stats.alive.min == INT64_MAX ? " -" : " ",
|
1410
|
-
hpb_stats.alive.min,
|
1411
|
-
hpb_stats.alive.mean,
|
1412
|
-
acc_stddev_dbl(&hpb_stats.alive),
|
1413
|
-
hpb_stats.reborn.max,
|
1414
|
-
hpb_stats.reborn.min == INT64_MAX ? " -" : " ",
|
1415
|
-
hpb_stats.reborn.min,
|
1416
|
-
hpb_stats.reborn.mean,
|
1417
|
-
acc_stddev_dbl(&hpb_stats.reborn),
|
1418
|
-
/* n.b.: unsafe to call rb_gc_count() in destructor */
|
1419
|
-
generation);
|
1420
|
-
}
|
1421
|
-
if (flags & DUMP_HPB_EACH) {
|
1422
|
-
struct alloc_hdr *h;
|
1423
|
-
|
1424
|
-
cds_list_for_each_entry(h, &hpb_stats.bodies, anode) {
|
1425
|
-
void *addr = hdr2ptr(h);
|
1426
|
-
|
1427
|
-
fprintf(fp, "%p\t%zu\n", addr, h->as.live.gen);
|
1428
|
-
}
|
1429
|
-
}
|
1430
|
-
}
|
1431
|
-
|
1432
|
-
/* rb_cloexec_open isn't usable by non-Ruby processes */
|
1433
|
-
#ifndef O_CLOEXEC
|
1434
|
-
# define O_CLOEXEC 0
|
1435
|
-
#endif
|
1436
|
-
|
1437
|
-
__attribute__ ((destructor))
|
1438
|
-
static void mwrap_dump_destructor(void)
|
1439
|
-
{
|
1440
|
-
const char *opt = getenv("MWRAP");
|
1441
|
-
const char *modes[] = { "a", "a+", "w", "w+", "r+" };
|
1442
|
-
struct dump_arg a = { .min = 0 };
|
1443
|
-
size_t i;
|
1444
|
-
int dump_fd;
|
1445
|
-
unsigned dump_heap = 0;
|
1446
|
-
char *dump_path;
|
1447
|
-
char *s;
|
1448
|
-
|
1449
|
-
if (!opt)
|
1450
|
-
return;
|
1451
|
-
|
1452
|
-
++locating;
|
1453
|
-
if ((dump_path = strstr(opt, "dump_path:")) &&
|
1454
|
-
(dump_path += sizeof("dump_path")) &&
|
1455
|
-
*dump_path) {
|
1456
|
-
char *end = strchr(dump_path, ',');
|
1457
|
-
if (end) {
|
1458
|
-
char *tmp = alloca(end - dump_path + 1);
|
1459
|
-
end = mempcpy(tmp, dump_path, end - dump_path);
|
1460
|
-
*end = 0;
|
1461
|
-
dump_path = tmp;
|
1462
|
-
}
|
1463
|
-
dump_fd = open(dump_path, O_CLOEXEC|O_WRONLY|O_APPEND|O_CREAT,
|
1464
|
-
0666);
|
1465
|
-
if (dump_fd < 0) {
|
1466
|
-
fprintf(stderr, "open %s failed: %s\n", dump_path,
|
1467
|
-
strerror(errno));
|
1468
|
-
goto out;
|
1469
|
-
}
|
1470
|
-
}
|
1471
|
-
else if (!sscanf(opt, "dump_fd:%d", &dump_fd))
|
1472
|
-
goto out;
|
1473
|
-
|
1474
|
-
if ((s = strstr(opt, "dump_min:")))
|
1475
|
-
sscanf(s, "dump_min:%zu", &a.min);
|
1476
|
-
|
1477
|
-
if ((s = strstr(opt, "dump_heap:")))
|
1478
|
-
sscanf(s, "dump_heap:%u", &dump_heap);
|
1479
|
-
|
1480
|
-
switch (dump_fd) {
|
1481
|
-
case 0: goto out;
|
1482
|
-
case 1: a.fp = stdout; break;
|
1483
|
-
case 2: a.fp = stderr; break;
|
1484
|
-
default:
|
1485
|
-
if (dump_fd < 0)
|
1486
|
-
goto out;
|
1487
|
-
a.fp = 0;
|
1488
|
-
|
1489
|
-
for (i = 0; !a.fp && i < 5; i++)
|
1490
|
-
a.fp = fdopen(dump_fd, modes[i]);
|
1491
|
-
|
1492
|
-
if (!a.fp) {
|
1493
|
-
fprintf(stderr, "failed to open fd=%d: %s\n",
|
1494
|
-
dump_fd, strerror(errno));
|
1495
|
-
goto out;
|
1496
|
-
}
|
1497
|
-
/* we'll leak some memory here, but this is a destructor */
|
1498
|
-
}
|
1499
|
-
if ((dump_heap & DUMP_HPB_EXCL) == 0)
|
1500
|
-
dump_to_file(&a);
|
1501
|
-
dump_hpb(a.fp, dump_heap);
|
1502
|
-
out:
|
1503
396
|
--locating;
|
1504
397
|
}
|