mwrap 1.0.0 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '049853543dbde8f93776318db059ac2d40ee82a241d080a8236a7d43efade454'
-  data.tar.gz: a29b74fef93affeb9ffa30276fe9f75f9f5952cbfae142a749ba1b9409153789
+  metadata.gz: 8559afec1946b7f545c944085aac5e205601deb82fa9f1529785dd3ef7526e5a
+  data.tar.gz: 34ea90410103ec59f367baa00e1e4df6424fe67af8608b5c7a3a0cb66fec2440
 SHA512:
-  metadata.gz: 52bad7de38eb6b256b1cecbb9a7a83da094602adf4162730fed0d58ff3fa3f605fcef77e724b3a3e8557bbbce597a9369d3673f2be2d176ddc841dc2cc3c2208
-  data.tar.gz: 2b446a6a8ceef8cc5b7a34211fc2d7cd07e50692bcca6fe4a13d115b359b8fde7042ae1a6f7c53829f2b65fb9fd4158492e1dd142633a30c672eccb6120a1d71
+  metadata.gz: ff91f6f250e7a18cb465a6dc04ddee374411a10a92cd1b3227e09a4ccd0be1b8597f475f71d5061bd5bc36e8c0418ce3bf6faa3a78b37aec5d21b0340f736031
+  data.tar.gz: 06311f2949ae4dae6bd2d0e95e6b0d147b44b6bae6cd4464e4c1d160fc1f5f3f99b2a6a3a3e711a10b8894da3bdb6220ccfeb72dfb18545170edbbc65dbad225

data/.document

@@ -0,0 +1,2 @@
+ext/mwrap/mwrap.c
+lib/mwrap_rack.rb

data/MANIFEST

@@ -8,5 +8,6 @@ bin/mwrap
 ext/mwrap/extconf.rb
 ext/mwrap/jhash.h
 ext/mwrap/mwrap.c
+lib/mwrap_rack.rb
 mwrap.gemspec
 test/test_mwrap.rb

data/README

@@ -4,20 +4,23 @@ mwrap is designed to answer the question:
 
    Which lines of Ruby are hitting malloc the most?
 
-mwrap wraps all malloc, calloc, and realloc calls to trace the Ruby
-source location of such calls and bytes allocated at each callsite.
-This functionality may be expanded in the future.
-
-It does not track allocation lifetimes, or frees, however.  It works
-best for allocations under GVL, but tries to track numeric caller
-addresses for allocations made without GVL so you can get an idea of how
-much memory usage certain extensions and native libraries use.
+mwrap wraps all malloc-family calls to trace the Ruby source
+location of such calls and bytes allocated at each callsite.
+As of mwrap 2.0.0, it can also function as a leak detector
+and show live allocations at every call site.  Depending on
+your application and workload, the overhead is roughly a 50%
+increase memory and runtime.
+
+It works best for allocations under GVL, but tries to track
+numeric caller addresses for allocations made without GVL so you
+can get an idea of how much memory usage certain extensions and
+native libraries use.
 
 It requires the concurrent lock-free hash table from the
 Userspace RCU project: https://liburcu.org/
 
-It does not require recompiling or rebuilding Ruby, but only supports
-Ruby trunk (2.6.0dev+) on a few platforms:
+It does not require recompiling or rebuilding Ruby, but only
+supports Ruby trunk (2.6.0dev+) on a few platforms:
 
 * GNU/Linux
 * FreeBSD (tested 11.1)
@@ -36,28 +39,35 @@ It may work on NetBSD, OpenBSD and DragonFly BSD.
 == Usage
 
 mwrap works as an LD_PRELOAD and supplies a mwrap RubyGem executable to
-improve ease-of-use.  You can set dump_fd: in the MWRAP environment
-variable to dump the results to a certain file descriptor at exit:
-
-	# Dump results to stderr at exit:
-	MWRAP=dump_fd:2 mwrap RUBY_COMMAND
-
-You may also set dump_path to append to a log file:
+improve ease-of-use.  You can set dump_path: in the MWRAP environment
+variable to append the results to a log file:
 
 	MWRAP=dump_path:/path/to/log mwrap RUBY_COMMAND
 
-You may also `require 'mwrap'' in your Ruby code and use
-Mwrap.dump, Mwrap.clear, Mwrap.reset, Mwrap.each, etc.
+	# And to display the locations with the most allocations:
+	sort -k1,1rn </path/to/log | $PAGER
+
+You may also `require "mwrap"' in your Ruby code and use
+Mwrap.dump, Mwrap.reset, Mwrap.each, etc.
 
 However, mwrap MUST be loaded via LD_PRELOAD to have any
-effect in tracking malloc use.
+effect in tracking malloc use.  However, it is safe to keep
+"require 'mwrap'" in performance-critical deployments,
+as overhead is only incurred when used as an LD_PRELOAD.
 
 The output of the mwrap dump is a text file with 3 columns:
 
 	total_bytes	call_count	location
 
 Where location is a Ruby source location (if made under GVL)
-or an address retrieved by backtrace_symbols(3)
+or an address retrieved by backtrace_symbols(3).  It is
+recommended to use the sort(1) command on either of the
+first two columns to find the hottest malloc locations.
+
+mwrap 2.0.0+ also supports a Rack application endpoint,
+it is documented at:
+
+	https://80x24.org/mwrap/MwrapRack.html
 
 == Known problems
 

data/bin/mwrap

@@ -26,4 +26,11 @@ if ENV['MWRAP'] =~ /dump_fd:(\d+)/
     opts[dump_fd] = dump_io
   end
 end
+
+# allow inheriting FDs from systemd
+n = ENV['LISTEN_FDS']
+if n && ENV['LISTEN_PID'].to_i == $$
+  n = 3 + n.to_i
+  (3...n).each { |fd| opts[fd] = IO.new(fd) }
+end
 exec *ARGV, opts

data/ext/mwrap/extconf.rb

@@ -10,4 +10,19 @@ have_library 'urcu-bp' or abort 'liburcu-bp not found'
 have_library 'dl'
 have_library 'c'
 have_library 'execinfo' # FreeBSD
+
+if try_link(<<'')
+int main(void) { return __builtin_add_overflow_p(0,0,(int)1); }
+
+  $defs << '-DHAVE_BUILTIN_ADD_OVERFLOW_P'
+end
+
+if try_link(<<'')
+int main(int a) { return __builtin_add_overflow(0,0,&a); }
+
+  $defs << '-DHAVE_BUILTIN_ADD_OVERFLOW_P'
+else
+  abort 'missing __builtin_add_overflow'
+end
+
 create_makefile 'mwrap'

data/ext/mwrap/mwrap.c

@@ -16,26 +16,40 @@
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <fcntl.h>
+#include <pthread.h>
 #include <urcu-bp.h>
 #include <urcu/rculfhash.h>
+#include <urcu/rculist.h>
 #include "jhash.h"
 
 static ID id_uminus;
+static unsigned int track_memalign;
 const char *rb_source_location_cstr(int *line); /* requires 2.6.0dev */
-static int *(*has_gvl_p)(void);
+extern int __attribute__((weak)) ruby_thread_has_gvl_p(void);
+extern void * __attribute__((weak)) ruby_current_execution_context_ptr;
+extern void * __attribute__((weak)) ruby_current_vm_ptr; /* for rb_gc_count */
+extern size_t __attribute__((weak)) rb_gc_count(void);
+extern VALUE __attribute__((weak)) rb_cObject;
+extern VALUE __attribute__((weak)) rb_yield(VALUE);
+
+/* true for glibc/dlmalloc/ptmalloc, not sure about jemalloc */
+#define ASSUMED_MALLOC_ALIGNMENT (sizeof(void *) * 2)
+
+int __attribute__((weak)) ruby_thread_has_gvl_p(void)
+{
+	return 0;
+}
+
 #ifdef __FreeBSD__
 void *__malloc(size_t);
-void *__calloc(size_t, size_t);
-void *__realloc(void *, size_t);
-static void *(*real_malloc)(size_t) = __malloc;
-static void *(*real_calloc)(size_t, size_t) = __calloc;
-static void *(*real_realloc)(void *, size_t) = __realloc;
-#  define RETURN_IF_NOT_READY() do {} while (0) /* nothing */
+void __free(void *);
+#  define real_malloc __malloc
+#  define real_free __free
 #else
-static int ready;
 static void *(*real_malloc)(size_t);
-static void *(*real_calloc)(size_t, size_t);
-static void *(*real_realloc)(void *, size_t);
+static void (*real_free)(void *);
+static int resolving_malloc;
+#endif /* !FreeBSD */
 
 /*
  * we need to fake an OOM condition while dlsym is running,
@@ -43,22 +57,33 @@ static void *(*real_realloc)(void *, size_t);
  * symbol for the jemalloc calloc, yet
  */
 #  define RETURN_IF_NOT_READY() do { \
-	if (!ready) { \
+	if (!real_malloc) { \
 		errno = ENOMEM; \
 		return NULL; \
 	} \
 } while (0)
 
-#endif /* !FreeBSD */
+static __thread size_t locating;
+static size_t generation;
+static size_t page_size;
+static struct cds_lfht *totals;
+union padded_mutex {
+	pthread_mutex_t mtx;
+	char pad[64];
+};
 
-/*
- * rb_source_location_cstr relies on GET_EC(), and it's possible
- * to have a native thread but no EC during the early and late
- * (teardown) phases of the Ruby process
- */
-static void **ec_loc;
+/* a round-robin pool of mutexes */
+#define MUTEX_NR   (1 << 6)
+#define MUTEX_MASK (MUTEX_NR - 1)
+static size_t mutex_i;
+static union padded_mutex mutexes[MUTEX_NR] = {
+	[0 ... (MUTEX_NR-1)].mtx = PTHREAD_MUTEX_INITIALIZER
+};
 
-static struct cds_lfht *totals;
+static pthread_mutex_t *mutex_assign(void)
+{
+	return &mutexes[uatomic_add_return(&mutex_i, 1) & MUTEX_MASK].mtx;
+}
 
 static struct cds_lfht *
 lfht_new(void)
@@ -69,19 +94,40 @@ lfht_new(void)
 __attribute__((constructor)) static void resolve_malloc(void)
 {
 	int err;
+	const char *opt;
+	++locating;
 
-#ifndef __FreeBSD__
-	real_malloc = dlsym(RTLD_NEXT, "malloc");
-	real_calloc = dlsym(RTLD_NEXT, "calloc");
-	real_realloc = dlsym(RTLD_NEXT, "realloc");
-	if (!real_calloc || !real_malloc || !real_realloc) {
-		fprintf(stderr, "missing calloc/malloc/realloc %p %p %p\n",
-			real_calloc, real_malloc, real_realloc);
+#ifdef __FreeBSD__
+	/*
+	 * PTHREAD_MUTEX_INITIALIZER on FreeBSD means lazy initialization,
+	 * which happens at pthread_mutex_lock, and that calls calloc
+	 */
+	{
+		size_t i;
+
+		for (i = 0; i < MUTEX_NR; i++) {
+			err = pthread_mutex_init(&mutexes[i].mtx, 0);
+			if (err) {
+				fprintf(stderr, "error: %s\n", strerror(err));
+				_exit(1);
+			}
+		}
+		/* initialize mutexes used by urcu-bp */
+		rcu_read_lock();
+		rcu_read_unlock();
+	}
+#else /* !FreeBSD (tested on GNU/Linux) */
+	if (!real_malloc) {
+		resolving_malloc = 1;
+		real_malloc = dlsym(RTLD_NEXT, "malloc");
+	}
+	real_free = dlsym(RTLD_NEXT, "free");
+	if (!real_malloc || !real_free) {
+		fprintf(stderr, "missing malloc/aligned_alloc/free\n"
+			"\t%p %p\n", real_malloc, real_free);
 		_exit(1);
 	}
-	ready = 1;
-#endif
-
+#endif /* !FreeBSD */
 	totals = lfht_new();
 	if (!totals)
 		fprintf(stderr, "failed to allocate totals table\n");
@@ -91,14 +137,27 @@ __attribute__((constructor)) static void resolve_malloc(void)
 				call_rcu_after_fork_child);
 	if (err)
 		fprintf(stderr, "pthread_atfork failed: %s\n", strerror(err));
+	page_size = sysconf(_SC_PAGESIZE);
+	opt = getenv("MWRAP");
+	if (opt && (opt = strstr(opt, "memalign:"))) {
+		if (!sscanf(opt, "memalign:%u", &track_memalign))
+			fprintf(stderr, "not an unsigned int: %s\n", opt);
+	}
+	--locating;
+}
 
-	has_gvl_p = dlsym(RTLD_DEFAULT, "ruby_thread_has_gvl_p");
+static void
+mutex_lock(pthread_mutex_t *m)
+{
+	int err = pthread_mutex_lock(m);
+	assert(err == 0);
+}
 
-	/*
-	 * resolve dynamically so it doesn't break when LD_PRELOAD-ed
-	 * into non-Ruby binaries
-	 */
-	ec_loc = dlsym(RTLD_DEFAULT, "ruby_current_execution_context_ptr");
+static void
+mutex_unlock(pthread_mutex_t *m)
+{
+	int err = pthread_mutex_unlock(m);
+	assert(err == 0);
 }
 
 #ifndef HAVE_MEMPCPY
@@ -114,8 +173,6 @@ my_mempcpy(void *dest, const void *src, size_t n)
 #define RETURN_ADDRESS(nr) \
   (uintptr_t)(__builtin_extract_return_addr(__builtin_return_address(nr)))
 
-static __thread size_t locating;
-
 #define INT2STR_MAX (sizeof(int) == 4 ? 10 : 19)
 static char *int2str(int num, char *dst, size_t * size)
 {
@@ -143,21 +200,58 @@ static char *int2str(int num, char *dst, size_t * size)
 	return NULL;
 }
 
+/*
+ * rb_source_location_cstr relies on GET_EC(), and it's possible
+ * to have a native thread but no EC during the early and late
+ * (teardown) phases of the Ruby process
+ */
 static int has_ec_p(void)
 {
-	return (ec_loc && *ec_loc);
+	return (ruby_thread_has_gvl_p() && ruby_current_vm_ptr &&
+		ruby_current_execution_context_ptr);
 }
 
+/* allocated via real_malloc/real_free */
 struct src_loc {
-	struct rcu_head rcu_head;
-	size_t calls;
+	pthread_mutex_t *mtx;
 	size_t total;
+	size_t allocations;
+	size_t frees;
+	size_t age_total; /* (age_total / frees) => mean age at free */
+	size_t max_lifespan;
 	struct cds_lfht_node hnode;
+	struct cds_list_head allocs; /* <=> alloc_hdr.node */
 	uint32_t hval;
 	uint32_t capa;
 	char k[];
 };
 
+/* every allocation has this in the header, maintain alignment with malloc  */
+struct alloc_hdr {
+	struct cds_list_head anode; /* <=> src_loc.allocs */
+	union {
+		struct {
+			size_t gen; /* rb_gc_count() */
+			struct src_loc *loc;
+		} live;
+		struct rcu_head dead;
+	} as;
+	void *real; /* what to call real_free on */
+	size_t size;
+};
+
+static char kbuf[PATH_MAX + INT2STR_MAX + sizeof(struct alloc_hdr) + 2];
+
+static struct alloc_hdr *ptr2hdr(void *p)
+{
+	return (struct alloc_hdr *)((uintptr_t)p - sizeof(struct alloc_hdr));
+}
+
+static void *hdr2ptr(struct alloc_hdr *h)
+{
+	return (void *)((uintptr_t)h + sizeof(struct alloc_hdr));
+}
+
 static int loc_is_addr(const struct src_loc *l)
 {
 	return l->capa == 0;
@@ -180,16 +274,14 @@ static int loc_eq(struct cds_lfht_node *node, const void *key)
 		memcmp(k->k, existing->k, loc_size(k)) == 0);
 }
 
-static void totals_add(struct src_loc *k)
+static struct src_loc *totals_add_rcu(struct src_loc *k)
 {
 	struct cds_lfht_iter iter;
 	struct cds_lfht_node *cur;
-	struct src_loc *l;
+	struct src_loc *l = 0;
 	struct cds_lfht *t;
 
-
 again:
-	rcu_read_lock();
 	t = rcu_dereference(totals);
 	if (!t) goto out_unlock;
 	cds_lfht_lookup(t, k->hval, loc_eq, k, &iter);
@@ -197,44 +289,58 @@ again:
 	if (cur) {
 		l = caa_container_of(cur, struct src_loc, hnode);
 		uatomic_add(&l->total, k->total);
-		uatomic_add(&l->calls, 1);
+		uatomic_add(&l->allocations, 1);
 	} else {
 		size_t n = loc_size(k);
-		l = malloc(sizeof(*l) + n);
+		l = real_malloc(sizeof(*l) + n);
 		if (!l) goto out_unlock;
-
 		memcpy(l, k, sizeof(*l) + n);
-		l->calls = 1;
+		l->mtx = mutex_assign();
+		l->age_total = 0;
+		l->max_lifespan = 0;
+		l->frees = 0;
+		l->allocations = 1;
+		CDS_INIT_LIST_HEAD(&l->allocs);
 		cur = cds_lfht_add_unique(t, k->hval, loc_eq, l, &l->hnode);
 		if (cur != &l->hnode) { /* lost race */
 			rcu_read_unlock();
-			free(l);
+			real_free(l);
+			rcu_read_lock();
 			goto again;
 		}
 	}
 out_unlock:
-	rcu_read_unlock();
+	return l;
+}
+
+static void update_stats_rcu_unlock(const struct src_loc *l)
+{
+	if (caa_likely(l)) rcu_read_unlock();
 }
 
-static void update_stats(size_t size, uintptr_t caller)
+static struct src_loc *update_stats_rcu_lock(size_t size, uintptr_t caller)
 {
-	struct src_loc *k;
+	struct src_loc *k, *ret = 0;
 	static const size_t xlen = sizeof(caller);
 	char *dst;
 
+	if (caa_unlikely(!totals)) return 0;
 	if (locating++) goto out; /* do not recurse into another *alloc */
 
-	if (has_gvl_p && has_gvl_p() && has_ec_p()) {
+	rcu_read_lock();
+	if (has_ec_p()) {
 		int line;
 		const char *ptr = rb_source_location_cstr(&line);
 		size_t len;
 		size_t int_size = INT2STR_MAX;
 
+		generation = rb_gc_count();
+
 		if (!ptr) goto unknown;
 
 		/* avoid vsnprintf or anything which could call malloc here: */
 		len = strlen(ptr);
-		k = alloca(sizeof(*k) + len + 1 + int_size + 1);
+		k = (void *)kbuf;
 		k->total = size;
 		dst = mempcpy(k->k, ptr, len);
 		*dst++ = ':';
@@ -243,7 +349,7 @@ static void update_stats(size_t size, uintptr_t caller)
 			*dst = 0;	/* terminate string */
 			k->capa = (uint32_t)(dst - k->k + 1);
 			k->hval = jhash(k->k, k->capa, 0xdeadbeef);
-			totals_add(k);
+			ret = totals_add_rcu(k);
 		} else {
 			rb_bug("bad math making key from location %s:%d\n",
 				ptr, line);
@@ -255,36 +361,281 @@ unknown:
 		memcpy(k->k, &caller, xlen);
 		k->capa = 0;
 		k->hval = jhash(k->k, xlen, 0xdeadbeef);
-		totals_add(k);
+		ret = totals_add_rcu(k);
 	}
 out:
 	--locating;
+	return ret;
+}
+
+size_t malloc_usable_size(void *p)
+{
+	return ptr2hdr(p)->size;
+}
+
+static void
+free_hdr_rcu(struct rcu_head *dead)
+{
+	struct alloc_hdr *h = caa_container_of(dead, struct alloc_hdr, as.dead);
+	real_free(h->real);
+}
+
+void free(void *p)
+{
+	if (p) {
+		struct alloc_hdr *h = ptr2hdr(p);
+		struct src_loc *l = h->as.live.loc;
+
+		if (!real_free) return; /* oh well, leak a little */
+		if (l) {
+			size_t age = generation - h->as.live.gen;
+
+			uatomic_set(&h->size, 0);
+			uatomic_add(&l->frees, 1);
+			uatomic_add(&l->age_total, age);
+
+			mutex_lock(l->mtx);
+			cds_list_del_rcu(&h->anode);
+			if (age > l->max_lifespan)
+				l->max_lifespan = age;
+			mutex_unlock(l->mtx);
+
+			call_rcu(&h->as.dead, free_hdr_rcu);
+		}
+		else {
+			real_free(h->real);
+		}
+	}
+}
+
+static void
+alloc_insert_rcu(struct src_loc *l, struct alloc_hdr *h, size_t size, void *real)
+{
+	/* we need src_loc to remain alive for the duration of this call */
+	if (!h) return;
+	h->size = size;
+	h->real = real;
+	h->as.live.loc = l;
+	h->as.live.gen = generation;
+	if (l) {
+		mutex_lock(l->mtx);
+		cds_list_add_rcu(&h->anode, &l->allocs);
+		mutex_unlock(l->mtx);
+	}
+}
+
+static size_t size_align(size_t size, size_t alignment)
+{
+	return ((size + (alignment - 1)) & ~(alignment - 1));
+}
+
+static bool ptr_is_aligned(void *ptr, size_t alignment)
+{
+	return ((uintptr_t)ptr & (alignment - 1)) == 0;
+}
+
+static void *ptr_align(void *ptr, size_t alignment)
+{
+	return (void *)(((uintptr_t)ptr + (alignment - 1)) & ~(alignment - 1));
+}
+
+static bool is_power_of_two(size_t n) { return (n & (n - 1)) == 0; }
+
+static int
+internal_memalign(void **pp, size_t alignment, size_t size, uintptr_t caller)
+{
+	struct src_loc *l;
+	struct alloc_hdr *h;
+	void *real;
+	size_t asize;
+	size_t d = alignment / sizeof(void*);
+	size_t r = alignment % sizeof(void*);
+
+	if (!real_malloc) return ENOMEM;
+
+	if (r != 0 || d == 0 || !is_power_of_two(d))
+		return EINVAL;
+
+	if (alignment <= ASSUMED_MALLOC_ALIGNMENT) {
+		void *p = malloc(size);
+		if (!p) return ENOMEM;
+		*pp = p;
+		return 0;
+	}
+	for (; alignment < sizeof(struct alloc_hdr); alignment *= 2)
+		; /* double alignment until >= sizeof(struct alloc_hdr) */
+	if (__builtin_add_overflow(size, alignment, &asize) ||
+	    __builtin_add_overflow(asize, sizeof(struct alloc_hdr), &asize))
+		return ENOMEM;
+
+	/* assert(asize == (alignment + size + sizeof(struct alloc_hdr))); */
+	l = track_memalign ? update_stats_rcu_lock(size, caller) : 0;
+	real = real_malloc(asize);
+	if (real) {
+		void *p = hdr2ptr(real);
+		if (!ptr_is_aligned(p, alignment))
+			p = ptr_align(p, alignment);
+		h = ptr2hdr(p);
+		alloc_insert_rcu(l, h, size, real);
+		*pp = p;
+	}
+	update_stats_rcu_unlock(l);
+
+	return real ? 0 : ENOMEM;
+}
+
+static void *
+memalign_result(int err, void *p)
+{
+	if (caa_unlikely(err)) {
+		errno = err;
+		return 0;
+	}
+	return p;
+}
+
+void *memalign(size_t alignment, size_t size)
+{
+	void *p;
+	int err = internal_memalign(&p, alignment, size, RETURN_ADDRESS(0));
+	return memalign_result(err, p);
+}
+
+int posix_memalign(void **p, size_t alignment, size_t size)
+{
+	return internal_memalign(p, alignment, size, RETURN_ADDRESS(0));
+}
+
+void *aligned_alloc(size_t, size_t) __attribute__((alias("memalign")));
+void cfree(void *) __attribute__((alias("free")));
+
+void *valloc(size_t size)
+{
+	void *p;
+	int err = internal_memalign(&p, page_size, size, RETURN_ADDRESS(0));
+	return memalign_result(err, p);
+}
+
+#if __GNUC__ < 7
+#  define add_overflow_p(a,b) __extension__({ \
+		__typeof__(a) _c; \
+		__builtin_add_overflow(a,b,&_c); \
+	})
+#else
+#  define add_overflow_p(a,b) \
+		__builtin_add_overflow_p((a),(b),(__typeof__(a+b))0)
+#endif
+
+void *pvalloc(size_t size)
+{
+	size_t alignment = page_size;
+	void *p;
+	int err;
+
+	if (add_overflow_p(size, alignment)) {
+		errno = ENOMEM;
+		return 0;
+	}
+	size = size_align(size, alignment);
+	err = internal_memalign(&p, alignment, size, RETURN_ADDRESS(0));
+	return memalign_result(err, p);
 }
 
-/*
- * Do we care for *memalign? ruby/gc.c uses it in ways this lib
- * doesn't care about, but maybe some gems use it, too.
- */
 void *malloc(size_t size)
 {
-	RETURN_IF_NOT_READY();
-	update_stats(size, RETURN_ADDRESS(0));
-	return real_malloc(size);
+	struct src_loc *l;
+	struct alloc_hdr *h;
+	size_t asize;
+	void *p;
+
+	if (__builtin_add_overflow(size, sizeof(struct alloc_hdr), &asize))
+		goto enomem;
+
+	/*
+	 * Needed for C++ global declarations using "new",
+	 * which happens before our constructor
+	 */
+#ifndef __FreeBSD__
+	if (!real_malloc) {
+		if (resolving_malloc) goto enomem;
+		resolving_malloc = 1;
+		real_malloc = dlsym(RTLD_NEXT, "malloc");
+	}
+#endif
+	l = update_stats_rcu_lock(size, RETURN_ADDRESS(0));
+	p = h = real_malloc(asize);
+	if (h) {
+		alloc_insert_rcu(l, h, size, h);
+		p = hdr2ptr(h);
+	}
+	update_stats_rcu_unlock(l);
+	if (caa_unlikely(!p)) errno = ENOMEM;
+	return p;
+enomem:
+	errno = ENOMEM;
+	return 0;
 }
 
 void *calloc(size_t nmemb, size_t size)
 {
+	void *p;
+	struct src_loc *l;
+	struct alloc_hdr *h;
+	size_t asize;
+
+	if (__builtin_mul_overflow(size, nmemb, &size)) {
+		errno = ENOMEM;
+		return 0;
+	}
+	if (__builtin_add_overflow(size, sizeof(struct alloc_hdr), &asize)) {
+		errno = ENOMEM;
+		return 0;
+	}
 	RETURN_IF_NOT_READY();
-	/* ruby_xcalloc already does overflow checking */
-	update_stats(nmemb * size, RETURN_ADDRESS(0));
-	return real_calloc(nmemb, size);
+	l = update_stats_rcu_lock(size, RETURN_ADDRESS(0));
+	p = h = real_malloc(asize);
+	if (p) {
+		alloc_insert_rcu(l, h, size, h);
+		p = hdr2ptr(h);
+		memset(p, 0, size);
+	}
+	update_stats_rcu_unlock(l);
+	if (caa_unlikely(!p)) errno = ENOMEM;
+	return p;
 }
 
 void *realloc(void *ptr, size_t size)
 {
+	void *p;
+	struct src_loc *l;
+	struct alloc_hdr *h;
+	size_t asize;
+
+	if (!size) {
+		free(ptr);
+		return 0;
+	}
+	if (__builtin_add_overflow(size, sizeof(struct alloc_hdr), &asize)) {
+		errno = ENOMEM;
+		return 0;
+	}
 	RETURN_IF_NOT_READY();
-	update_stats(size, RETURN_ADDRESS(0));
-	return real_realloc(ptr, size);
+
+	l = update_stats_rcu_lock(size, RETURN_ADDRESS(0));
+	p = h = real_malloc(asize);
+	if (p) {
+		alloc_insert_rcu(l, h, size, h);
+		p = hdr2ptr(h);
+	}
+	update_stats_rcu_unlock(l);
+
+	if (ptr && p) {
+		struct alloc_hdr *old = ptr2hdr(ptr);
+		memcpy(p, ptr, old->size < size ? old->size : size);
+		free(ptr);
+	}
+	if (caa_unlikely(!p)) errno = ENOMEM;
+	return p;
 }
 
 struct dump_arg {
@@ -314,7 +665,7 @@ static void *dump_to_file(void *x)
 			p = s[0];
 		}
 		fprintf(a->fp, "%16zu %12zu %s\n",
-			l->total, l->calls, (const char *)p);
+			l->total, l->allocations, (const char *)p);
 		if (s) free(s);
 	}
 out_unlock:
@@ -349,6 +700,7 @@ static VALUE mwrap_dump(int argc, VALUE * argv, VALUE mod)
 		io = *((VALUE *)dlsym(RTLD_DEFAULT, "rb_stderr"));
 
 	a.min = NIL_P(min) ? 0 : NUM2SIZET(min);
+	io = rb_io_get_io(io);
 	io = rb_io_get_write_io(io);
 	GetOpenFile(io, fptr);
 	a.fp = rb_io_stdio_file(fptr);
@@ -358,49 +710,6 @@ static VALUE mwrap_dump(int argc, VALUE * argv, VALUE mod)
 	return Qnil;
 }
 
-static void
-free_src_loc(struct rcu_head *head)
-{
-	struct src_loc *l = caa_container_of(head, struct src_loc, rcu_head);
-	free(l);
-}
-
-static void *totals_clear(void *ign)
-{
-	struct cds_lfht *new, *old;
-	struct cds_lfht_iter iter;
-	struct src_loc *l;
-
-	new = lfht_new();
-	rcu_read_lock();
-	old = rcu_dereference(totals);
-	rcu_assign_pointer(totals, new);
-	cds_lfht_for_each_entry(old, &iter, l, hnode) {
-		cds_lfht_del(old, &l->hnode);
-		call_rcu(&l->rcu_head, free_src_loc);
-	}
-	rcu_read_unlock();
-
-	synchronize_rcu(); /* ensure totals points to new */
-	cds_lfht_destroy(old, NULL);
-	return 0;
-}
-
-/*
- * call-seq:
- *
- *	Mwrap.clear -> nil
- *
- * Atomically replaces the totals table and destroys the old one.
- * This resets all statistics. It is more expensive than `Mwrap.reset'
- * as new allocations will need to be made to repopulate the new table.
- */
-static VALUE mwrap_clear(VALUE mod)
-{
-	rb_thread_call_without_gvl(totals_clear, 0, 0, 0);
-	return Qnil;
-}
-
 static void *totals_reset(void *ign)
 {
 	struct cds_lfht *t;
@@ -411,7 +720,10 @@ static void *totals_reset(void *ign)
 	t = rcu_dereference(totals);
 	cds_lfht_for_each_entry(t, &iter, l, hnode) {
 		uatomic_set(&l->total, 0);
-		uatomic_set(&l->calls, 0);
+		uatomic_set(&l->allocations, 0);
+		uatomic_set(&l->frees, 0);
+		uatomic_set(&l->age_total, 0);
+		uatomic_set(&l->max_lifespan, 0);
 	}
 	rcu_read_unlock();
 	return 0;
@@ -423,8 +735,8 @@ static void *totals_reset(void *ign)
  *	Mwrap.reset -> nil
  *
  * Resets the the total tables by zero-ing all counters.
- * This resets all statistics and is less costly than `Mwrap.clear'
- * but is not an atomic operation.
+ * This resets all statistics.  This is not an atomic operation
+ * as other threads (outside of GVL) may increment counters.
  */
 static VALUE mwrap_reset(VALUE mod)
 {
@@ -432,13 +744,40 @@ static VALUE mwrap_reset(VALUE mod)
 	return Qnil;
 }
 
-static VALUE dump_ensure(VALUE ignored)
+/* :nodoc: */
+static VALUE mwrap_clear(VALUE mod)
+{
+	return mwrap_reset(mod);
+}
+
+static VALUE rcu_unlock_ensure(VALUE ignored)
 {
 	rcu_read_unlock();
 	--locating;
 	return Qfalse;
 }
 
+static VALUE location_string(struct src_loc *l)
+{
+	VALUE ret, tmp;
+
+	if (loc_is_addr(l)) {
+		char **s = backtrace_symbols((void *)l->k, 1);
+		tmp = rb_str_new_cstr(s[0]);
+		free(s);
+	}
+	else {
+		tmp = rb_str_new(l->k, l->capa - 1);
+	}
+
+	/* deduplicate and try to free up some memory */
+	ret = rb_funcall(tmp, id_uminus, 0);
+	if (!OBJ_FROZEN_RAW(tmp))
+		rb_str_resize(tmp, 0);
+
+	return ret;
+}
+
 static VALUE dump_each_rcu(VALUE x)
 {
 	struct dump_arg *a = (struct dump_arg *)x;
@@ -448,27 +787,17 @@ static VALUE dump_each_rcu(VALUE x)
 
 	t = rcu_dereference(totals);
 	cds_lfht_for_each_entry(t, &iter, l, hnode) {
-		VALUE v[3];
+		VALUE v[6];
 		if (l->total <= a->min) continue;
 
-		if (loc_is_addr(l)) {
-			char **s = backtrace_symbols((void *)l->k, 1);
-			v[1] = rb_str_new_cstr(s[0]);
-			free(s);
-		}
-		else {
-			v[1] = rb_str_new(l->k, l->capa - 1);
-		}
-
-		/* deduplicate and try to free up some memory */
-		v[0] = rb_funcall(v[1], id_uminus, 0);
-		if (!OBJ_FROZEN_RAW(v[1]))
-			rb_str_resize(v[1], 0);
-
+		v[0] = location_string(l);
 		v[1] = SIZET2NUM(l->total);
-		v[2] = SIZET2NUM(l->calls);
+		v[2] = SIZET2NUM(l->allocations);
+		v[3] = SIZET2NUM(l->frees);
+		v[4] = SIZET2NUM(l->age_total);
+		v[5] = SIZET2NUM(l->max_lifespan);
 
-		rb_yield_values2(3, v);
+		rb_yield_values2(6, v);
 		assert(rcu_read_ongoing());
 	}
 	return Qnil;
@@ -477,10 +806,12 @@ static VALUE dump_each_rcu(VALUE x)
 /*
  * call-seq:
  *
- * 	Mwrap.each([min]) { |location,total_bytes,call_count| ... }
+ *	Mwrap.each([min]) do |location,total,allocations,frees,age_total,max_lifespan|
+ *	  ...
+ *	end
  *
  * Yields each entry of the of the table to a caller-supplied block.
- * +min+ may be specified to filter out lines with +total_bytes+
+ * +min+ may be specified to filter out lines with +total+ bytes
  * equal-to-or-smaller-than the supplied minimum.
  */
 static VALUE mwrap_each(int argc, VALUE * argv, VALUE mod)
@@ -494,7 +825,212 @@ static VALUE mwrap_each(int argc, VALUE * argv, VALUE mod)
 	++locating;
 	rcu_read_lock();
 
-	return rb_ensure(dump_each_rcu, (VALUE)&a, dump_ensure, 0);
+	return rb_ensure(dump_each_rcu, (VALUE)&a, rcu_unlock_ensure, 0);
+}
+
+static size_t
+src_loc_memsize(const void *p)
+{
+	return sizeof(struct src_loc);
+}
+
+static const rb_data_type_t src_loc_type = {
+	"source_location",
+	/* no marking, no freeing */
+	{ 0, 0, src_loc_memsize, /* reserved */ },
+	/* parent, data, [ flags ] */
+};
+
+static VALUE cSrcLoc;
+
+static int
+extract_addr(const char *str, size_t len, void **p)
+{
+	const char *c;
+#if defined(__GLIBC__)
+	return ((c = memrchr(str, '[', len)) && sscanf(c, "[%p]", p));
+#else /* tested FreeBSD */
+	return ((c = strstr(str, "0x")) && sscanf(c, "%p", p));
+#endif
+}
+
+/*
+ * call-seq:
+ *	Mwrap[location] -> Mwrap::SourceLocation
+ *
+ * Returns the associated Mwrap::SourceLocation given the +location+
+ * String.  +location+ is either a Ruby source location path:line
+ * (e.g. "/path/to/foo.rb:5") or a hexadecimal memory address with
+ * square-braces part yielded by Mwrap.dump (e.g. "[0xdeadbeef]")
+ */
+static VALUE mwrap_aref(VALUE mod, VALUE loc)
+{
+	const char *str = StringValueCStr(loc);
+	int len = RSTRING_LENINT(loc);
+	struct src_loc *k = 0;
+	uintptr_t p;
+	struct cds_lfht_iter iter;
+	struct cds_lfht_node *cur;
+	struct cds_lfht *t;
+	struct src_loc *l;
+	VALUE val = Qnil;
+
+	if (extract_addr(str, len, (void **)&p)) {
+		k = (void *)kbuf;
+		memcpy(k->k, &p, sizeof(p));
+		k->capa = 0;
+		k->hval = jhash(k->k, sizeof(p), 0xdeadbeef);
+	} else {
+		k = (void *)kbuf;
+		memcpy(k->k, str, len + 1);
+		k->capa = len + 1;
+		k->hval = jhash(k->k, k->capa, 0xdeadbeef);
+	}
+
+	if (!k) return val;
+
+	rcu_read_lock();
+	t = rcu_dereference(totals);
+	if (!t) goto out_unlock;
+
+	cds_lfht_lookup(t, k->hval, loc_eq, k, &iter);
+	cur = cds_lfht_iter_get_node(&iter);
+	if (cur) {
+		l = caa_container_of(cur, struct src_loc, hnode);
+		val = TypedData_Wrap_Struct(cSrcLoc, &src_loc_type, l);
+	}
+out_unlock:
+	rcu_read_unlock();
+	return val;
+}
+
+static VALUE src_loc_each_i(VALUE p)
+{
+	struct alloc_hdr *h;
+	struct src_loc *l = (struct src_loc *)p;
+
+	cds_list_for_each_entry_rcu(h, &l->allocs, anode) {
+		size_t gen = uatomic_read(&h->as.live.gen);
+		size_t size = uatomic_read(&h->size);
+
+		if (size) {
+			VALUE v[2];
+			v[0] = SIZET2NUM(size);
+			v[1] = SIZET2NUM(gen);
+
+			rb_yield_values2(2, v);
+		}
+	}
+
+	return Qfalse;
+}
+
+static struct src_loc *src_loc_get(VALUE self)
+{
+	struct src_loc *l;
+	TypedData_Get_Struct(self, struct src_loc, &src_loc_type, l);
+	assert(l);
+	return l;
+}
+
+/*
+ * call-seq:
+ *	loc = Mwrap[location]
+ *	loc.each { |size,generation| ... }
+ *
+ * Iterates through live allocations for a given Mwrap::SourceLocation,
+ * yielding the +size+ (in bytes) and +generation+ of each allocation.
+ * The +generation+ is the value of the GC.count method at the time
+ * the allocation was made.
+ *
+ * This functionality is only available in mwrap 2.0.0+
+ */
+static VALUE src_loc_each(VALUE self)
+{
+	struct src_loc *l = src_loc_get(self);
+
+	assert(locating == 0 && "forgot to clear locating");
+	++locating;
+	rcu_read_lock();
+	rb_ensure(src_loc_each_i, (VALUE)l, rcu_unlock_ensure, 0);
+	return self;
+}
+
+/*
+ * The the mean lifespan (in GC generations) of allocations made from this
+ * location.  This does not account for live allocations.
+ */
+static VALUE src_loc_mean_lifespan(VALUE self)
+{
+	struct src_loc *l = src_loc_get(self);
+	size_t tot, frees;
+
+	frees = uatomic_read(&l->frees);
+	tot = uatomic_read(&l->age_total);
+	return DBL2NUM(frees ? ((double)tot/(double)frees) : HUGE_VAL);
+}
+
+/* The number of frees made from this location */
+static VALUE src_loc_frees(VALUE self)
+{
+	return SIZET2NUM(uatomic_read(&src_loc_get(self)->frees));
+}
+
+/* The number of allocations made from this location */
+static VALUE src_loc_allocations(VALUE self)
+{
+	return SIZET2NUM(uatomic_read(&src_loc_get(self)->allocations));
+}
+
+/* The total number of bytes allocated from this location */
+static VALUE src_loc_total(VALUE self)
+{
+	return SIZET2NUM(uatomic_read(&src_loc_get(self)->total));
+}
+
+/*
+ * The maximum age (in GC generations) of an allocation before it was freed.
+ * This does not account for live allocations.
+ */
+static VALUE src_loc_max_lifespan(VALUE self)
+{
+	return SIZET2NUM(uatomic_read(&src_loc_get(self)->max_lifespan));
+}
+
+/*
+ * Returns a frozen String location of the given SourceLocation object.
+ */
+static VALUE src_loc_name(VALUE self)
+{
+	struct src_loc *l = src_loc_get(self);
+	VALUE ret;
+
+	++locating;
+	ret = location_string(l);
+	--locating;
+	return ret;
+}
+
+static VALUE reset_locating(VALUE ign) { --locating; return Qfalse; }
+
+/*
+ * call-seq:
+ *
+ *	Mwrap.quiet do |depth|
+ *	  # expensive sort/calculate/emitting results of Mwrap.each
+ *	  # affecting statistics of the rest of the app
+ *	end
+ *
+ * Stops allocation tracking inside the block.  This is useful for
+ * monitoring code which calls other Mwrap (or ObjectSpace/GC)
+ * functions which unavoidably allocate memory.
+ *
+ * This feature was added in mwrap 2.0.0+
+ */
+static VALUE mwrap_quiet(VALUE mod)
+{
+	size_t cur = ++locating;
+	return rb_ensure(rb_yield, SIZET2NUM(cur), reset_locating, 0);
 }
 
 /*
@@ -515,19 +1051,47 @@ static VALUE mwrap_each(int argc, VALUE * argv, VALUE mod)
  * * dump_fd: a writable FD to dump to
  * * dump_path: a path to dump to, the file is opened in O_APPEND mode
  * * dump_min: the minimum allocation size (total) to dump
+ * * memalign: use `1' to enable tracking the memalign family
  *
  * If both `dump_fd' and `dump_path' are specified, dump_path takes
  * precedence.
+ *
+ * Tracking the memalign family of functions is misleading for Ruby
+ * applications, as heap page allocations can happen anywhere a
+ * Ruby object is allocated, even in the coldest code paths.
+ * Furthermore, it is rarely-used outside of the Ruby object allocator.
+ * Thus tracking memalign functions is disabled by default.
  */
 void Init_mwrap(void)
 {
-	VALUE mod = rb_define_module("Mwrap");
+	VALUE mod;
+
+	++locating;
+	mod = rb_define_module("Mwrap");
 	id_uminus = rb_intern("-@");
 
+	/*
+	 * Represents a location in source code or library
+	 * address which calls a memory allocation.  It is
+	 * updated automatically as allocations are made, so
+	 * there is no need to reload or reread it from Mwrap#[].
+	 * This class is only available since mwrap 2.0.0+.
+	 */
+	cSrcLoc = rb_define_class_under(mod, "SourceLocation", rb_cObject);
 	rb_define_singleton_method(mod, "dump", mwrap_dump, -1);
-	rb_define_singleton_method(mod, "clear", mwrap_clear, 0);
 	rb_define_singleton_method(mod, "reset", mwrap_reset, 0);
+	rb_define_singleton_method(mod, "clear", mwrap_clear, 0);
 	rb_define_singleton_method(mod, "each", mwrap_each, -1);
+	rb_define_singleton_method(mod, "[]", mwrap_aref, 1);
+	rb_define_singleton_method(mod, "quiet", mwrap_quiet, 0);
+	rb_define_method(cSrcLoc, "each", src_loc_each, 0);
+	rb_define_method(cSrcLoc, "frees", src_loc_frees, 0);
+	rb_define_method(cSrcLoc, "allocations", src_loc_allocations, 0);
+	rb_define_method(cSrcLoc, "total", src_loc_total, 0);
+	rb_define_method(cSrcLoc, "mean_lifespan", src_loc_mean_lifespan, 0);
+	rb_define_method(cSrcLoc, "max_lifespan", src_loc_max_lifespan, 0);
+	rb_define_method(cSrcLoc, "name", src_loc_name, 0);
+	--locating;
 }
 
 /* rb_cloexec_open isn't usable by non-Ruby processes */
@@ -538,18 +1102,18 @@ void Init_mwrap(void)
 __attribute__ ((destructor))
 static void mwrap_dump_destructor(void)
 {
-        const char *opt = getenv("MWRAP");
-        const char *modes[] = { "a", "a+", "w", "w+", "r+" };
-        struct dump_arg a;
-        size_t i;
-        int dump_fd;
+	const char *opt = getenv("MWRAP");
+	const char *modes[] = { "a", "a+", "w", "w+", "r+" };
+	struct dump_arg a;
+	size_t i;
+	int dump_fd;
 	char *dump_path;
 
 	if (!opt)
 		return;
 
-        ++locating;
-        if ((dump_path = strstr(opt, "dump_path:")) &&
+	++locating;
+	if ((dump_path = strstr(opt, "dump_path:")) &&
 			(dump_path += sizeof("dump_path")) &&
 			*dump_path) {
 		char *end = strchr(dump_path, ',');
@@ -594,5 +1158,5 @@ static void mwrap_dump_destructor(void)
 	}
 	dump_to_file(&a);
 out:
-    --locating;
+	--locating;
 }