ruby_memprofiler_pprof 0.0.1

data/ext/ruby_memprofiler_pprof/pprof_out.c

@@ -0,0 +1,285 @@
+#include <ruby.h>
+#include <zlib.h>
+
+#include "ruby_memprofiler_pprof.h"
+
+// allocator/free function for our upb_arena. Contract is:
+// If "size" is 0 then the function acts like free(), otherwise it acts like
+// realloc().  Only "oldsize" bytes from a previous allocation are preserved.
+static void *mpp_pprof_upb_arena_malloc(upb_alloc *alloc, void *ptr, size_t oldsize, size_t size) {
+    if (size == 0) {
+        mpp_free(ptr);
+        return NULL;
+    } else {
+        return mpp_realloc(ptr, size);
+    }
+}
+
+// Initialize an already-allocated serialization context.
+struct mpp_pprof_serctx *mpp_pprof_serctx_new() {
+    struct mpp_pprof_serctx *ctx = mpp_xmalloc(sizeof(struct mpp_pprof_serctx));
+    ctx->allocator.func = mpp_pprof_upb_arena_malloc;
+    ctx->arena = upb_Arena_Init(NULL, 0, &ctx->allocator);
+    ctx->profile_proto = perftools_profiles_Profile_new(ctx->arena);
+    ctx->strindex = NULL;
+    return ctx;
+}
+
+// Destroys the serialization context. After this call, any stringtab indexes it held
+// are released, and any memory from its internal state is freed. *ctx itself is also
+// freed and must not be dereferenced after this.
+void mpp_pprof_serctx_destroy(struct mpp_pprof_serctx *ctx) {
+    upb_Arena_Free(ctx->arena);
+    if (ctx->strindex) {
+        mpp_strtab_index_destroy(ctx->strindex);
+    }
+    mpp_free(ctx);
+}
+
+
+struct mpp_pprof_serctx_add_location_ctx {
+    struct mpp_pprof_serctx *ctx;
+    int is_error;
+    char *errbuf;
+    size_t errbuflen;
+};
+
+static int mpp_pprof_serctx_add_location(
+    struct mpp_rb_loctab *loctab, struct mpp_rb_loctab_location *location, void *ctx
+) {
+    struct mpp_pprof_serctx_add_location_ctx *thunkctx = ctx;
+    struct perftools_profiles_Location *loc_proto = perftools_profiles_Profile_add_location(thunkctx->ctx->profile_proto, thunkctx->ctx->arena);
+
+
+    perftools_profiles_Location_set_id(loc_proto, location->id);
+    perftools_profiles_Line *line_proto = perftools_profiles_Location_add_line(loc_proto, thunkctx->ctx->arena);
+    perftools_profiles_Line_set_function_id(line_proto, location->function->id);
+    perftools_profiles_Line_set_line(line_proto, location->line_number);
+
+    return ST_CONTINUE;
+}
+
+struct mpp_pprof_serctx_add_function_ctx {
+    struct mpp_pprof_serctx *ctx;
+    int is_error;
+    char *errbuf;
+    size_t errbuflen;
+};
+
+static int mpp_pprof_serctx_add_function(
+    struct mpp_rb_loctab *loctab, struct mpp_rb_loctab_function *function, void *ctx
+) {
+    struct mpp_pprof_serctx_add_function_ctx *thunkctx = ctx;
+    struct perftools_profiles_Function *fn_proto = perftools_profiles_Profile_add_function(thunkctx->ctx->profile_proto, thunkctx->ctx->arena);
+
+    perftools_profiles_Function_set_id(fn_proto, function->id);
+
+#define FN_SET_STRINTERN_FIELD(field, str)                                                                  \
+    do {                                                                                                    \
+        int64_t interned = mpp_strtab_index_of(thunkctx->ctx->strindex, (str));                             \
+        if (interned == -1) {                                                                               \
+            ruby_snprintf(                                                                                  \
+                thunkctx->errbuf, thunkctx->errbuflen,                                                      \
+                "non-interned string %s passed for Function.#field", (str)                                  \
+            );                                                                                              \
+            return -1;                                                                                      \
+        }                                                                                                   \
+        perftools_profiles_Function_set_##field(fn_proto, interned);                                        \
+    } while (0)
+
+    FN_SET_STRINTERN_FIELD(name, function->function_name);
+    FN_SET_STRINTERN_FIELD(system_name, function->function_name);
+    FN_SET_STRINTERN_FIELD(filename, function->file_name);
+
+#undef FN_SET_STRINTERN_FIELD
+
+
+    return ST_CONTINUE;
+}
+
+
+// This method is used to set the location table used in backtrace samples to be added
+//
+// This method will actually take a reference to all relevant strings currently in loctab->strtab via
+// a call to mpp_strtab_index; thus, after this method returns, it is safe for another thread to
+// continue using the stringtab because we have what we need in our index.
+int mpp_pprof_serctx_set_loctab(
+    struct mpp_pprof_serctx *ctx, struct mpp_rb_loctab *loctab, char *errbuf, size_t errbuflen
+) {
+    ctx->loctab = loctab;
+
+    // Intern some strings we'll need to produce our output
+    mpp_strtab_intern_cstr(loctab->strtab, "allocations", &ctx->internstr_allocations, NULL);
+    mpp_strtab_intern_cstr(loctab->strtab, "count", &ctx->internstr_count, NULL);
+    mpp_strtab_intern_cstr(loctab->strtab, "allocation_size", &ctx->internstr_allocation_size, NULL);
+    mpp_strtab_intern_cstr(loctab->strtab, "bytes", &ctx->internstr_bytes, NULL);
+    mpp_strtab_intern_cstr(loctab->strtab, "retained_objects", &ctx->internstr_retained_objects, NULL);
+    mpp_strtab_intern_cstr(loctab->strtab, "retained_size", &ctx->internstr_retained_size, NULL);
+
+    ctx->strindex = mpp_strtab_index(loctab->strtab);
+    MPP_ASSERT_MSG(ctx->strindex, "mpp_strtab_index returned 0");
+
+    // Set up the string table in the protobuf
+    upb_StringView *stringtab_list_proto =
+        perftools_profiles_Profile_resize_string_table(ctx->profile_proto, ctx->strindex->str_list_len, ctx->arena);
+    for (int64_t i = 0; i < ctx->strindex->str_list_len; i++) {
+        upb_StringView *stringtab_proto = &stringtab_list_proto[i];
+        struct mpp_strtab_el *intern_tab_el = ctx->strindex->str_list[i];
+        stringtab_proto->data = intern_tab_el->str;
+        stringtab_proto->size = intern_tab_el->str_len;
+    }
+
+    // Set up the sample types etc.
+    perftools_profiles_ValueType *allocations_vt =
+        perftools_profiles_Profile_add_sample_type(ctx->profile_proto, ctx->arena);
+    perftools_profiles_ValueType *allocation_size_vt =
+        perftools_profiles_Profile_add_sample_type(ctx->profile_proto, ctx->arena);
+    perftools_profiles_ValueType *retained_objects_vt =
+        perftools_profiles_Profile_add_sample_type(ctx->profile_proto, ctx->arena);
+    perftools_profiles_ValueType *retained_size_vt =
+        perftools_profiles_Profile_add_sample_type(ctx->profile_proto, ctx->arena);
+#define VT_SET_STRINTERN_FIELD(vt, field, str)                                                              \
+    do {                                                                                                    \
+        int64_t interned = mpp_strtab_index_of(ctx->strindex, (str));                                       \
+        if (interned == -1) {                                                                               \
+            ruby_snprintf(errbuf, errbuflen, "non-interned string %s passed for ValueType.#field", (str));  \
+            return -1;                                                                                      \
+        }                                                                                                   \
+        perftools_profiles_ValueType_set_##field(vt, interned);                                             \
+    } while (0)
+
+    VT_SET_STRINTERN_FIELD(allocations_vt, type, ctx->internstr_allocations);
+    VT_SET_STRINTERN_FIELD(allocations_vt, unit, ctx->internstr_count);
+    VT_SET_STRINTERN_FIELD(allocation_size_vt, type, ctx->internstr_allocation_size);
+    VT_SET_STRINTERN_FIELD(allocation_size_vt, unit, ctx->internstr_bytes);
+    VT_SET_STRINTERN_FIELD(retained_objects_vt, type, ctx->internstr_retained_objects);
+    VT_SET_STRINTERN_FIELD(retained_objects_vt, unit, ctx->internstr_count);
+    VT_SET_STRINTERN_FIELD(retained_size_vt, type, ctx->internstr_retained_size);
+    VT_SET_STRINTERN_FIELD(retained_size_vt, unit, ctx->internstr_bytes);
+#undef VT_SET_STRINTERN_FIELD
+
+    // Set up the location array.
+    struct mpp_pprof_serctx_add_location_ctx loc_add_ctx;
+    loc_add_ctx.ctx = ctx;
+    loc_add_ctx.errbuf = errbuf;
+    loc_add_ctx.errbuflen = errbuflen;
+    loc_add_ctx.is_error = 0;
+    mpp_rb_loctab_each_location(ctx->loctab, mpp_pprof_serctx_add_location, &loc_add_ctx);
+    if (loc_add_ctx.is_error) {
+        return -1;
+    }
+
+    // And the same for functions pretty much
+    struct mpp_pprof_serctx_add_function_ctx fn_add_ctx;
+    fn_add_ctx.ctx = ctx;
+    fn_add_ctx.errbuf = errbuf;
+    fn_add_ctx.errbuflen = errbuflen;
+    fn_add_ctx.is_error = 0;
+    mpp_rb_loctab_each_function(ctx->loctab, mpp_pprof_serctx_add_function, &fn_add_ctx);
+    if (loc_add_ctx.is_error) {
+        return -1;
+    }
+
+    return 0;
+}
+
+int mpp_pprof_serctx_add_sample(
+    struct mpp_pprof_serctx *ctx, struct mpp_sample *sample, int sample_type, char *errbuf, size_t errbuflen
+) {
+    perftools_profiles_Sample *sample_proto = perftools_profiles_Profile_add_sample(ctx->profile_proto, ctx->arena);
+    uint64_t *location_ids =
+        perftools_profiles_Sample_resize_location_id(sample_proto, sample->bt->frames_count, ctx->arena);
+
+    // We need to iterate through this backtrace backwards; Ruby makes it easy for us to get this backtrace
+    // in most-recent-call-last format, but pprof wants most-recent-call-first.
+    for (int64_t i = 0; i < sample->bt->frames_count; i++) {
+        location_ids[i] = sample->bt->frame_locations[sample->bt->frames_count - i - 1];
+    }
+
+    // Values are (allocation_count, allocation_size, retained_count, retained_size).
+    int64_t allocation_count = 0;
+    int64_t allocation_size = 0;
+    int64_t retained_count = 0;
+    int64_t retained_size = 0;
+    if (sample_type == MPP_SAMPLE_TYPE_ALLOCATION) {
+        allocation_count = 1;
+        allocation_size = (int64_t)sample->allocation_size;
+    }
+    if (sample_type == MPP_SAMPLE_TYPE_HEAP) {
+        retained_count = 1;
+        retained_size = (int64_t)sample->current_size;
+    }
+    perftools_profiles_Sample_add_value(sample_proto, allocation_count, ctx->arena);
+    perftools_profiles_Sample_add_value(sample_proto, allocation_size, ctx->arena);
+    perftools_profiles_Sample_add_value(sample_proto, retained_count, ctx->arena);
+    perftools_profiles_Sample_add_value(sample_proto, retained_size, ctx->arena);
+    return 0;
+}
+
+// Serializes the contained protobuf, and gzips the result. Writes a pointer to the memory in *buf_out,
+// and its length to buflen_out. The returned pointer is freed when mpp_pprof_serctx_destroy() is called,
+// and should NOT be individually freed by the caller in any way (and nor is it valid after the call
+// to destroy()).
+int mpp_pprof_serctx_serialize(
+    struct mpp_pprof_serctx *ctx, char **buf_out, size_t *buflen_out, char *errbuf, size_t errbuflen
+) {
+    // It looks like some codepaths might leak the protobuf_data pointer, but that's OK - it's in
+    // the ctx->arena so it'll get freed when ctx does.
+    size_t protobuf_data_len;
+    char *protobuf_data = perftools_profiles_Profile_serialize(ctx->profile_proto, ctx->arena, &protobuf_data_len);
+
+    // Gzip it as per standard.
+    z_stream strm;
+    int r;
+    int retval = 0;
+
+    strm.zalloc = Z_NULL;
+    strm.zfree = Z_NULL;
+    strm.opaque = Z_NULL;
+    strm.msg = NULL;
+    int windowBits = 15;
+    int GZIP_ENCODING = 16;
+    r = deflateInit2(
+        &strm, Z_DEFAULT_COMPRESSION, Z_DEFLATED,
+        windowBits | GZIP_ENCODING, 8, Z_DEFAULT_STRATEGY
+    );
+    if (r != Z_OK) {
+        ruby_snprintf(errbuf, errbuflen, "error initializing zlib (errno %d: %s)", r, strm.msg ?: "");
+        return -1;
+    }
+    strm.avail_in = (unsigned int)protobuf_data_len;
+    strm.next_in = (unsigned char *)protobuf_data;
+
+    const size_t out_chunk_size = 4096;
+    char *gzip_data = upb_Arena_Malloc(ctx->arena, out_chunk_size);
+    size_t gzip_data_allocd_len = out_chunk_size;
+    strm.avail_out = out_chunk_size;
+    strm.next_out = (unsigned char *)gzip_data;
+    while(true) {
+        int flush = strm.avail_in == 0 ? Z_FINISH : Z_NO_FLUSH;
+        r = deflate(&strm, flush);
+        if (r == Z_STREAM_END) {
+            break;
+        } else if (r != Z_OK) {
+            // uh wat?
+            ruby_snprintf(errbuf, errbuflen, "error doing zlib output (errno %d: %s)", r, strm.msg ?: "");
+            retval = -1;
+            goto zstream_free;
+        }
+
+        if (strm.avail_out == 0) {
+            size_t old_gzip_data_allocd_len = gzip_data_allocd_len;
+            gzip_data_allocd_len += out_chunk_size;
+            gzip_data = upb_Arena_Realloc(ctx->arena, gzip_data, old_gzip_data_allocd_len, gzip_data_allocd_len);
+            strm.avail_out = out_chunk_size;
+            strm.next_out = (unsigned char *)(gzip_data + old_gzip_data_allocd_len);
+        }
+    }
+    // Set output pointers
+    *buf_out = gzip_data;
+    *buflen_out = gzip_data_allocd_len - strm.avail_out;
+
+zstream_free:
+    deflateEnd(&strm);
+    return retval;
+}

data/ext/ruby_memprofiler_pprof/ruby_memprofiler_pprof.c

@@ -0,0 +1,11 @@
+#include <ruby.h>
+#include "ruby_memprofiler_pprof.h"
+
+// This should be the only symbol actually visible to Ruby
+__attribute__(( visibility("default" )))
+void Init_ruby_memprofiler_pprof_ext() {
+    mpp_rand_init();
+    rb_ext_ractor_safe(true);
+    rb_define_module("MemprofilerPprof");
+    mpp_setup_collector_class();
+}

data/ext/ruby_memprofiler_pprof/ruby_memprofiler_pprof.h

@@ -0,0 +1,301 @@
+#ifndef __RUBY_MEMPROFILER_PPROF_H
+#define __RUBY_MEMPROFILER_PPROF_H
+
+#include <pthread.h>
+#include <stdint.h>
+
+#include <ruby.h>
+
+// UPB header files trip up a BUNCH of -Wshorten-64-to-32
+// Also ignore -Wpragmas so that if -Wshorten-64-to-32 isn't present
+// (it's a clang only thing), GCC doesn't warn about the unknown warning.
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wpragmas"
+#pragma GCC diagnostic ignored "-Wshorten-64-to-32"
+#include <upb/upb.h>
+#include "pprof.upb.h"
+#pragma GCC diagnostic pop
+
+// ======== COMPAT DECLARATIONS ========
+
+// For handling differences in ruby versions
+#ifndef HAVE_RB_GC_MARK_MOVABLE
+#define rb_gc_mark_movable(v) rb_gc_mark(v)
+#endif
+
+#ifndef RB_PASS_KEYWORDS
+#define RB_SCAN_ARGS_LAST_HASH_KEYWORDS 3
+#define rb_scan_args_kw(kw, c, v, s, ...) rb_scan_args(c, v, s, __VA_ARGS__)
+#endif
+
+#ifndef HAVE_RB_EXT_RACTOR_SAFE
+#define rb_ext_ractor_safe(x) do {} while (0)
+#endif
+
+// Apparently "I just want a random number, without thinking about whether it's
+// threadsafe, without thinking about whether some other part of the process needs
+// the global seed to be set to some deterministic value, and without calling into
+// the kernel every time" is... too much to ask for.
+// BSD has arc4random(3) for this, but for glibc we have to use one of the threadsafe
+// RNG's and seed a global instance of it, guarded by a mutex....
+// These methods wrap all that rubbish up.
+uint32_t mpp_rand();
+void mpp_rand_init();
+
+
+// These declarations just wrap some things from the standard library that should "always succeed", but call
+// our assertion macro if they fail to abort the program.
+void *mpp_xmalloc(size_t sz);
+void *mpp_realloc(void *mem, size_t newsz);
+void mpp_free(void *mem);
+void mpp_pthread_mutex_lock(pthread_mutex_t *m);
+void mpp_pthread_mutex_unlock(pthread_mutex_t *m);
+int mpp_pthread_mutex_trylock(pthread_mutex_t *m);
+void mpp_pthread_mutex_init(pthread_mutex_t *m, const pthread_mutexattr_t *attr);
+void mpp_pthread_mutex_destroy(pthread_mutex_t *m);
+void mpp_pthread_mutexattr_init(pthread_mutexattr_t *a);
+void mpp_pthread_mutexattr_destroy(pthread_mutexattr_t *a);
+void mpp_pthread_mutexattr_settype(pthread_mutexattr_t *a, int type);
+void mpp_pthread_atfork(void (*prepare)(void), void (*parent)(void), void (*child)(void));
+
+// Need a handy assertion macro. It would be nice to re-use rb_bug for some of this, but that actually
+// requires the GVL (it walks the Ruby stack frames, for one) and we (want to) run some code outside
+// the GVL, so this assertion macro has to be threadsafe. So we just implement it in pretty much a
+// similar way to how stdlib's assert() works (plus some stuff to prefix the gem name to the abort message
+// so users know who is at fault).
+#define MPP_ASSERT_STRINGIFY1(x) #x
+#define MPP_ASSERT_STRINGIFY2(x) MPP_ASSERT_STRINGIFY1(x)
+#define MPP_ASSERT__LINE MPP_ASSERT_STRINGIFY2(__LINE__)
+__attribute__ ((noreturn))
+void mpp_assert_fail(const char *msg, const char *assertion, const char *file, const char *line, const char *fn);
+#define MPP_ASSERT_MSG(expr, msg)                                                   \
+    do {                                                                            \
+        if ((expr) == 0) {                                                          \
+            mpp_assert_fail((msg), #expr, __FILE__, MPP_ASSERT__LINE, __func__);    \
+        }                                                                           \
+    } while (0)
+#define MPP_ASSERT_FAIL(expr) MPP_ASSERT_MSG(expr, 0)
+
+// Log a debug message to "somewhere". This could be smarter in future, but for now, this'll do.
+// The implementation here does not depend on holding the GVL.
+// Will automatically add a trailing newline.
+void mpp_log_debug(const char *pattern, ...);
+
+// ======== STRTAB DECLARATIONS ========
+
+#define MPP_STRTAB_USE_STRLEN (-1)
+#define MPP_STRTAB_UNKNOWN_LITERAL "(unknown)"
+#define MPP_STRTAB_UNKNOWN_LITERAL_LEN ((int)strlen(MPP_STRTAB_UNKNOWN_LITERAL))
+
+// Specialisation of the st_hash for "strings with length".
+struct mpp_strtab_key {
+    const char *str;    // Pointer to the string; n.b. it does NOT need a null-terminator
+    size_t str_len;     // Size of the string, not including any null terminator.
+};
+
+// I copied this magic number out of st.c from Ruby.
+#define FNV1_32A_INIT 0x811c9dc5
+
+struct mpp_strtab_el {
+    // Pointer to null-terminated string that has been interned
+    char *str;
+    // Length of str, NOT INCLUDING the null terminator
+    size_t str_len;
+    // Number of times this string has been interned. When the refcount drops to zero, the string
+    // is removed from the table and str is free'd.
+    uint64_t refcount;
+    // We cleverly keep the key _inside_ the value, so we don't need a separate bunch of malloc'd
+    // memory for each _key_ as well as each _value_.
+    struct mpp_strtab_key key;
+};
+
+struct mpp_strtab {
+    // The actual table which contains a mapping of (string hash) -> (str_intern_tab_el *)
+    st_table *table;
+    // Number of entries in table.
+    int64_t table_count;
+    // (approximate) allocated size of the _entries_ in the st_table (but not the st_table
+    // itself).
+    size_t table_entry_size;
+    // Value of "" interned in the table.
+    const char *interned_empty_str;
+};
+
+struct mpp_strtab_index {
+    // The table this index is a part of and came from.
+    struct mpp_strtab *tab;
+    // The list & length of interned strings.
+    struct mpp_strtab_el **str_list;
+    int64_t str_list_len;
+    // This st_hash is used to convert already-interned pointers to index into str_list.
+    // It's a map of (uintptr_t) -> (int64_t)
+    st_table *pos_table;
+};
+
+
+// NOTE - there's better documentation of what these methods do in strtab.c itself.
+
+// Initializes a new, empty string intern table. This will allocate memory that remains owned by the strtab
+// module and saves it in tab. It also allocates memory for struct intern_tab itself.
+struct mpp_strtab *mpp_strtab_new();
+
+// Destroys a string intern table, including freeing the underlying memory used by tab, and freeing
+// the memory pointed to by tab itself.
+void mpp_strtab_destroy(struct mpp_strtab *tab);
+
+// Get the size of all memory used by the table
+size_t mpp_strtab_memsize(struct mpp_strtab *tab);
+
+// Intern new strings (or increment the refcount of already-interned ones)
+void mpp_strtab_intern(
+    struct mpp_strtab *tab, const char *str, int str_len,
+    const char **interned_str_out, size_t *interned_str_len_out
+);
+void mpp_strtab_intern_rbstr(
+    struct mpp_strtab *tab, VALUE rbstr,
+    const char **interned_str_out, size_t *interned_str_len_out
+);
+void mpp_strtab_intern_cstr(
+    struct mpp_strtab *tab, const char *str,
+    const char **interned_str_out, size_t *interned_str_len_out
+);
+
+// Decrement the refcount of elements in the intern table.
+void mpp_strtab_release(struct mpp_strtab *tab, const char *str, size_t str_len);
+void mpp_strtab_release_rbstr(struct mpp_strtab *tab, VALUE rbstr);
+
+// Methods for building a zero=-based list of interned pointers, for building the final string table
+// in the pprof protobuf.
+struct mpp_strtab_index *mpp_strtab_index(struct mpp_strtab *tab);
+void mpp_strtab_index_destroy(struct mpp_strtab_index *ix);
+int64_t mpp_strtab_index_of(struct mpp_strtab_index *ix, const char *interned_ptr);
+typedef void (*mpp_strtab_each_fn)(int64_t el_ix, const char *interned_str, size_t interned_str_len, void *ctx);
+void mpp_strtab_each(struct mpp_strtab_index *ix, mpp_strtab_each_fn fn, void *ctx);
+
+// ======== BACKTRACE DECLARATIONS ========
+
+#define MPP_BT_METHOD_CFP 1
+#define MPP_BT_METHOD_SLOWRB 2
+
+struct mpp_rb_backtrace {
+    // The array of frames - most recent call FIRST.
+    uint64_t *frame_locations;
+    int64_t frames_count;
+    // Memory size of frames, which might actually be bigger than
+    // sizeof(frame_locations[0]) * frames_count
+    size_t memsize;
+};
+
+struct mpp_rb_loctab_function {
+    // Refcount
+    int refcount;
+    // Interned pointer to function name
+    const char *function_name;
+    size_t function_name_len;
+    // Interned pointer to file name
+    const char *file_name;
+    size_t file_name_len;
+    // Line number where the function starts
+    int64_t line_number;
+    // Function ID of self
+    uint64_t id;
+};
+
+struct mpp_rb_loctab_location {
+    // Refcount
+    int refcount;
+    // Function & line number. Note that we do _NOT_ hold a reference
+    // to the function here. Instead, each backtrace frame holds a reference
+    // to both the function and the location.
+    struct mpp_rb_loctab_function *function;
+    // Line number
+    int64_t line_number;
+    // Location id of self
+    uint64_t id;
+};
+
+struct mpp_rb_loctab {
+    // The table of (uint64_t location_id) -> (struct mpp_rb_loctab_location *)
+    st_table *locations;
+    int64_t location_count;
+    // The table of (uint64_t function_id) -> (struct mpp_rb_locatab_function *)
+    st_table *functions;
+    int64_t function_count;
+    // The string interning table that all the names are related to
+    struct mpp_strtab *strtab;
+};
+
+struct mpp_rb_loctab *mpp_rb_loctab_new(struct mpp_strtab *strtab);
+void mpp_rb_loctab_destroy(struct mpp_rb_loctab *loctab);
+void mpp_rb_backtrace_capture(struct mpp_rb_loctab *locatb, struct mpp_rb_backtrace **bt_out);
+void mpp_rb_backtrace_capture_slowrb(struct mpp_rb_loctab *locatb, struct mpp_rb_backtrace **bt_out);
+void mpp_rb_backtrace_destroy(struct mpp_rb_loctab *locatb, struct mpp_rb_backtrace *bt);
+size_t mpp_rb_backtrace_memsize(struct mpp_rb_backtrace *bt);
+size_t mpp_rb_loctab_memsize(struct mpp_rb_loctab *loctab);
+typedef int (*mpp_rb_loctab_each_location_cb)(struct mpp_rb_loctab *loctab, struct mpp_rb_loctab_location *loc, void *ctx);
+void mpp_rb_loctab_each_location(struct mpp_rb_loctab *loctab, mpp_rb_loctab_each_location_cb cb, void *ctx);
+typedef int (*mpp_rb_loctab_each_function_cb)(struct mpp_rb_loctab *loctab, struct mpp_rb_loctab_function *loc, void *ctx);
+void mpp_rb_loctab_each_function(struct mpp_rb_loctab *loctab, mpp_rb_loctab_each_function_cb cb, void *ctx);
+
+
+// ======= MAIN DATA STRUCTURE DECLARATIONS ========
+struct mpp_sample {
+    // The backtrace for this sample
+    struct mpp_rb_backtrace *bt;
+    // Sample has a refcount - because it's used both in the heap profiling and in the allocation profiling.
+    int64_t refcount;
+    // How big this allocation was.
+    size_t allocation_size;
+    // How big this object _currently_ is
+    size_t current_size;
+    // Weak reference to what was allocated. Validate that it's alive by consulting the live object table first.
+    VALUE allocated_value_weak;
+    // Next element in the allocation profiling sample list. DO NOT use this in the heap profiling table.
+    struct mpp_sample *next_alloc;
+};
+
+// ======== PROTO SERIALIZATION ROUTINES ========
+struct mpp_pprof_serctx {
+    // Defines the allocation routine & memory arena used by this serialisation context. When the ctx
+    // is destroyed, we free the entire arena, so no other (protobuf) memory needs to be individually
+    // freed.
+    upb_alloc allocator;
+    upb_Arena *arena;
+    // Location table used for looking up fucntion & location IDs to strings.
+    struct mpp_rb_loctab *loctab;
+    // String intern index; recall that holding this object does _not_ require that we have exclusive
+    // use of the underlying string intern table, so it's safe for us to use this in a separate thread.
+    struct mpp_strtab_index *strindex;
+    // The protobuf representation we are building up.
+    perftools_profiles_Profile *profile_proto;
+
+    // We need to keep interned copies of some strings that will wind up in the protobuf output.
+    // This is so that we can put constant values like "allocations" and "count" into our pprof output
+    // (the pprof format requires these strings to be in the string table along with the rest of them)
+    const char *internstr_allocations;
+    const char *internstr_count;
+    const char *internstr_allocation_size;
+    const char *internstr_bytes;
+    const char *internstr_retained_objects;
+    const char *internstr_retained_size;
+};
+
+#define MPP_SAMPLE_TYPE_ALLOCATION 1
+#define MPP_SAMPLE_TYPE_HEAP 2
+
+struct mpp_pprof_serctx *mpp_pprof_serctx_new();
+void mpp_pprof_serctx_destroy(struct mpp_pprof_serctx *ctx);
+int mpp_pprof_serctx_set_loctab(
+    struct mpp_pprof_serctx *ctx, struct mpp_rb_loctab *loctab, char *errbuf, size_t errbuflen
+);
+int mpp_pprof_serctx_add_sample(
+    struct mpp_pprof_serctx *ctx, struct mpp_sample *sample, int sample_type, char *errbuf, size_t errbuflen
+);
+int mpp_pprof_serctx_serialize(
+    struct mpp_pprof_serctx *ctx, char **buf_out, size_t *buflen_out, char *errbuf, size_t errbuflen
+);
+
+// ======== COLLECTOR RUBY CLASS ========
+void mpp_setup_collector_class();
+
+#endif