pf2 0.12.0 → 0.14.0

data/ext/pf2/session.c

@@ -4,6 +4,7 @@
 #include <stdbool.h>
 #include <stdio.h>
 #include <stdlib.h>
+#include <string.h>
 #include <sys/time.h>
 #include <time.h>
 
@@ -25,8 +26,11 @@ static struct pf2_session *global_current_session = NULL;
 static void *sample_collector_thread(void *arg);
 static void drain_ringbuffer(struct pf2_session *session);
 static void sigprof_handler(int sig, siginfo_t *info, void *ucontext);
-bool ensure_sample_capacity(struct pf2_session *session);
 static void pf2_session_stop(struct pf2_session *session);
+static size_t intern_location(struct pf2_session *session, VALUE cme, int lineno);
+static size_t intern_stack(struct pf2_session *session, const size_t *frames, size_t depth);
+static size_t intern_native_stack(struct pf2_session *session, const uintptr_t *frames, size_t depth);
+static bool insert_sample(struct pf2_session *session, const struct pf2_sample *sample);
 
 VALUE
 rb_pf2_session_initialize(int argc, VALUE *argv, VALUE self)
@@ -157,7 +161,7 @@ sample_collector_thread(void *arg)
         // Take samples from the ring buffer
         drain_ringbuffer(session);
 
-        // Sleep for 100 ms
+        // Sleep for 10 ms
         // TODO: Replace with high watermark callback
         struct timespec ts = { .tv_sec = 0, .tv_nsec = 10 * 1000000, }; // 10 ms
         nanosleep(&ts, NULL);
@@ -171,16 +175,12 @@ drain_ringbuffer(struct pf2_session *session)
 {
     struct pf2_sample sample;
     while (pf2_ringbuffer_pop(session->rbuf, &sample) == true) {
-        // Ensure we have capacity before adding a new sample
-        if (!ensure_sample_capacity(session)) {
-            // Failed to expand buffer
+        if (!insert_sample(session, &sample)) {
             atomic_fetch_add_explicit(&session->dropped_sample_count, 1, memory_order_relaxed);
-            PF2_DEBUG_LOG("Failed to expand sample buffer. Dropping sample\n");
-            break;
+            PF2_DEBUG_LOG("Failed to record sample. Dropping sample\n");
+        } else {
+            atomic_fetch_add_explicit(&session->collected_sample_count, 1, memory_order_relaxed);
         }
-
-        session->samples[session->samples_index++] = sample;
-        atomic_fetch_add_explicit(&session->collected_sample_count, 1, memory_order_relaxed);
     }
 }
 
@@ -195,6 +195,12 @@ sigprof_handler(int sig, siginfo_t *info, void *ucontext)
 
     struct pf2_session *session = global_current_session;
 
+    // Pending signals may be delivered even after the session is stopped.
+    // Simply ignore those.
+    if (session == NULL || session->is_running == false) {
+        return;
+    }
+
     // If garbage collection is in progress, don't collect samples.
     if (atomic_load_explicit(&session->is_marking, memory_order_acquire)) {
         PF2_DEBUG_LOG("Dropping sample: Garbage collection is in progress\n");
@@ -231,27 +237,117 @@ sigprof_handler(int sig, siginfo_t *info, void *ucontext)
 #endif
 }
 
-// Ensures that the session's sample array has capacity for at least one more sample
-// Returns true if successful, false if memory allocation failed
-bool
-ensure_sample_capacity(struct pf2_session *session)
+static size_t
+intern_location(struct pf2_session *session, VALUE cme, int lineno)
+{
+    struct pf2_location_key key = { .cme = cme, .lineno = lineno };
+    int absent;
+    khint_t k = pf2_location_table_put(session->location_table, key, &absent);
+    if (k == kh_end(session->location_table)) { return (size_t)-1; }
+    if (absent) {
+        kh_val(session->location_table, k) = kh_size(session->location_table) - 1;
+    }
+    return kh_val(session->location_table, k);
+}
+
+static size_t
+intern_stack(struct pf2_session *session, const size_t *frames, size_t depth)
+{
+    struct pf2_stack_key skey = { .frames = frames, .depth = depth };
+    int absent;
+    khint_t k = pf2_stack_table_put(session->stack_table, skey, &absent);
+    if (k == kh_end(session->stack_table)) { return (size_t)-1; }
+    if (absent) {
+        size_t *copy = NULL;
+        if (depth > 0) {
+            copy = malloc(sizeof(size_t) * depth);
+            // TODO: if allocation fails, remove stack_table entry to avoid dangling stack-local key.
+            if (copy == NULL) return (size_t)-1;
+            memcpy(copy, frames, sizeof(size_t) * depth);
+        }
+        kh_key(session->stack_table, k).frames = copy;
+        kh_key(session->stack_table, k).depth = depth;
+        kh_val(session->stack_table, k) = kh_size(session->stack_table) - 1;
+    }
+    return kh_val(session->stack_table, k);
+}
+
+static size_t
+intern_native_stack(struct pf2_session *session, const uintptr_t *frames, size_t depth)
+{
+    struct pf2_native_stack_key skey = { .frames = frames, .depth = depth };
+    int absent;
+    khint_t k = pf2_native_stack_table_put(session->native_stack_table, skey, &absent);
+    if (k == kh_end(session->native_stack_table)) { return (size_t)-1; }
+    if (absent) {
+        uintptr_t *copy = NULL;
+        if (depth > 0) {
+            copy = malloc(sizeof(uintptr_t) * depth);
+            if (copy == NULL) return (size_t)-1;
+            memcpy(copy, frames, sizeof(uintptr_t) * depth);
+        }
+        kh_key(session->native_stack_table, k).frames = copy;
+        kh_key(session->native_stack_table, k).depth = depth;
+        kh_val(session->native_stack_table, k) = kh_size(session->native_stack_table) - 1;
+    }
+    return kh_val(session->native_stack_table, k);
+}
+
+static bool
+insert_sample(struct pf2_session *session, const struct pf2_sample *sample)
 {
-    // Check if we need to expand
-    if (session->samples_index < session->samples_capacity) {
-        return true;
+    size_t frame_ids[PF2_SAMPLE_MAX_RUBY_DEPTH];
+
+    // Convert each frame to a location
+    for (int i = 0; i < sample->depth; i++) {
+        frame_ids[i] = intern_location(session, sample->cmes[i], sample->linenos[i]);
+        if (frame_ids[i] == (size_t)-1) { return false; }
     }
 
-    // Calculate new size (double the current size)
-    size_t new_capacity = session->samples_capacity * 2;
+    // Obtain stack_id for the array of locations
+    size_t stack_id = intern_stack(session, frame_ids, (size_t)sample->depth);
+    if (stack_id == (size_t)-1) { return false; }
+
+    size_t native_stack_id = intern_native_stack(session, sample->native_stack, sample->native_stack_depth);
+    if (native_stack_id == (size_t)-1) { return false; }
 
-    // Reallocate the array
-    struct pf2_sample *new_samples = realloc(session->samples, new_capacity * sizeof(struct pf2_sample));
-    if (new_samples == NULL) {
-        return false;
+    // Increment the observation count for this stack_id
+    int absent;
+    struct pf2_combined_stack_key ckey = {
+        .ruby_stack_id = stack_id,
+        .native_stack_id = native_stack_id
+    };
+    khint_t k = pf2_sample_table_put(session->sample_table, ckey, &absent);
+    if (k == kh_end(session->sample_table)) { return false; }
+    struct pf2_sample_stats *stats = &kh_val(session->sample_table, k);
+    if (absent) {
+        // This is the first time this stack was observed. Initialize stats.
+        stats->count = 0;
+        stats->timestamps = NULL;
+        stats->thread_ids = NULL;
+        stats->timestamps_count = 0;
+        stats->timestamps_capacity = 0;
     }
 
-    session->samples = new_samples;
-    session->samples_capacity = new_capacity;
+    // count
+    stats->count += 1;
+    // timestamps
+    if (stats->timestamps_count == stats->timestamps_capacity) {
+        size_t new_cap = stats->timestamps_capacity ? stats->timestamps_capacity * 2 : 16;
+        uint64_t *new_ts = realloc(stats->timestamps, sizeof(uint64_t) * new_cap);
+        uintptr_t *new_threads = realloc(stats->thread_ids, sizeof(uintptr_t) * new_cap);
+        if (new_ts == NULL || new_threads == NULL) {
+            free(new_ts);
+            free(new_threads);
+            return false;
+        }
+        stats->timestamps = new_ts;
+        stats->thread_ids = new_threads;
+        stats->timestamps_capacity = new_cap;
+    }
+    stats->timestamps[stats->timestamps_count] = sample->timestamp_ns;
+    stats->thread_ids[stats->timestamps_count] = (uintptr_t)sample->context_pthread;
+    stats->timestamps_count++;
 
     return true;
 }
@@ -298,8 +394,8 @@ pf2_session_stop(struct pf2_session *session)
     if (setitimer(which_timer, &zero_timer, NULL) == -1) {
         rb_raise(rb_eRuntimeError, "Failed to stop timer");
     }
-    global_current_session = NULL;
 #endif
+    global_current_session = NULL;
 
     // Terminate the collector thread
     session->is_running = false;
@@ -356,12 +452,22 @@ pf2_session_alloc(VALUE self)
         rb_raise(rb_eNoMemError, "Failed to allocate memory");
     }
 
-    // samples, samples_index, samples_capacity
-    session->samples_index = 0;
-    session->samples_capacity = 500; // 10 seconds worth of samples at 50 Hz
-    session->samples = malloc(sizeof(struct pf2_sample) * session->samples_capacity);
-    if (session->samples == NULL) {
-        rb_raise(rb_eNoMemError, "Failed to allocate memory");
+    // location_table, stack_table, native_stack_table, sample_table
+    session->location_table = pf2_location_table_init();
+    if (session->location_table == NULL) {
+        rb_raise(rb_eNoMemError, "Failed to allocate location table");
+    }
+    session->stack_table = pf2_stack_table_init();
+    if (session->stack_table == NULL) {
+        rb_raise(rb_eNoMemError, "Failed to allocate stack table");
+    }
+    session->native_stack_table = pf2_native_stack_table_init();
+    if (session->native_stack_table == NULL) {
+        rb_raise(rb_eNoMemError, "Failed to allocate native stack table");
+    }
+    session->sample_table = pf2_sample_table_init();
+    if (session->sample_table == NULL) {
+        rb_raise(rb_eNoMemError, "Failed to allocate stack sample table");
     }
 
     // collected_sample_count, dropped_sample_count
@@ -403,11 +509,11 @@ pf2_session_dmark(void *sess)
         head = (head + 1) % rbuf->size;
     }
 
-    // Iterate over all samples in the samples array and mark them
-    for (size_t i = 0; i < session->samples_index; i++) {
-        sample = &session->samples[i];
-        for (int i = 0; i < sample->depth; i++) {
-            rb_gc_mark(sample->cmes[i]);
+    // Mark Ruby VALUEs stored in location_table keys
+    if (session->location_table) {
+        khint_t k;
+        kh_foreach(session->location_table, k) {
+            rb_gc_mark(kh_key(session->location_table, k).cme);
         }
     }
 
@@ -429,7 +535,33 @@ pf2_session_dfree(void *sess)
 
     pf2_configuration_free(session->configuration);
     pf2_ringbuffer_free(session->rbuf);
-    free(session->samples);
+
+    if (session->sample_table) {
+        khint_t k;
+        kh_foreach(session->sample_table, k) {
+            free(kh_val(session->sample_table, k).timestamps);
+            free(kh_val(session->sample_table, k).thread_ids);
+        }
+        pf2_sample_table_destroy(session->sample_table);
+    }
+    if (session->stack_table) {
+        khint_t k;
+        kh_foreach(session->stack_table, k) {
+            free((void *)kh_key(session->stack_table, k).frames);
+        }
+        pf2_stack_table_destroy(session->stack_table);
+    }
+    if (session->native_stack_table) {
+        khint_t k;
+        kh_foreach(session->native_stack_table, k) {
+            free((void *)kh_key(session->native_stack_table, k).frames);
+        }
+        pf2_native_stack_table_destroy(session->native_stack_table);
+    }
+    if (session->location_table) {
+        pf2_location_table_destroy(session->location_table);
+    }
+
     free(session->collector_thread);
     free(session);
 }
@@ -440,7 +572,6 @@ pf2_session_dsize(const void *sess)
     const struct pf2_session *session = sess;
     return (
         sizeof(struct pf2_session)
-        + sizeof(struct pf2_sample) * session->samples_capacity
         + sizeof(struct pf2_sample) * session->rbuf->size
     );
 }

data/ext/pf2/session.h

@@ -3,6 +3,8 @@
 
 #include <pthread.h>
 #include <stdatomic.h>
+#include <stdint.h>
+#include <limits.h>
 #include <sys/time.h>
 
 #include <ruby.h>
@@ -11,6 +13,152 @@
 #include "ringbuffer.h"
 #include "sample.h"
 
+#include "khashl.h"
+
+// Maps for sample storage
+
+// BEGIN generic helpers
+
+static inline khint_t hash_size_t(size_t v)
+{
+#if SIZE_MAX == UINT_MAX
+    return kh_hash_uint32((khint_t)v);
+#else
+    return kh_hash_uint64((khint64_t)v);
+#endif
+}
+static inline int eq_size_t(size_t a, size_t b) { return a == b; }
+
+// END generic helpers
+
+// BEGIN location_table
+
+struct pf2_location_key {
+    VALUE cme;
+    int lineno;
+};
+static inline khint_t hash_location_key(struct pf2_location_key key)
+{
+    khint_t h = hash_size_t((size_t)key.cme);
+    h ^= (khint_t)key.lineno + 0x9e3779b9U + (h << 6) + (h >> 2);
+    return h;
+}
+static inline int eq_location_key(struct pf2_location_key a, struct pf2_location_key b)
+{
+    return a.cme == b.cme && a.lineno == b.lineno;
+}
+
+// END location_table
+
+// BEGIN stack_table (Ruby stack)
+
+struct pf2_stack_key {
+    const size_t *frames; // pointer to an immutable array of location_ids
+    size_t depth;
+};
+static inline khint_t hash_stack_key(struct pf2_stack_key key)
+{
+    khint_t h = hash_size_t(key.depth);
+    for (size_t i = 0; i < key.depth; i++) {
+        h ^= hash_size_t(key.frames[i]) + 0x9e3779b9U + (h << 6) + (h >> 2);
+    }
+    return h;
+}
+static inline int eq_stack_key(struct pf2_stack_key a, struct pf2_stack_key b)
+{
+    if (a.depth != b.depth) return 0;
+    for (size_t i = 0; i < a.depth; i++) {
+        if (a.frames[i] != b.frames[i]) return 0;
+    }
+    return 1;
+}
+
+// END stack_table
+
+// BEGIN native_stack_table (raw PCs)
+
+struct pf2_native_stack_key {
+    const uintptr_t *frames; // pointer to an immutable array of PCs
+    size_t depth;
+};
+static inline khint_t hash_native_stack_key(struct pf2_native_stack_key key)
+{
+    khint_t h = hash_size_t(key.depth);
+    for (size_t i = 0; i < key.depth; i++) {
+        h ^= kh_hash_uint64((khint64_t)key.frames[i]) + 0x9e3779b9U + (h << 6) + (h >> 2);
+    }
+    return h;
+}
+static inline int eq_native_stack_key(struct pf2_native_stack_key a, struct pf2_native_stack_key b)
+{
+    if (a.depth != b.depth) return 0;
+    for (size_t i = 0; i < a.depth; i++) {
+        if (a.frames[i] != b.frames[i]) return 0;
+    }
+    return 1;
+}
+
+// END native_stack_table
+
+// BEGIN combined_sample_table
+
+struct pf2_combined_stack_key {
+    size_t ruby_stack_id;
+    size_t native_stack_id;
+};
+static inline khint_t hash_combined_stack_key(struct pf2_combined_stack_key key)
+{
+    khint_t h = hash_size_t(key.ruby_stack_id);
+    h ^= hash_size_t(key.native_stack_id) + 0x9e3779b9U + (h << 6) + (h >> 2);
+    return h;
+}
+static inline int eq_combined_stack_key(struct pf2_combined_stack_key a, struct pf2_combined_stack_key b)
+{
+    return a.ruby_stack_id == b.ruby_stack_id && a.native_stack_id == b.native_stack_id;
+}
+
+// END combined_sample_table
+
+struct pf2_sample_stats {
+    // The number of times this sample was observed.
+    size_t count;
+    // Timestamps which this sample was observed. This array's length = # of samples.
+    // TODO: Make timestamp collection optional?
+    uint64_t *timestamps;
+    // Thread ids corresponding to each timestamp.
+    uintptr_t *thread_ids;
+    // timestamps.length
+    size_t timestamps_count;
+    size_t timestamps_capacity;
+};
+
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wunused-function"
+// location table: key = (cme, lineno), val = location_id
+KHASHL_MAP_INIT(static, pf2_location_table, pf2_location_table, struct pf2_location_key, size_t, hash_location_key, eq_location_key)
+// stack table: key = array of location_ids, val = stack_id
+KHASHL_MAP_INIT(static, pf2_stack_table, pf2_stack_table, struct pf2_stack_key, size_t, hash_stack_key, eq_stack_key)
+// native stack table: key = array of PCs, val = native_stack_id
+KHASHL_MAP_INIT(static, pf2_native_stack_table, pf2_native_stack_table, struct pf2_native_stack_key, size_t, hash_native_stack_key, eq_native_stack_key)
+// sample table: key = (ruby_stack_id, native_stack_id), val = aggregated counts/timestamps
+KHASHL_MAP_INIT(static, pf2_sample_table, pf2_sample_table, struct pf2_combined_stack_key, struct pf2_sample_stats, hash_combined_stack_key, eq_combined_stack_key)
+#pragma GCC diagnostic pop
+
+struct pf2_sess_sample {
+    size_t *stack; // array of location_indexes
+    size_t stack_count;
+    size_t *native_stack; // array of location_indexes
+    size_t native_stack_count;
+    uintptr_t ruby_thread_id;
+    uint64_t elapsed_ns;
+};
+
+struct pf2_sess_location {
+    size_t function_index;
+    int32_t lineno;
+    size_t address;
+};
+
 struct pf2_session {
     bool is_running;
 #ifdef HAVE_TIMER_CREATE
@@ -22,9 +170,10 @@ struct pf2_session {
     atomic_bool is_marking; // Whether garbage collection is in progress
     pthread_t *collector_thread;
 
-    struct pf2_sample *samples; // Dynamic array of samples
-    size_t samples_index;
-    size_t samples_capacity;    // Current capacity of the samples array
+    pf2_location_table *location_table;
+    pf2_stack_table *stack_table;
+    pf2_native_stack_table *native_stack_table;
+    pf2_sample_table *sample_table;
 
     struct timespec start_time_realtime;
     struct timespec start_time; // When profiling started

data/lib/pf2/cli.rb

@@ -60,7 +60,6 @@ module Pf2
 
       profile = Marshal.load(File.read(argv[0]))
       report = Pf2::Reporter::FirefoxProfilerSer2.new(profile).emit
-      report = JSON.generate(report)
 
       if options[:output_file]
         File.write(options[:output_file], report)

data/lib/pf2/reporter/firefox_profiler_ser2.rb

@@ -61,7 +61,7 @@ module Pf2
           counters: [],
           threads: thread_reports,
         }
-        FirefoxProfilerSer2.deep_camelize_keys(report)
+        JSON.generate(FirefoxProfilerSer2.deep_camelize_keys(report))
       end
 
       class ThreadReport