vernier 1.4.0 → 1.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: faf587f758e0b9a68adc377bf27c0f35eb0605703b088e3e122f969f513c864d
-  data.tar.gz: 6cbefbfa78fb3ef1e92c7cfdd5ba0dd43f59b1f63cce6583fe18f4e667fd61db
+  metadata.gz: 2a5b535bebdbcc91aa853b53657893c37531e8acffe5f00df1753ca1c52e7508
+  data.tar.gz: 7aa9509fdfb009956bd2eeab6b74bc8fa832e078d9c8fce92445c82a5699ea27
 SHA512:
-  metadata.gz: 402ce92fbe0f408906fe368e0e2edc637dd68f08476a690877166cb1628068fd6bbfd11878d625276e8f3f785c20adc49ef19494d9e81162d201930b4c64c81f
-  data.tar.gz: 8b7d6c687873ed991b23a508231471b445d6960b47bac2045f1f9eee9f5d82584490a0278f2344d0001e3f5367fffcdfc29fbc284bed448c4155e4381aa26292
+  metadata.gz: 45584b395431f8135928011d03d427f42aa747dda76c4bca3bc063837571e3cc168b3fbc04cd268353e17299f47c7be3c6c486e0885488c6e7b5aba94bcb4190
+  data.tar.gz: d8df8d3a0ac7ce752cfb4c8fc0f57e0af1bf7640a3f228f52a8cd4a32211cbe21686338ca6068257d49df0a2a0206773da89d93aa673a2915c5ea5881d1f4be2

data/examples/fiber_stalls.rb

@@ -0,0 +1,51 @@
+require "bundler/inline"
+gemfile do
+  source 'https://rubygems.org'
+
+  gem "async"
+end
+
+require "async"
+require "async/queue"
+
+def measure
+  x = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+  yield
+  Process.clock_gettime(Process::CLOCK_MONOTONIC) - x
+end
+
+def fib(n)
+  if n < 2
+    n
+  else
+    fib(n-2) + fib(n-1)
+  end
+end
+
+# find fib that takes ~50ms
+fib_i = 50.times.find { |i| measure { fib(i) } >= 0.05 }
+sleep_i = measure { fib(fib_i) }
+
+Async {
+  latch = Async::Queue.new
+
+  workers = [
+    Async {
+      latch.pop # block until ready to measure
+
+      100.times {
+        sleep(sleep_i)
+        # stalls happen here. This worker wants to be scheduled so it can
+        # continue the loop, but will be blocked by another worker executing fib
+      }
+    },
+    Async {
+      latch.pop # block until ready to measure
+
+      100.times { fib(fib_i) }
+    },
+  ]
+
+  2.times { latch << nil }
+  workers.each(&:wait)
+}

data/exe/vernier

@@ -1,5 +1,7 @@
 #!/usr/bin/env ruby
 
+$LOAD_PATH.unshift File.expand_path("../../lib", __FILE__)
+
 require "optparse"
 require "vernier/version"
 
@@ -18,6 +20,9 @@ FLAGS:
         o.on('--output [FILENAME]', String, "output filename") do |s|
           options[:output] = s
         end
+        o.on('--output-dir [DIRECTORY]', String, "output directory (default .)") do |s|
+          options[:output_dir] = s
+        end
         o.on('--interval [MICROSECONDS]', Integer, "sampling interval (default 500)") do |i|
           options[:interval] = i
         end
@@ -52,60 +57,14 @@ FLAGS:
 
     def self.inverted_tree(top, file)
       # Print the inverted tree from a Vernier profile
-      require "json"
-
-      is_gzip = File.binread(file, 2) == "\x1F\x8B".b # check for gzip header
-
-      json = if is_gzip
-        require "zlib"
-        Zlib::GzipReader.open(file) { |gz| gz.read }
-      else
-        File.read file
-      end
-
-      info = JSON.load json
-
-      main = info["threads"].find { |thread| thread["isMainThread"] }
-
-      weight_by_frame = Hash.new(0)
-
-      stack_frames = main["stackTable"]["frame"]
-      frame_table = main["frameTable"]["func"]
-      func_table = main["funcTable"]["name"]
-      string_array = main["stringArray"]
+      require "vernier/parsed_profile"
+      require "vernier/output/top"
+      require "vernier/output/file_listing"
 
-      main["samples"]["stack"].zip(main["samples"]["weight"]).each do |stack, weight|
-        top_frame_index = stack_frames[stack]
-        func_index = frame_table[top_frame_index]
-        string_index = func_table[func_index]
-        str = string_array[string_index]
-        weight_by_frame[str] += weight
-      end
-
-      total = weight_by_frame.values.inject :+
-
-      header = ["Samples", "%", ""]
-      widths = header.map(&:bytesize)
-
-      columns = weight_by_frame.sort_by { |k,v| v }.reverse.first(top).map { |k,v|
-        entry = [v.to_s, ((v / total.to_f) * 100).round(1).to_s, k]
-        entry.each_with_index { |str, i| widths[i] = str.bytesize if widths[i] < str.bytesize }
-        entry
-      }
-
-      print_separator widths
-      print_row header, widths
-      print_separator widths
-      columns.each { print_row(_1, widths) }
-      print_separator widths
-    end
-
-    def self.print_row(list, widths)
-      puts("|" + list.map.with_index { |str, i| " " + str.ljust(widths[i] + 1) }.join("|") + "|")
-    end
+      parsed_profile = Vernier::ParsedProfile.read_file(file)
 
-    def self.print_separator(widths)
-      puts("+" + widths.map { |i| "-" * (i + 2) }.join("+") + "+")
+      puts Vernier::Output::Top.new(parsed_profile).output
+      puts Vernier::Output::FileListing.new(parsed_profile).output
     end
   end
 end

data/ext/vernier/extconf.rb

@@ -2,6 +2,7 @@
 
 require "mkmf"
 
+$CXXFLAGS += " -fvisibility=hidden "
 $CXXFLAGS += " -std=c++14 "
 $CXXFLAGS += " -ggdb3 -Og "
 
@@ -11,5 +12,6 @@ have_struct_member("rb_internal_thread_event_data_t", "thread", ["ruby/thread.h"
 have_func("rb_profile_thread_frames", "ruby/debug.h")
 
 have_func("pthread_setname_np")
+have_func("pthread_condattr_setclock")
 
 create_makefile("vernier/vernier")

data/ext/vernier/memory.cc

@@ -0,0 +1,144 @@
+#include <mutex>
+#include <stdio.h>
+#include <unistd.h>
+#include <vector>
+
+#include "vernier.hh"
+#include "timestamp.hh"
+#include "periodic_thread.hh"
+
+#if defined(__APPLE__)
+
+// Based loosely on https://github.com/zombocom/get_process_mem
+#include <libproc.h>
+
+uint64_t memory_rss() {
+    pid_t pid = getpid();
+
+    struct proc_taskinfo tinfo;
+    int st = proc_pidinfo(pid, PROC_PIDTASKINFO, 0,
+                         &tinfo, sizeof(tinfo));
+
+    if (st != sizeof(tinfo)) {
+        fprintf(stderr, "VERNIER: warning: proc_pidinfo failed\n");
+        return 0;
+    }
+
+    return tinfo.pti_resident_size;
+}
+
+#elif defined(__linux__)
+
+uint64_t memory_rss() {
+    long rss = 0;
+
+    // I'd heard that you shouldn't read /proc/*/smaps with fopen and family,
+    // but maybe it's fine for statm which is much smaller and will almost
+    // certainly fit in any internal buffer.
+    FILE *file = fopen("/proc/self/statm", "r");
+    if (!file) return 0;
+    if (fscanf(file, "%*s%ld", &rss) != 1) {
+        fclose(file);
+        return 0;
+    }
+    fclose(file);
+    return rss * sysconf(_SC_PAGESIZE);
+}
+
+#else
+
+// Unsupported
+uint64_t memory_rss() {
+    return 0;
+}
+
+#endif
+
+VALUE rb_cMemoryTracker;
+
+static VALUE rb_memory_rss(VALUE self) {
+    return ULL2NUM(memory_rss());
+}
+
+class MemoryTracker : public PeriodicThread {
+    public:
+        struct Record {
+            TimeStamp timestamp;
+            uint64_t memory_rss;
+        };
+        std::vector<Record> results;
+        std::mutex mutex;
+
+        MemoryTracker() : PeriodicThread(TimeStamp::from_milliseconds(10)) {
+        }
+
+        void run_iteration() {
+            record();
+        }
+
+        void record() {
+            const std::lock_guard<std::mutex> lock(mutex);
+            results.push_back(Record{TimeStamp::Now(), memory_rss()});
+        }
+};
+
+static const rb_data_type_t rb_memory_tracker_type = {
+    .wrap_struct_name = "vernier/memory_tracker",
+    .function = {
+        //.dmemsize = memory_tracker_memsize,
+        //.dmark = memory_tracker_mark,
+        //.dfree = memory_tracker_free,
+    },
+};
+
+VALUE memory_tracker_start(VALUE self) {
+    MemoryTracker *memory_tracker;
+    TypedData_Get_Struct(self, MemoryTracker, &rb_memory_tracker_type, memory_tracker);
+    memory_tracker->start();
+    return self;
+}
+
+VALUE memory_tracker_stop(VALUE self) {
+    MemoryTracker *memory_tracker;
+    TypedData_Get_Struct(self, MemoryTracker, &rb_memory_tracker_type, memory_tracker);
+
+    memory_tracker->stop();
+    return self;
+}
+
+VALUE memory_tracker_record(VALUE self) {
+    MemoryTracker *memory_tracker;
+    TypedData_Get_Struct(self, MemoryTracker, &rb_memory_tracker_type, memory_tracker);
+    memory_tracker->record();
+    return self;
+}
+
+VALUE memory_tracker_results(VALUE self) {
+    MemoryTracker *memory_tracker;
+    TypedData_Get_Struct(self, MemoryTracker, &rb_memory_tracker_type, memory_tracker);
+    VALUE timestamps = rb_ary_new();
+    VALUE memory = rb_ary_new();
+    for (const auto& record: memory_tracker->results) {
+        rb_ary_push(timestamps, ULL2NUM(record.timestamp.nanoseconds()));
+        rb_ary_push(memory, ULL2NUM(record.memory_rss));
+    }
+    return rb_ary_new_from_args(2, timestamps, memory);
+}
+
+VALUE memory_tracker_alloc(VALUE self) {
+    auto memory_tracker = new MemoryTracker();
+    VALUE obj = TypedData_Wrap_Struct(self, &rb_memory_tracker_type, memory_tracker);
+    return obj;
+}
+
+void Init_memory() {
+  rb_cMemoryTracker = rb_define_class_under(rb_mVernier, "MemoryTracker", rb_cObject);
+  rb_define_alloc_func(rb_cMemoryTracker, memory_tracker_alloc);
+
+  rb_define_method(rb_cMemoryTracker, "start", memory_tracker_start, 0);
+  rb_define_method(rb_cMemoryTracker, "stop", memory_tracker_stop, 0);
+  rb_define_method(rb_cMemoryTracker, "results", memory_tracker_results, 0);
+  rb_define_method(rb_cMemoryTracker, "record", memory_tracker_record, 0);
+
+  rb_define_singleton_method(rb_mVernier, "memory_rss", rb_memory_rss, 0);
+}

data/ext/vernier/periodic_thread.hh

@@ -0,0 +1,141 @@
+#include "ruby.h"
+
+#include <atomic>
+#include "timestamp.hh"
+
+#ifdef __APPLE__
+
+#include <mach/mach.h>
+#include <mach/mach_time.h>
+#include <pthread.h>
+
+// https://developer.apple.com/library/archive/technotes/tn2169/_index.html
+inline void upgrade_thread_priority(pthread_t pthread) {
+    mach_timebase_info_data_t timebase_info;
+    mach_timebase_info(&timebase_info);
+
+    const uint64_t NANOS_PER_MSEC = 1000000ULL;
+    double clock2abs = ((double)timebase_info.denom / (double)timebase_info.numer) * NANOS_PER_MSEC;
+
+    thread_time_constraint_policy_data_t policy;
+    policy.period      = 0;
+
+    // FIXME: I really don't know what these value should be
+    policy.computation = (uint32_t)(5 * clock2abs); // 5 ms of work
+    policy.constraint  = (uint32_t)(10 * clock2abs);
+    policy.preemptible = FALSE;
+
+    int kr = thread_policy_set(pthread_mach_thread_np(pthread_self()),
+                   THREAD_TIME_CONSTRAINT_POLICY,
+                   (thread_policy_t)&policy,
+                   THREAD_TIME_CONSTRAINT_POLICY_COUNT);
+
+    if (kr != KERN_SUCCESS) {
+        mach_error("thread_policy_set:", kr);
+        exit(1);
+    }
+}
+#else
+inline void upgrade_thread_priority(pthread_t pthread) {
+}
+#endif
+
+class PeriodicThread {
+    pthread_t pthread;
+    TimeStamp interval;
+
+    pthread_mutex_t running_mutex = PTHREAD_MUTEX_INITIALIZER;
+    pthread_cond_t running_cv;
+    std::atomic_bool running;
+
+    public:
+        PeriodicThread(TimeStamp interval) : interval(interval), running(false) {
+            pthread_condattr_t attr;
+            pthread_condattr_init(&attr);
+#if HAVE_PTHREAD_CONDATTR_SETCLOCK
+            pthread_condattr_setclock(&attr, CLOCK_MONOTONIC);
+#endif
+            pthread_cond_init(&running_cv, &attr);
+        }
+
+        void set_interval(TimeStamp timestamp) {
+            interval = timestamp;
+        }
+
+        static void *thread_entrypoint(void *arg) {
+            upgrade_thread_priority(pthread_self());
+
+            static_cast<PeriodicThread *>(arg)->run();
+            return NULL;
+        }
+
+        void run() {
+#if HAVE_PTHREAD_SETNAME_NP
+#ifdef __APPLE__
+        pthread_setname_np("Vernier profiler");
+#else
+        pthread_setname_np(pthread_self(), "Vernier profiler");
+#endif
+#endif
+
+            TimeStamp next_sample_schedule = TimeStamp::Now();
+            bool done = false;
+            while (!done) {
+                TimeStamp sample_complete = TimeStamp::Now();
+
+                run_iteration();
+
+                next_sample_schedule += interval;
+
+                if (next_sample_schedule < sample_complete) {
+                    next_sample_schedule = sample_complete + interval;
+                }
+
+                pthread_mutex_lock(&running_mutex);
+                if (running) {
+#if HAVE_PTHREAD_CONDATTR_SETCLOCK
+                    struct timespec next_sample_ts = next_sample_schedule.timespec();
+#else
+                    auto offset = TimeStamp::NowRealtime() - TimeStamp::Now();
+                    struct timespec next_sample_ts = (next_sample_schedule + offset).timespec();
+#endif
+                    int ret;
+                    do {
+                        ret = pthread_cond_timedwait(&running_cv, &running_mutex, &next_sample_ts);
+                    } while(running && ret == EINTR);
+                }
+                done = !running;
+                pthread_mutex_unlock(&running_mutex);
+            }
+        }
+
+        virtual void run_iteration() = 0;
+
+        void start() {
+            pthread_mutex_lock(&running_mutex);
+            if (!running) {
+                running = true;
+
+                int ret = pthread_create(&pthread, NULL, &thread_entrypoint, this);
+                if (ret != 0) {
+                    perror("pthread_create");
+                    rb_bug("VERNIER: pthread_create failed");
+                }
+            }
+            pthread_mutex_unlock(&running_mutex);
+        }
+
+        void stop() {
+            pthread_mutex_lock(&running_mutex);
+            bool was_running = running;
+            if (running) {
+                running = false;
+                pthread_cond_broadcast(&running_cv);
+            }
+            pthread_mutex_unlock(&running_mutex);
+            if (was_running)
+                pthread_join(pthread, NULL);
+            pthread = 0;
+        }
+};
+

data/ext/vernier/signal_safe_semaphore.hh

@@ -0,0 +1,72 @@
+#ifndef SIGNAL_SAFE_SEMAPHORE_HH
+#define SIGNAL_SAFE_SEMAPHORE_HH
+
+#if defined(__APPLE__)
+/* macOS */
+#include <dispatch/dispatch.h>
+#elif defined(__FreeBSD__)
+/* FreeBSD */
+#include <pthread_np.h>
+#include <semaphore.h>
+#else
+/* Linux */
+#include <semaphore.h>
+#include <sys/syscall.h> /* for SYS_gettid */
+#endif
+
+// A basic semaphore built on sem_wait/sem_post
+// post() is guaranteed to be async-signal-safe
+class SignalSafeSemaphore {
+#ifdef __APPLE__
+    dispatch_semaphore_t sem;
+#else
+    sem_t sem;
+#endif
+
+    public:
+
+    SignalSafeSemaphore(unsigned int value = 0) {
+#ifdef __APPLE__
+        sem = dispatch_semaphore_create(value);
+#else
+        sem_init(&sem, 0, value);
+#endif
+    };
+
+    ~SignalSafeSemaphore() {
+#ifdef __APPLE__
+        dispatch_release(sem);
+#else
+        sem_destroy(&sem);
+#endif
+    };
+
+    void wait() {
+#ifdef __APPLE__
+        dispatch_semaphore_wait(sem, DISPATCH_TIME_FOREVER);
+#else
+        // Use sem_timedwait so that we get a crash instead of a deadlock for
+        // easier debugging
+        struct timespec ts = (TimeStamp::NowRealtime() + TimeStamp::from_seconds(5)).timespec();
+
+        int ret;
+        do {
+            ret = sem_timedwait(&sem, &ts);
+        } while (ret && errno == EINTR);
+        if (ret != 0) {
+            rb_bug("VERNIER: sem_timedwait waited over 5 seconds");
+        }
+        assert(ret == 0);
+#endif
+    }
+
+    void post() {
+#ifdef __APPLE__
+        dispatch_semaphore_signal(sem);
+#else
+        sem_post(&sem);
+#endif
+    }
+};
+
+#endif

data/ext/vernier/timestamp.hh

@@ -0,0 +1,138 @@
+#ifndef TIMESTAMP_HH
+#define TIMESTAMP_HH
+
+#include <iostream>
+#include <stdint.h>
+#include <sys/time.h>
+#include <unistd.h>
+
+class TimeStamp {
+    static const uint64_t nanoseconds_per_second = 1000000000;
+    uint64_t value_ns;
+
+    TimeStamp(uint64_t value_ns) : value_ns(value_ns) {}
+
+    public:
+    TimeStamp() : value_ns(0) {}
+
+    static TimeStamp Now() {
+        struct timespec ts;
+        clock_gettime(CLOCK_MONOTONIC, &ts);
+        return TimeStamp(ts.tv_sec * nanoseconds_per_second + ts.tv_nsec);
+    }
+
+    static TimeStamp NowRealtime() {
+        struct timespec ts;
+        clock_gettime(CLOCK_REALTIME, &ts);
+        return TimeStamp(ts.tv_sec * nanoseconds_per_second + ts.tv_nsec);
+    }
+
+    static TimeStamp Zero() {
+        return TimeStamp(0);
+    }
+
+    // SleepUntil a specified timestamp
+    // Highly accurate manual sleep time
+    static void SleepUntil(const TimeStamp &target_time) {
+        if (target_time.zero()) return;
+        struct timespec ts = target_time.timespec();
+
+        int res;
+        do {
+            // do nothing until it's time :)
+            sleep(0);
+        } while (target_time > TimeStamp::Now());
+    }
+
+    static TimeStamp from_seconds(uint64_t s) {
+        return TimeStamp::from_milliseconds(s * 1000);
+    }
+
+    static TimeStamp from_milliseconds(uint64_t ms) {
+        return TimeStamp::from_microseconds(ms * 1000);
+    }
+
+    static TimeStamp from_microseconds(uint64_t us) {
+        return TimeStamp::from_nanoseconds(us * 1000);
+    }
+
+    static TimeStamp from_nanoseconds(uint64_t ns) {
+        return TimeStamp(ns);
+    }
+
+    TimeStamp operator-(const TimeStamp &other) const {
+        TimeStamp result = *this;
+        return result -= other;
+    }
+
+    TimeStamp &operator-=(const TimeStamp &other) {
+        if (value_ns > other.value_ns) {
+            value_ns = value_ns - other.value_ns;
+        } else {
+            // underflow
+            value_ns = 0;
+        }
+        return *this;
+    }
+
+    TimeStamp operator+(const TimeStamp &other) const {
+        TimeStamp result = *this;
+        return result += other;
+    }
+
+    TimeStamp &operator+=(const TimeStamp &other) {
+        uint64_t new_value = value_ns + other.value_ns;
+        value_ns = new_value;
+        return *this;
+    }
+
+    bool operator<(const TimeStamp &other) const {
+        return value_ns < other.value_ns;
+    }
+
+    bool operator<=(const TimeStamp &other) const {
+        return value_ns <= other.value_ns;
+    }
+
+    bool operator>(const TimeStamp &other) const {
+        return value_ns > other.value_ns;
+    }
+
+    bool operator>=(const TimeStamp &other) const {
+        return value_ns >= other.value_ns;
+    }
+
+    bool operator==(const TimeStamp &other) const {
+        return value_ns == other.value_ns;
+    }
+
+    bool operator!=(const TimeStamp &other) const {
+        return value_ns != other.value_ns;
+    }
+
+    uint64_t nanoseconds() const {
+        return value_ns;
+    }
+
+    uint64_t microseconds() const {
+        return value_ns / 1000;
+    }
+
+    bool zero() const {
+        return value_ns == 0;
+    }
+
+    struct timespec timespec() const {
+        struct timespec ts;
+        ts.tv_sec = nanoseconds() / nanoseconds_per_second;
+        ts.tv_nsec = (nanoseconds() % nanoseconds_per_second);
+        return ts;
+    }
+};
+
+inline std::ostream& operator<<(std::ostream& os, const TimeStamp& info) {
+    os << info.nanoseconds() << "ns";
+    return os;
+}
+
+#endif