pf2 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 8e386916b39ce131297e3e29ec4eca107fe9b2ebe98e69db68f1c84091fac724
+  data.tar.gz: 84a658c4f5f336a39a52b2ed58ccb600e7b4f541cd58b4aeea041863db4edfc9
+SHA512:
+  metadata.gz: 16d6addbe2abbf348158dca6f2c7e2d048048a22d8a93b194556195e0c9d7c00c51c50f8b82bd462b263d7db6bc8211da1112d5d436fcfdafba08591b618e901
+  data.tar.gz: 0603d44895aeea77484c6862cbd2660dffa131c4ad5761aaa94e5c67c44bce04491b7ab29078897b095b44d07fc31dbb7351056cbbc005d3f097c2cb83744fc0

data/CHANGELOG.md

@@ -0,0 +1,5 @@
+## [0.1.0] - 2023-10-04
+
+- Initial release
+
+## [Unreleased]

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2023 Daisuke Aritomo
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,26 @@
+# Pf2
+
+A sampling-based profiler for Ruby.
+Works only on a patched version of CRuby (MRI) at the moment.
+
+## Installation
+
+TBD
+
+## Usage
+
+TBD
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/osyoyu/pf2.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,8 @@
+require 'bundler/gem_tasks'
+require 'rake/extensiontask'
+
+task default: %i[]
+
+Rake::ExtensionTask.new 'pf2' do |ext|
+  ext.lib_dir = 'lib/pf2'
+end

data/exe/pf2

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+
+require 'pf2/cli'
+exit Pf2::CLI.run(ARGV)

data/ext/pf2/extconf.rb

@@ -0,0 +1,5 @@
+require 'mkmf'
+
+abort 'missing rb_profile_thread_frames()' unless have_func 'rb_profile_thread_frames'
+
+create_makefile 'pf2/pf2'

data/ext/pf2/pf2.c

@@ -0,0 +1,246 @@
+#include <errno.h>
+#include <signal.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+
+#include <ruby.h>
+#include <ruby/debug.h>
+#include <ruby/thread.h>
+
+#define MAX_BUFFER_SIZE 3000
+
+struct pf2_buffer_t {
+    VALUE framebuffer[MAX_BUFFER_SIZE];
+    int linebuffer[MAX_BUFFER_SIZE];
+};
+
+// Ruby functions
+void Init_pf2(void);
+VALUE rb_start(VALUE self, VALUE debug);
+VALUE rb_stop(VALUE self);
+
+static void pf2_start(void);
+static void pf2_stop(void);
+static void pf2_signal_handler(int signo);
+static void pf2_postponed_job(void *_);
+
+static void pf2_record(struct pf2_buffer_t *buffer);
+static VALUE find_or_create_thread_results(VALUE results, pid_t thread_id);
+
+// Buffer to record rb_profile_frames() results
+struct pf2_buffer_t buffer;
+// The time when the profiler started
+struct timespec initial_time;
+// Debug print?
+bool _debug = false;
+
+void
+Init_pf2(void)
+{
+    VALUE rb_mPf2 = rb_define_module("Pf2");
+    rb_define_module_function(rb_mPf2, "start", rb_start, 1);
+    rb_define_module_function(rb_mPf2, "stop", rb_stop, 0);
+}
+
+VALUE
+rb_start(VALUE self, VALUE debug) {
+    _debug = RTEST(debug);
+
+    /**
+     * {
+     *   sequence: 0,
+     *   threads: {},
+     * }
+     */
+    VALUE results = rb_hash_new();
+    rb_hash_aset(results, ID2SYM(rb_intern_const("sequence")), INT2FIX(0));
+    rb_hash_aset(results, ID2SYM(rb_intern_const("threads")), rb_hash_new());
+
+    rb_iv_set(self, "@results", results);
+
+    pf2_start();
+
+    if (_debug) {
+        rb_funcall(rb_mKernel, rb_intern("puts"), 1, rb_str_new_cstr("[debug] Pf2 started"));
+    }
+
+    return results;
+}
+
+VALUE
+rb_stop(VALUE self) {
+    pf2_stop();
+
+    if (_debug) {
+        rb_funcall(rb_mKernel, rb_intern("puts"), 1, rb_str_new_cstr("[debug] Pf2 stopped"));
+    }
+
+    return rb_iv_get(self, "@results");
+}
+
+static void
+pf2_start(void)
+{
+    clock_gettime(CLOCK_MONOTONIC, &initial_time);
+
+    // Configure timer for every 10 ms
+    // TODO: Make interval configurable
+    struct itimerval timer;
+    timer.it_value.tv_sec = 1;
+    timer.it_value.tv_usec = 0;
+    timer.it_interval.tv_sec = 0;
+    timer.it_interval.tv_usec = 10 * 1000; // 10 ms
+    if (signal(SIGALRM, pf2_signal_handler) == SIG_ERR) {
+        rb_syserr_fail(errno, "Failed to configure profiling timer");
+    };
+    if (setitimer(ITIMER_REAL, &timer, NULL) == -1) {
+        rb_syserr_fail(errno, "Failed to configure profiling timer");
+    };
+}
+
+static void
+pf2_stop(void)
+{
+    struct itimerval timer = { 0 }; // stop
+    setitimer(ITIMER_REAL, &timer, NULL);
+}
+
+// async-signal-safe
+static void
+pf2_signal_handler(int signo)
+{
+    rb_postponed_job_register_one(0, pf2_postponed_job, 0);
+}
+
+static void
+pf2_postponed_job(void *_) {
+    pf2_record(&buffer);
+};
+
+// Buffer structure
+static void
+pf2_record(struct pf2_buffer_t *buffer)
+{
+    // get the current time
+    struct timespec ts;
+    clock_gettime(CLOCK_MONOTONIC, &ts);
+
+    VALUE rb_mPf2 = rb_const_get(rb_cObject, rb_intern("Pf2"));
+    VALUE results = rb_iv_get(rb_mPf2, "@results");
+
+    // Iterate over all Threads
+    VALUE threads = rb_iv_get(rb_mPf2, "@@threads");
+    for (int i = 0; i < RARRAY_LEN(threads); i++) {
+        VALUE thread = rb_ary_entry(threads, i);
+        VALUE thread_status = rb_funcall(thread, rb_intern("status"), 0);
+        if (NIL_P(thread) || thread_status == Qfalse) {
+            // Thread is dead, just ignore
+            continue;
+        }
+
+        pid_t thread_id = NUM2INT(rb_funcall(thread, rb_intern("native_thread_id"), 0));
+        VALUE thread_results = find_or_create_thread_results(results, thread_id);
+        assert(!NIL_P(thread_results));
+
+        // The actual querying
+        int stack_depth = rb_profile_thread_frames(thread, 0, MAX_BUFFER_SIZE, buffer->framebuffer, buffer->linebuffer);
+
+        // TODO: Reimplement Pf2-internal data structures without CRuby
+        // (which will allow us to release the GVL at this point)
+        // rb_thread_call_without_gvl(...);
+
+        VALUE frames_table = rb_hash_lookup(thread_results, ID2SYM(rb_intern_const("frames")));
+        assert(!NIL_P(frames_table));
+        VALUE samples = rb_hash_lookup(thread_results, ID2SYM(rb_intern_const("samples")));
+        assert(!NIL_P(samples));
+
+        // Dig down the stack (top of call stack -> bottom (root))
+        VALUE stack_tree_p = rb_hash_lookup(thread_results, ID2SYM(rb_intern_const("stack_tree")));
+        for (int i = stack_depth - 1; i >= 0; i--) {
+            assert(NIL_P(buffer->framebuffer[i]));
+
+            // Collect & record frame information
+            VALUE frame_obj_id = rb_obj_id(buffer->framebuffer[i]);
+            VALUE frame_table_entry = rb_hash_aref(frames_table, frame_obj_id);
+            if (NIL_P(frame_table_entry)) {
+                frame_table_entry = rb_hash_new();
+                rb_hash_aset(frame_table_entry, ID2SYM(rb_intern_const("full_label")), rb_profile_frame_full_label(buffer->framebuffer[i]));
+                rb_hash_aset(frames_table, frame_obj_id, frame_table_entry);
+            }
+
+            VALUE children = rb_hash_aref(stack_tree_p, ID2SYM(rb_intern_const("children")));
+            VALUE next_node = rb_hash_lookup(children, frame_obj_id);
+            // If this is the first time we see this frame, register it to the stack tree
+            if (NIL_P(next_node)) { // not found
+                next_node = rb_hash_new();
+
+                // Increment sequence
+                VALUE next =
+                    rb_funcall(
+                        rb_hash_lookup(results, ID2SYM(rb_intern_const("sequence"))),
+                        rb_intern("+"),
+                        1,
+                        INT2FIX(1)
+                    );
+                rb_hash_aset(results, ID2SYM(rb_intern_const("sequence")), next);
+
+                rb_hash_aset(next_node, ID2SYM(rb_intern_const("node_id")), INT2FIX(next));
+                rb_hash_aset(next_node, ID2SYM(rb_intern_const("frame_id")), frame_obj_id);
+                rb_hash_aset(next_node, ID2SYM(rb_intern_const("full_label")), rb_profile_frame_full_label(buffer->framebuffer[i]));
+                rb_hash_aset(next_node, ID2SYM(rb_intern_const("children")), rb_hash_new());
+
+                rb_hash_aset(children, frame_obj_id, next_node);
+            }
+
+            VALUE stack_tree_id = rb_hash_aref(next_node, ID2SYM(rb_intern_const("node_id")));
+
+            // If on leaf
+            if (i == 0) {
+                // Record sample
+                VALUE sample = rb_hash_new();
+                rb_hash_aset(sample, ID2SYM(rb_intern_const("stack_tree_id")), stack_tree_id);
+                unsigned long long nsec = (ts.tv_sec - initial_time.tv_sec) * 1000000000 + ts.tv_nsec - initial_time.tv_nsec;
+                rb_hash_aset(sample, ID2SYM(rb_intern_const("timestamp")), ULL2NUM(nsec));
+                rb_ary_push(samples, sample);
+            }
+
+            stack_tree_p = next_node;
+        }
+    }
+}
+
+static VALUE
+find_or_create_thread_results(VALUE results, pid_t thread_id) {
+    assert(!NIL_P(results));
+    assert(!NIL_P(thread));
+
+    VALUE threads = rb_hash_aref(results, ID2SYM(rb_intern_const("threads")));
+    VALUE thread_results = rb_hash_aref(threads, INT2NUM(thread_id));
+    if (NIL_P(thread_results)) {
+        /**
+         * {
+         *   thread_id: 1,
+         *   frames: [],
+         *   stack_tree: {
+         *     node_id: ...,
+         *     children: {}
+         *   },
+         *   samples: [],
+         * }
+         */
+        thread_results = rb_hash_new();
+        rb_hash_aset(thread_results, ID2SYM(rb_intern_const("thread_id")), INT2NUM(thread_id));
+
+        rb_hash_aset(thread_results, ID2SYM(rb_intern_const("frames")), rb_hash_new());
+        VALUE stack_tree = rb_hash_aset(thread_results, ID2SYM(rb_intern_const("stack_tree")), rb_hash_new());
+        rb_hash_aset(stack_tree, ID2SYM(rb_intern_const("node_id")), ID2SYM(rb_intern_const("root")));
+        rb_hash_aset(stack_tree, ID2SYM(rb_intern_const("children")), rb_hash_new());
+        rb_hash_aset(thread_results, ID2SYM(rb_intern_const("samples")), rb_ary_new());
+        rb_hash_aset(thread_results, ID2SYM(rb_intern_const("gvl_timings")), rb_ary_new());
+
+        rb_hash_aset(threads, INT2NUM(thread_id), thread_results);
+    }
+    return thread_results;
+}

data/lib/pf2/cli.rb

@@ -0,0 +1,42 @@
+require 'optparse'
+
+require 'pf2'
+require 'pf2/reporter'
+
+module Pf2
+  class CLI
+    def self.run(...)
+      new.run(...)
+    end
+
+    def run(argv)
+      options = {}
+      option_parser = OptionParser.new do |opts|
+        opts.on('-v', '--version', 'Prints version') do
+          puts Pf2::VERSION
+          exit
+        end
+
+        opts.on('-h', '--help', 'Prints this help') do
+          puts opts
+        end
+
+        opts.on('-o', '--output FILE', 'Output file') do |path|
+          options[:output_file] = path
+        end
+      end
+      option_parser.parse!(argv)
+
+      profile = Marshal.load(IO.binread(ARGV[0]))
+      report = JSON.generate(Pf2::Reporter.new(profile).emit)
+
+      if options[:output_file]
+        File.write(options[:output_file], report)
+      else
+        puts report
+      end
+
+      return 0
+    end
+  end
+end

data/lib/pf2/reporter.rb

@@ -0,0 +1,260 @@
+require 'json'
+
+module Pf2
+  # Generates Firefox Profiler's "processed profile format"
+  # https://github.com/firefox-devtools/profiler/blob/main/docs-developer/processed-profile-format.md
+  class Reporter
+    def initialize(profile)
+      @profile = profile
+    end
+
+    def inspect
+      "" # TODO: provide something better
+    end
+
+    def emit
+      x = {
+        meta: {
+          interval: 10, # ms; TODO: replace with actual interval
+          start_time: 0,
+          process_type: 0,
+          product: 'ruby',
+          stackwalk: 0,
+          version: 19,
+          preprocessed_profile_version: 28,
+          symbolicated: true,
+          categories: [
+            {
+              name: "Logs",
+              color: "grey",
+              subcategories: ["Unused"],
+            },
+            {
+              name: "Ruby",
+              color: "red",
+              subcategories: ["Code"],
+            },
+            {
+              name: "Native",
+              color: "lightblue",
+              subcategories: ["Code"],
+            },
+          ],
+        },
+        libs: [],
+        counters: [],
+        threads: @profile[:threads].values.map {|th| ThreadReport.new(th).emit }
+      }
+      Reporter.deep_camelize_keys(x)
+    end
+
+    class ThreadReport
+      def initialize(thread)
+        @thread = thread
+
+        # Populated in other methods
+        @func_id_map = {}
+        @frame_id_map = {}
+        @stack_tree_id_map = {}
+
+        @string_table = {}
+      end
+
+      def inspect
+        "" # TODO: provide something better
+      end
+
+      def emit
+        func_table = build_func_table
+        frame_table = build_frame_table
+        stack_table = build_stack_table
+        samples = build_samples
+
+        string_table = build_string_table
+
+        {
+          process_type: 'default',
+          process_name: 'ruby',
+          process_startup_time: 0,
+          process_shutdown_time: nil,
+          register_time: 0,
+          unregister_time: nil,
+          paused_ranges: [],
+          name: "Thread (tid: #{@thread[:thread_id]})",
+          is_main_thread: true,
+          is_js_tracer: true,
+          pid: 1,
+          tid: @thread[:thread_id],
+          samples: samples,
+          markers: markers,
+          stack_table: stack_table,
+          frame_table: frame_table,
+          string_array: build_string_table,
+          func_table: func_table,
+          resource_table: {
+            lib: [],
+            name: [],
+            host: [],
+            type: [],
+            length: 0,
+          },
+          native_symbols: [],
+        }
+      end
+
+      def build_samples
+        ret = {
+          event_delay: [],
+          stack: [],
+          time: [],
+          duration: [],
+          # weight: nil,
+          # weight_type: 'samples',
+        }
+
+        @thread[:samples].each do |sample|
+          ret[:stack] << @stack_tree_id_map[sample[:stack_tree_id]]
+          ret[:time] << sample[:timestamp] / 1000000 # ns -> ms
+          ret[:duration] << 1
+          ret[:event_delay] << 0
+        end
+
+        ret[:length] = ret[:stack].length
+        ret
+      end
+
+      def build_frame_table
+        ret = {
+          address: [],
+          category: [],
+          subcategory: [],
+          func: [],
+          inner_window_id: [],
+          implementation: [],
+          line: [],
+          column: [],
+          optimizations: [],
+        }
+
+        @thread[:frames].each.with_index do |(id, frame), i|
+          ret[:address] << nil
+          ret[:category] << 1
+          ret[:subcategory] << 1
+          ret[:func] << i # TODO
+          ret[:inner_window_id] << nil
+          ret[:implementation] << nil
+          ret[:line] << nil
+          ret[:column] << nil
+          ret[:optimizations] << nil
+
+          @frame_id_map[id] = i
+        end
+
+        ret[:length] = ret[:address].length
+        ret
+      end
+
+      def build_func_table
+        ret = {
+          name: [],
+          is_js: [],
+          relevant_for_js: [],
+          resource: [],
+          file_name: [],
+          line_number: [],
+          column_number: [],
+        }
+
+        @thread[:frames].each.with_index do |(id, frame), i|
+          ret[:name] << string_id(frame[:full_label])
+          ret[:is_js] << false
+          ret[:relevant_for_js] << false
+          ret[:resource] << -1
+          ret[:file_name] << nil
+          ret[:line_number] << nil
+          ret[:column_number] << nil
+
+          @func_id_map[id] = i
+        end
+
+        ret[:length] = ret[:name].length
+        ret
+      end
+
+      def build_stack_table
+        ret = {
+          frame: [],
+          category: [],
+          subcategory: [],
+          prefix: [],
+        }
+
+        queue = []
+
+        @thread[:stack_tree][:children].each {|_, c| queue << [nil, c] }
+
+        loop do
+          break if queue.size == 0
+
+          prefix, node = queue.shift
+          ret[:frame] << @frame_id_map[node[:frame_id]]
+          ret[:category] << 1
+          ret[:subcategory] << nil
+          ret[:prefix] << prefix
+
+          # The index of this frame - children can refer to this frame using this index as prefix
+          frame_index = ret[:frame].length - 1
+          @stack_tree_id_map[node[:node_id]] = frame_index
+
+          # Enqueue children nodes
+          node[:children].each {|_, c| queue << [frame_index, c] }
+        end
+
+        ret[:length] = ret[:frame].length
+        ret
+      end
+
+      def build_string_table
+        @string_table.sort_by {|_, v| v}.map {|s| s[0] }
+      end
+
+      def string_id(str)
+        return @string_table[str] if @string_table.has_key?(str)
+        @string_table[str] = @string_table.length
+        @string_table[str]
+      end
+
+      def markers
+        {
+          data: [],
+          name: [],
+          time: [],
+          category: [],
+          length: 0
+        }
+      end
+    end
+
+    # Util functions
+    class << self
+      def snake_to_camel(s)
+        return "isJS" if s == "is_js"
+        return "relevantForJS" if s == "relevant_for_js"
+        return "innerWindowID" if s == "inner_window_id"
+        s.split('_').inject([]) {|buffer, p| buffer.push(buffer.size == 0 ? p : p.capitalize) }.join
+      end
+
+      def deep_camelize_keys(value)
+        case value
+        when Array
+          value.map {|v| deep_camelize_keys(v) }
+        when Hash
+          Hash[value.map {|k, v| [snake_to_camel(k.to_s).to_sym, deep_camelize_keys(v)] }]
+        else
+          value
+        end
+      end
+    end
+  end
+end
+

data/lib/pf2/version.rb

@@ -0,0 +1,3 @@
+module Pf2
+  VERSION = '0.1.0'
+end

data/lib/pf2.rb

@@ -0,0 +1,16 @@
+require_relative 'pf2/pf2'
+require_relative 'pf2/version'
+
+module Pf2
+  class Error < StandardError; end
+
+  @@threads = []
+
+  def self.threads
+    @@threads
+  end
+
+  def self.threads=(th)
+    @@threads = th
+  end
+end

metadata

@@ -0,0 +1,74 @@
+--- !ruby/object:Gem::Specification
+name: pf2
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Daisuke Aritomo
+autorequire:
+bindir: exe
+cert_chain: []
+date: 2023-10-04 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rake-compiler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description:
+email:
+- osyoyu@osyoyu.com
+executables:
+- pf2
+extensions:
+- ext/pf2/extconf.rb
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- LICENSE.txt
+- README.md
+- Rakefile
+- exe/pf2
+- ext/pf2/extconf.rb
+- ext/pf2/pf2.c
+- lib/pf2.rb
+- lib/pf2/cli.rb
+- lib/pf2/reporter.rb
+- lib/pf2/version.rb
+homepage: https://github.com/osyoyu/pf2
+licenses:
+- MIT
+metadata:
+  allowed_push_host: https://rubygems.org
+  homepage_uri: https://github.com/osyoyu/pf2
+  source_code_uri: https://github.com/osyoyu/pf2
+  changelog_uri: https://github.com/osyoyu/pf2/blob/master/CHANGELOG.md
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.3.0.dev
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.5.0.dev
+signing_key:
+specification_version: 4
+summary: Yet another Ruby profiler
+test_files: []