rbkit 0.0.1 → 0.1.6

data/ext/rbkit_tracer.h

@@ -9,27 +9,19 @@
 #include "msgpack.h"
 
 // Structure is used to store profiling data
-struct gc_hooks {
+typedef struct _rbkit_logger {
   VALUE hooks[3];
   VALUE enabled;
-  void (*funcs[3])(void *data, int event_index);
-  void *args[3];
   void *data;
   st_table *object_table;
   st_table *str_table;
   VALUE newobj_trace;
   VALUE freeobj_trace;
-  int keep_remains;
   msgpack_sbuffer *sbuf;
   msgpack_packer *msgpacker;
-};
+} rbkit_logger;
 
 char * tracer_string_recv(void *socket);
 int tracer_string_send(void *socket, const char *message);
-void pack_value_object(msgpack_packer *packer, VALUE value);
-void pack_string(msgpack_packer *packer, char *string);
-void pack_timestamp(msgpack_packer *packer);
-void pack_event_header(msgpack_packer *packer, const char *event_type, int map_size);
-void pack_pointer(msgpack_packer *packer, VALUE object_id);
 
 #endif

data/lib/rbkit.rb

@@ -1,57 +1,64 @@
 require "rbkit_tracer"
 require "rbkit/timer"
+require "rbkit/rbkit_gc"
 require "objspace"
 
 # Class implements user friendly interface in pure Ruby for profiler.
 module Rbkit
+  DEFAULT_PUB_PORT = 5555
+  DEFAULT_REQ_PORT = 5556
+
   class Profiler
     attr_accessor :pub_port, :request_port
 
     def initialize(pub_port, request_port)
+      [pub_port, request_port].each{|port| validate_port_range(port) }
       @pub_port = pub_port
       @request_port = request_port
       @profiler_thread = nil
       @stop_thread = false
       @server_running = false
       @gc_stats_timer = Rbkit::Timer.new(5) do
-        data = GC.stat
-        no_of_allocated_pages = data[:heap_length]
-        max_objects_per_page = GC::INTERNAL_CONSTANTS[:HEAP_OBJ_LIMIT]
-        size_of_one_obj = GC::INTERNAL_CONSTANTS[:RVALUE_SIZE]
-        data[:total_heap_size] = no_of_allocated_pages * max_objects_per_page *
-          size_of_one_obj
-        data[:total_memsize] = ObjectSpace.memsize_of_all
-        Rbkit.send_hash_as_event(data, "gc_stats")
+        data = RbkitGC.stat
+        Rbkit.send_hash_as_event(data, Rbkit::EVENT_TYPES[:gc_stats])
       end
       @message_dispatch_timer = Rbkit::Timer.new(1) do
         Rbkit.send_messages
       end
     end
 
-    def start_server(enable_profiling: false)
-      return if @server_running
+    def start_server(enable_object_trace: false, enable_gc_stats: false)
+      if @server_running || !Rbkit.start_stat_server(pub_port, request_port)
+        $stderr.puts "Rbkit server couldn't bind to socket, check if it is already" \
+          " running. Profiling data will not be available."
+        return false
+      end
+      Rbkit.start_stat_tracing if enable_object_trace
+      @enable_gc_stats = enable_gc_stats
+      @server_running = true
       @profiler_thread = Thread.new do
-        Rbkit.start_stat_server(pub_port, request_port)
-        Rbkit.start_stat_tracing if enable_profiling
         loop do
           break if @stop_thread
           incoming_request = Rbkit.poll_for_request
           process_incoming_request(incoming_request)
-          @gc_stats_timer.run
+          @gc_stats_timer.run if @enable_gc_stats
           @message_dispatch_timer.run
           # Let us sleep this thread for a bit, so as other things can run.
           sleep(0.05)
         end
       end
-      @server_running = true
+      at_exit { make_clean_exit(exiting: true) }
+      true
     end
 
     def process_incoming_request(incoming_request)
       case incoming_request
       when "start_memory_profile"
         Rbkit.start_stat_tracing
+        @enable_gc_stats = true
       when "stop_memory_profile"
         Rbkit.stop_stat_tracing
+        @enable_gc_stats = false
       when "trigger_gc"
         GC.start
       when "objectspace_snapshot"
@@ -60,42 +67,57 @@ module Rbkit
     end
 
     def stop_server
-      return if !@server_running
       Rbkit.stop_stat_server
-      @server_running = false
     end
 
-    def make_clean_exit
+    def make_clean_exit(exiting: false)
+      return false if !@server_running
       @stop_thread = true
       stop_server
+      @server_running = false
+      true
     end
-  end
 
-  ########### Rbkit API ###########
+    private
 
-  # Starts the server and enables memory profiling tracepoints
-  def self.start_profiling(pub_port = nil, request_port = nil)
-    @profiler = Rbkit::Profiler.new(pub_port, request_port)
-    @profiler.start_server(enable_profiling: true)
-    at_exit do
-      self.stop_server
+    def validate_port_range(port)
+      raise ArgumentError, 'Invalid port value' unless (1024..65000).include?(port)
     end
   end
 
-  # Just starts the server and waits for instructions.
-  # The client needs to connect to the request_port and send
-  # commands over the wire. The client also needs to connect
-  # to the pub_port and subscribe to the responses from the server.
-  def self.start_server(pub_port = nil, request_port = nil)
-    @profiler = Rbkit::Profiler.new(pub_port, request_port)
+  ########### Rbkit API ###########
+
+  # Starts the Rbkit server and waits for a client to connect and issue
+  # commands to the request_port, until then there's zero performance overhead.
+  # Profiling data is sent asynchronously over pub_port.
+  # This method can be called early in a ruby application so that
+  # whenever profiling needs to be done, the client can attach itself to the
+  # inactive server, do the profiling and leave.
+  def self.start_server(pub_port: DEFAULT_PUB_PORT, request_port: DEFAULT_REQ_PORT)
+    @profiler ||= Rbkit::Profiler.new(pub_port, request_port)
     @profiler.start_server
-    at_exit do
-      self.stop_server
-    end
+  end
+
+  # Starts the server with all tracepoints enabled by default. User can
+  # optionally disable tracepoints using the optional arguments.
+  # This method can be used to profile the startup process of a ruby
+  # application where sending commands from the client to enable
+  # profiling is not feasible.
+  def self.start_profiling(pub_port: DEFAULT_PUB_PORT, request_port: DEFAULT_REQ_PORT,
+                          enable_object_trace: true, enable_gc_stats: true)
+    @profiler ||= Rbkit::Profiler.new(pub_port, request_port)
+    @profiler.start_server(enable_object_trace: enable_object_trace,
+                           enable_gc_stats: enable_gc_stats)
   end
 
   # Stops profiling and brings down the rbkit server if it's running
   def self.stop_server
-    @profiler.make_clean_exit
+    if !@profiler.nil? && @profiler.make_clean_exit
+      @profiler = nil
+      true
+    else
+      $stderr.puts "Cannot stop Rbkit server. Is it running?"
+      false
+    end
   end
 end

data/lib/rbkit/rbkit_gc.rb

@@ -0,0 +1,79 @@
+require "objspace"
+
+module Rbkit
+  class RbkitGC
+    # Returns a standardized hash containing the data
+    # returned by GC.stat
+    # @return [Hash] Keys :
+    #   [count] Count of major and minor GCs so far
+    #   [minor_gc_count] Count of minor GCs
+    #   [major_gc_count] Count of major GCs
+    #   [heap_allocated_pages] Count of allocated pages (heap_eden_pages + heap_tomb_pages)
+    #   [heap_eden_pages] Count of pages which has atleast one live object
+    #   [heap_tomb_pages] Count of pages which don't have any object yet
+    #   [heap_allocatable_pages] Count of pages that will be allocated if Ruby runs out of heap
+    #   [heap_sorted_length] Count of total number of sorted pages (>= heap_allocated_pages + heap_allocatable_pages)
+    #   [heap_live_slots] Count of slots in all pages having live objects
+    #   [heap_free_slots] Count of free slots in all pages
+    #   [heap_final_slots] Count of zombie objects
+    #   [heap_swept_slots] Count of slots swept after last GC
+    #   [old_objects] Count of old generation objects
+    #   [old_objects_limit] Old generation object count after which GC is triggered
+    #   [total_allocated_objects] Number of created objects in the lifetime of the process
+    #   [total_freed_objects] Number of freed objects in the lifetime of the process
+    #   [malloc_increase_bytes] Malloc'ed bytes since last GC
+    #   [malloc_increase_bytes_limit] Minor GC is triggered when malloc_increase_bytes exceeds this value
+    #   [oldmalloc_increase_bytes] Malloc'ed bytes for old objects since last major GC
+    #   [oldmalloc_increase_bytes_limit] Major GC is triggered with oldmalloc_increase_bytes exceeds this value
+    #   [total_heap_size] heap_allocated_pages * max slots per page * size of one slot
+    #   [total_memsize] ObjectSpace.memsize_of_all
+    def self.stat
+      stats = {}
+      data = GC.stat
+
+      stats[:count] = data[:count]
+      stats[:minor_gc_count] = data[:minor_gc_count]
+      stats[:major_gc_count] = data[:major_gc_count]
+
+      if RUBY_VERSION >= "2.2.0"
+        [
+          :heap_allocated_pages, :heap_eden_pages, :heap_tomb_pages,
+          :heap_allocatable_pages, :heap_sorted_length, :heap_live_slots,
+          :heap_free_slots, :heap_final_slots, :heap_swept_slots,
+          :old_objects, :old_objects_limit, :total_allocated_objects,
+          :total_freed_objects, :malloc_increase_bytes,
+          :malloc_increase_bytes_limit, :oldmalloc_increase_bytes,
+          :oldmalloc_increase_bytes_limit
+        ].each do |key|
+          stats[key] = data[key]
+        end
+      elsif RUBY_VERSION >= "2.1.0"
+        stats[:heap_allocated_pages] = data[:heap_used]
+        stats[:heap_eden_pages] = data[:heap_eden_page_length]
+        stats[:heap_tomb_pages] = data[:heap_tomb_page_length]
+        stats[:heap_allocatable_pages] = data[:heap_increment]
+        stats[:heap_sorted_length] = data[:heap_length]
+        stats[:heap_live_slots] = data[:heap_live_slot]
+        stats[:heap_free_slots] = data[:heap_free_slot]
+        stats[:heap_final_slots] = data[:heap_final_slot]
+        stats[:heap_swept_slots] = data[:heap_swept_slot]
+        stats[:old_objects] = data[:old_object]
+        stats[:old_objects_limit] = data[:old_object_limit]
+        stats[:total_allocated_objects] = data[:total_allocated_object]
+        stats[:total_freed_objects] = data[:total_freed_object]
+        stats[:malloc_increase_bytes] = data[:malloc_increase]
+        stats[:malloc_increase_bytes_limit] = data[:malloc_limit]
+        stats[:oldmalloc_increase_bytes] = data[:oldmalloc_increase]
+        stats[:oldmalloc_increase_bytes_limit] = data[:oldmalloc_limit]
+      end
+
+      no_of_allocated_pages = stats[:heap_allocated_pages] rescue 0
+      max_objects_per_page = GC::INTERNAL_CONSTANTS[:HEAP_OBJ_LIMIT]
+      size_of_one_obj = GC::INTERNAL_CONSTANTS[:RVALUE_SIZE]
+      stats[:total_heap_size] = no_of_allocated_pages * max_objects_per_page *
+        size_of_one_obj
+      stats[:total_memsize] = ObjectSpace.memsize_of_all
+      stats
+    end
+  end
+end

data/lib/rbkit/version.rb

@@ -1,3 +1,3 @@
 module Rbkit
-  VERSION = "0.0.1"
+  VERSION = "0.1.6"
 end

data/rbkit.gemspec

@@ -14,6 +14,7 @@ Gem::Specification.new do |s|
   if s.respond_to? :required_rubygems_version=
     s.required_rubygems_version = Gem::Requirement.new(">= 0")
   end
+  s.required_ruby_version = '>= 2.1.0'
   s.authors = ["Hemant Kumar", "Emil Soman", "Kashyap"]
   s.description = %q{Something small for process management}
   s.email = %q{hemant@codemancers.com emil@codemancers.com kashyap@codemancers.com}

data/setup.rb

@@ -0,0 +1,37 @@
+require 'rbconfig'
+require 'fileutils'
+
+class Setup
+  attr_accessor :site_dir, :site_lib_dir, :site_arch_dir
+
+  def initialize
+    @site_dir = RbConfig::CONFIG["sitedir"]
+    @site_lib_dir = RbConfig::CONFIG["sitelibdir"]
+    @site_arch_dir = RbConfig::CONFIG["sitearchdir"]
+  end
+
+  def compile
+    Dir.chdir 'ext' do
+      if File.exist?("Makefile")
+        system("make clean")
+      end
+
+      system("#{Gem.ruby} extconf.rb")
+      system("make")
+    end
+  end
+
+  def copy_files
+    FileUtils.cp_r "lib/.", site_lib_dir, verbose: true
+    ext_path =
+      File.absolute_path "ext/rbkit_tracer.#{RbConfig::MAKEFILE_CONFIG['DLEXT']}"
+    FileUtils.cp_r ext_path, site_arch_dir, verbose: true
+  end
+end
+
+if __FILE__ == $0
+  Setup.new.tap do |setup|
+    setup.compile
+    setup.copy_files
+  end
+end

data/spec/gc_stat_spec.rb

@@ -0,0 +1,31 @@
+require 'spec_helper'
+
+describe 'gc_stat' do
+  let(:stat) { Rbkit::RbkitGC.stat }
+  it 'should have the correct gc stat keys' do
+    expect(stat.keys).to eql [
+        :count,
+        :minor_gc_count,
+        :major_gc_count,
+        :heap_allocated_pages,
+        :heap_eden_pages,
+        :heap_tomb_pages,
+        :heap_allocatable_pages,
+        :heap_sorted_length,
+        :heap_live_slots,
+        :heap_free_slots,
+        :heap_final_slots,
+        :heap_swept_slots,
+        :old_objects,
+        :old_objects_limit,
+        :total_allocated_objects,
+        :total_freed_objects,
+        :malloc_increase_bytes,
+        :malloc_increase_bytes_limit,
+        :oldmalloc_increase_bytes,
+        :oldmalloc_increase_bytes_limit,
+        :total_heap_size,
+        :total_memsize ]
+  end
+end
+

data/spec/hash_event_spec.rb

@@ -0,0 +1,29 @@
+require 'spec_helper'
+require 'msgpack'
+
+describe 'send_hash_as_event' do
+  let(:hash) { {'foo' => 'bar', 123 => "hello world"} }
+  let(:payload) { Rbkit::MESSAGE_FIELDS[:payload] }
+  let(:event_type) { Rbkit::MESSAGE_FIELDS[:event_type] }
+  describe 'when event_type is known' do
+    before do
+      Rbkit.start_profiling(enable_gc_stats: false, enable_object_trace: false)
+      Rbkit.send_hash_as_event(hash, Rbkit::EVENT_TYPES[:gc_stats])
+      packed_message = Rbkit.get_queued_messages
+      @message = MessagePack.unpack packed_message
+      Rbkit.stop_server
+    end
+    it 'should create a custom event with the serialized hash' do
+      expect(@message[payload].first[event_type]).to eql Rbkit::EVENT_TYPES[:gc_stats]
+      expect(@message[payload].first[payload]).to eql hash
+    end
+  end
+
+  describe 'when event_type is not known' do
+    it 'should raise NotImplementedError with a meaningful message' do
+      expect { Rbkit.send_hash_as_event(hash, 100) }
+        .to raise_error(NotImplementedError,
+          "Rbkit : Unpacking of event type '100' not implemented")
+    end
+  end
+end

data/spec/obj_created_spec.rb

@@ -0,0 +1,52 @@
+require 'spec_helper'
+require 'support/have_message_matcher'
+require 'msgpack'
+
+describe "obj_created event" do
+  let(:payload) { Rbkit::MESSAGE_FIELDS[:payload] }
+  let(:class_name) { Rbkit::MESSAGE_FIELDS[:class_name] }
+  let(:event_type) { Rbkit::MESSAGE_FIELDS[:event_type] }
+  let(:object_id) { Rbkit::MESSAGE_FIELDS[:object_id] }
+  let(:foo_info) do
+    @message_list[payload]
+      .select{|x| x[event_type] == Rbkit::EVENT_TYPES[:obj_created] &&
+        x[payload][class_name] =='Foo' }
+  end
+  let(:bar_info) do
+    @message_list[payload]
+      .select{|x| x[event_type] == Rbkit::EVENT_TYPES[:obj_created] &&
+        x[payload][class_name] =='Bar'}
+  end
+  let(:short_lived_bar_info) do
+    @message_list[payload]
+      .select{|x| x[event_type] == Rbkit::EVENT_TYPES[:obj_created] &&
+        x[payload][class_name] =='ShortLivedBar' }
+  end
+  before(:all) do
+    Rbkit.start_profiling(enable_gc_stats: false, enable_object_trace: true)
+    @foo_obj = Foo.new
+    packed_message = Rbkit.get_queued_messages
+    Rbkit.stop_server
+    @message_list  = MessagePack.unpack packed_message
+  end
+  it "should be part of message list" do
+    expect(@message_list).to have_message(Rbkit::EVENT_TYPES[:obj_created])
+  end
+
+  it 'should record objects only once' do
+    expect(foo_info.size).to eql 1
+    expect(bar_info.size).to eql 1
+    expect(short_lived_bar_info.size).to eql 1
+  end
+
+  it 'should record correct object_id' do
+    expect(foo_info.first[payload][object_id]).to eql @foo_obj.object_id
+    expect(bar_info.first[payload][object_id]).to eql @foo_obj.bar.object_id
+  end
+
+  it 'should record correct class_name' do
+    expect(foo_info.first[payload][class_name]).to eql @foo_obj.class.to_s
+    expect(bar_info.first[payload][class_name]).to eql @foo_obj.bar.class.to_s
+  end
+end
+

data/spec/obj_destroyed_spec.rb

@@ -0,0 +1,44 @@
+require 'spec_helper'
+require 'support/have_message_matcher'
+require 'msgpack'
+
+describe "obj_destroyed event" do
+  let(:payload) { Rbkit::MESSAGE_FIELDS[:payload] }
+  let(:event_type) { Rbkit::MESSAGE_FIELDS[:event_type] }
+  let(:object_id) { Rbkit::MESSAGE_FIELDS[:object_id] }
+  let(:class_name) { Rbkit::MESSAGE_FIELDS[:class_name] }
+  let(:foo_info) do
+    @message_list[payload]
+      .select{|x| x[event_type] == Rbkit::EVENT_TYPES[:obj_destroyed] &&
+        x[payload][object_id] == @foo_obj.object_id }
+  end
+  let(:short_lived_bar_info) do
+    short_lived_bar_object_id = @message_list[payload]
+      .find{|x| x[event_type] == Rbkit::EVENT_TYPES[:obj_created] &&
+        x[payload][class_name] == 'ShortLivedBar'}[payload][object_id]
+    @message_list[payload]
+      .select{|x| x[event_type] == Rbkit::EVENT_TYPES[:obj_destroyed] &&
+        x[payload][object_id] == short_lived_bar_object_id}
+  end
+  before(:all) do
+    Rbkit.start_profiling(enable_gc_stats: false, enable_object_trace: true)
+    @foo_obj = Foo.new
+    GC.start
+    packed_message = Rbkit.get_queued_messages
+    Rbkit.stop_server
+    @message_list  = MessagePack.unpack packed_message
+  end
+  it "should be part of message list" do
+    expect(@message_list).to have_message(Rbkit::EVENT_TYPES[:obj_destroyed])
+  end
+
+  it 'should record the deleted object' do
+    expect(short_lived_bar_info.size).to eql 1
+  end
+
+  it 'should not record the live object' do
+    expect(foo_info.size).to eql 0
+  end
+end
+
+