bleak_house 3.1 → 3.2

data/ext/bleak_house/logger/snapshot.h

@@ -0,0 +1,99 @@
+#include "ruby.h"
+#include "env.h"
+#include "node.h"
+#include "st.h"
+#include "re.h"
+#include "util.h"
+#include "intern.h"
+#include "string.h"
+
+/* Histogram of most common Rails classes */
+static char * builtins[] = {
+ "String",
+ "Array",
+ "Hash",
+ "Class",
+ "Regexp",
+ "Proc",
+ "ActionController::Routing::DividerSegment",
+ "Gem::Version",
+ "Gem::Version::Requirement",
+ "Bignum",
+ "Symbol",
+ "Time",
+ "MatchData",
+ "Gem::Specification",
+ "ActionController::Routing::StaticSegment",
+ "Gem::Dependency",
+ "Module",
+ "ActionController::Routing::DynamicSegment",
+ "Range",
+ "ActionController::Routing::Route",
+ "Float",
+ "HashWithIndifferentAccess",
+ "Method",
+ "Enumerable",
+ "Comparable",
+ "Set",
+ "File",
+ "Object",
+ "NameError",
+ "Thread",
+  "_node",
+  "_none",
+  "_blktag",
+  "_undef",
+  "_varmap",
+  "_scope",
+  "_unknown" 
+};
+
+#define BUILTINS_SIZE 30
+#define SPECIALS_SIZE 7
+
+typedef struct RVALUE {
+    union {
+        struct {
+            unsigned long flags; /* always 0 for freed obj */
+            struct RVALUE *next;
+        } free;
+        struct RBasic  basic;
+        struct RObject object;
+        struct RClass  klass;
+        struct RFloat  flonum;
+        struct RString string;
+        struct RArray  array;
+        struct RRegexp regexp;
+        struct RHash   hash;
+        struct RData   data;
+        struct RStruct rstruct;
+        struct RBignum bignum;
+        struct RFile   file;
+        struct RNode   node;
+        struct RMatch  match;
+        struct RVarmap varmap;
+        struct SCOPE   scope;
+    } as;
+} RVALUE;
+
+struct heaps_slot {
+    void *membase;
+    RVALUE *slot;
+    int limit;
+};
+
+typedef struct st_table_entry st_table_entry;
+
+struct st_table_entry {
+    unsigned int hash;
+    st_data_t key;
+    st_data_t record;
+    st_table_entry *next;
+};
+
+struct st_table * rb_parse_sym_tbl();
+struct heaps_slot * rb_gc_heap_slots();
+int rb_gc_heaps_used();
+int rb_gc_heaps_length();
+
+int lookup_builtin(char *);

data/init.rb

@@ -1,2 +1,3 @@
 
-require 'bleak_house'
+puts "Please use the gem version of BleakHouse from now on."
+exit!

data/install.rb

@@ -1 +1,2 @@
-puts File.open("#{File.dirname(__FILE__)}/README").read
+
+puts "Please use the gem version of BleakHouse from now on."

data/lib/bleak_house.rb

@@ -1,13 +1,7 @@
 
-if ENV['BLEAK_HOUSE']
+require 'bleak_house/logger'
 
-  require 'dispatcher' # rails  
-  
-  require 'bleak_house/bleak_house'
-  require 'bleak_house/mem_logger'
-  require 'bleak_house/dispatcher'
-  require 'bleak_house/action_controller'
-
-  BleakHouse.warn "enabled (log/#{RAILS_ENV}_bleak_house.log) (#{BleakHouse.log_interval} requests per frame)"
-    
+if ENV['RAILS_ENV'] and ENV['BLEAK_HOUSE']
+  require 'bleak_house/rails'
+  BleakHouse::Rails.warn "enabled (log/bleak_house_#{RAILS_ENV}.dump)"    
 end

data/lib/bleak_house/analyzer.rb

@@ -0,0 +1,5 @@
+
+require 'ccsv'
+
+require 'bleak_house/support/core_extensions'
+require 'bleak_house/analyzer/analyzer'

data/lib/bleak_house/analyzer/analyzer.rb

@@ -0,0 +1,203 @@
+
+require 'rubygems'
+require 'fileutils'
+require 'yaml'
+require 'pp'
+
+module BleakHouse
+
+  class Analyzer 
+  
+    MAGIC_KEYS = {
+      -1 => 'timestamp',
+      -2 => 'mem usage/swap',
+      -3 => 'mem usage/real',
+      -4 => 'tag',
+      -5 => 'heap/filled',
+      -6 => 'heap/free'
+    }    
+    
+    INITIAL_SKIP = 10
+    
+    CLASS_KEYS = eval('[nil, ' + # skip 0
+      open(
+        File.dirname(__FILE__) + '/../../../ext/bleak_house/logger/snapshot.h'
+      ).read[/\{(.*?)\}/m, 1] + ']')
+    
+    def self.calculate!(frame, index, total)
+      bsize = frame['births'].size
+      dsize = frame['deaths'].size
+      
+      # Avoid divide by zero errors
+      frame['meta']['ratio'] = ratio = (bsize - dsize) / (bsize + dsize + 1).to_f
+      frame['meta']['impact'] = begin
+        Math.log10((bsize - dsize).abs.to_i / 10.0)
+      rescue Errno::ERANGE
+        0
+      end
+      
+      puts "  #{index * 100 / total}%: #{frame['meta']['tag']} (#{bsize} births, #{dsize} deaths, ratio #{format('%.2f', frame['meta']['ratio'])}, impact #{format('%.2f', frame['meta']['impact'])})"
+    end
+    
+    # Parses and correlates a BleakHouse::Logger output file.
+    def self.run(logfile)
+      unless File.exists? logfile
+        puts "No data file found: #{logfile}"
+        exit 
+      end
+      
+      puts "Working..."
+      
+      cachefile = logfile + ".cache"
+      frames = []
+      last_population = []
+      frame = nil
+      ix = nil
+      
+      if File.exist?(cachefile) and File.stat(cachefile).mtime > File.stat(logfile).mtime
+        # Cache is fresh
+        puts "Using cache"
+        frames = Marshal.load(File.open(cachefile).read)
+        
+        puts "#{frames.size} frames"
+        
+        frames[0..-3].each_with_index do |frame, index|
+          calculate!(frame, index + 1, frames.size - 1)
+        end
+        
+      else                        
+        # Rebuild frames
+        total_frames = `grep '^-1' #{logfile} | wc`.to_i
+        
+        puts "#{total_frames} frames"     
+        
+        Ccsv.foreach(logfile) do |row|                
+        
+          # Stupid is fast
+          i = row[0].to_i
+          row[0] = i if i != 0
+          i = row[1].to_i
+          row[1] = i if i != 0
+          
+          if row[0].to_i < 0          
+            # Get frame meta-information
+            if MAGIC_KEYS[row[0]] == 'timestamp'
+  
+              # The frame has ended; process the last one            
+              if frame
+                population = frame['objects'].keys
+                births = population - last_population
+                deaths = last_population - population
+                last_population = population
+    
+                # assign births
+                frame['births'] = frame['objects'].slice(births)
+                
+                # assign deaths to previous frame
+                if final = frames[-2]
+                  final['deaths'] = final['objects'].slice(deaths)
+                  final.delete 'objects'
+                  calculate!(final, frames.size - 1, total_frames)
+                end
+              end
+  
+              # Set up a new frame
+              frame = {}
+              frames << frame
+              frame['objects'] ||= {}
+              frame['meta'] ||= {}
+              
+              #puts "  Frame #{frames.size} opened"
+            end
+            
+            frame['meta'][MAGIC_KEYS[row[0]]] = row[1]
+          else
+            # Assign live objects
+            frame['objects'][row[1]] = row[0]
+          end
+        end
+        
+        # Cache the result
+        File.open(cachefile, 'w') do |f|
+          f.write Marshal.dump(frames)
+        end
+        
+      end
+            
+      # See what objects are still laying around
+      population = frames.last['objects'].reject do |key, value|
+        frames.first['births'][key] == value
+      end
+
+      # Remove bogus frames
+      frames = frames[INITIAL_SKIP..-3]       
+      
+      total_births = frames.inject(0) do |births, frame|
+        births + frame['births'].size
+      end
+      total_deaths = frames.inject(0) do |deaths, frame|
+        deaths + frame['deaths'].size
+      end
+      
+      puts "\n#{total_births} births, #{total_deaths} deaths."
+      
+      leakers = {}
+      
+      # Find the sources of the leftover objects in the final population
+      population.each do |id, klass|
+        leaker = frames.detect do |frame|
+          frame['births'][id] == klass
+        end
+        if leaker
+          tag = leaker['meta']['tag']
+          klass = CLASS_KEYS[klass] if klass.is_a? Fixnum
+          leakers[tag] ||= Hash.new(0)
+          leakers[tag][klass] += 1
+        end
+      end
+      
+      # Sort
+      leakers = leakers.map do |tag, value| 
+        [tag, value.sort_by do |klass, count| 
+          -count
+        end]
+      end.sort_by do |tag, value|
+        Hash[*value.flatten].values.inject(0) {|i, v| i - v}
+      end
+      
+      puts "\nTags sorted by immortal leaks:"
+      leakers.each do |tag, value|
+        requests = frames.select do |frame|
+          frame['meta']['tag'] == tag
+        end.size
+        puts "  #{tag} leaks, averaged over #{requests} requests:"
+        value.each do |klass, count|
+          count = count/requests          
+          puts "    #{count} #{klass}" if count > 0
+        end
+      end
+            
+      impacts = {}
+      
+      frames.each do |frame|
+        impacts[frame['meta']['tag']] ||= []
+        impacts[frame['meta']['tag']] << frame['meta']['impact'] * frame['meta']['ratio']
+      end      
+      impacts = impacts.map do |tag, values|
+        [tag, values.inject(0) {|acc, i| acc + i} / values.size.to_f]
+      end.sort_by do |tag, impact|
+        -impact
+      end
+      
+      puts "\nTags sorted by impact * ratio:"
+      
+      impacts.each do |tag, total|
+        puts "  #{format('%.4f', total).rjust(7)}: #{tag}"
+      end
+
+      puts "\nBye"
+
+    end
+    
+  end
+end

data/lib/bleak_house/logger.rb

@@ -0,0 +1,3 @@
+
+require 'bleak_house/logger/snapshot'
+require 'bleak_house/logger/mem_usage'

data/lib/bleak_house/logger/mem_usage.rb

@@ -0,0 +1,13 @@
+
+module BleakHouse
+  
+  class Logger
+   
+    # Returns an array of the running process's real and virtual memory usage, in kilobytes.
+    def mem_usage
+      a = `ps -o vsz,rss -p #{Process.pid}`.split(/\s+/)[-2..-1].map{|el| el.to_i}
+      [a.first - a.last, a.last]
+    end
+
+  end  
+end

data/lib/bleak_house/rails.rb

@@ -0,0 +1,6 @@
+
+require 'dispatcher'
+
+require 'bleak_house/rails/action_controller.rb'
+require 'bleak_house/rails/bleak_house.rb'
+require 'bleak_house/rails/dispatcher.rb'

data/lib/bleak_house/{action_controller.rb → rails/action_controller.rb}

@@ -1,14 +1,15 @@
 
+# Override ActionController::Base.process and process_with_exception to make sure the request tag for the snapshot gets set as a side-effect of request processing.
 class ActionController::Base
   class << self
     def process_with_bleak_house(request, *args) 
-      BleakHouse.set_request_name request
+      BleakHouse::Rails.set_request_name request
       process_without_bleak_house(request, *args)
     end
     alias_method_chain :process, :bleak_house
     
     def process_with_exception_with_bleak_house(request, *args)
-      BleakHouse.set_request_name request, "/error"
+      BleakHouse::Rails.set_request_name request, "/error"
       process_with_exception_without_bleak_house(request, *args)
     end
     alias_method_chain :process_with_exception, :bleak_house    

data/lib/bleak_house/rails/bleak_house.rb

@@ -0,0 +1,58 @@
+
+module BleakHouse
+  module Rails  
+    class << self
+
+      def last_request_name
+        @@last_request_name
+      end
+      
+      def last_request_name=(obj)
+        @@last_request_name = obj
+      end
+      
+      # Avoid making four more strings on each request.
+      CONTROLLER_KEY = 'controller'
+      ACTION_KEY = 'action'
+      GSUB_SEARCH = '/'
+      GSUB_REPLACEMENT = '__'
+    
+      # Sets the request name on the BleakHouse object to match this Rails request. Called from <tt>ActionController::Base.process</tt>. Assign to <tt>last_request_name</tt> yourself if you are not using BleakHouse within Rails.
+      def set_request_name(request, other = nil)
+        self.last_request_name = "#{
+          request.parameters[CONTROLLER_KEY].gsub(GSUB_SEARCH, GSUB_REPLACEMENT) # mangle namespaced controller names
+        }/#{
+          request.parameters[ACTION_KEY]
+        }/#{
+          request.request_method
+        }#{
+          other
+        }"
+      end
+    
+      def debug(s) #:nodoc:
+        s = "** bleak_house: #{s}"
+        RAILS_DEFAULT_LOGGER.debug s if RAILS_DEFAULT_LOGGER
+      end
+        
+      def warn(s) #:nodoc:
+        s = "** bleak_house: #{s}"
+        if RAILS_DEFAULT_LOGGER
+          RAILS_DEFAULT_LOGGER.warn s
+        else
+          $stderr.puts s
+        end
+      end
+    end
+  
+    LOGFILE = "#{RAILS_ROOT}/log/bleak_house_#{RAILS_ENV}.dump"
+    if File.exists?(LOGFILE)
+      File.rename(LOGFILE, "#{LOGFILE}.old") 
+      warn "renamed old logfile"
+    end
+    
+    WITH_SPECIALS = false
+    
+    MEMLOGGER = Logger.new    
+  end
+end