bleak_house 3 → 3.0.1

data/lib/bleak_house/analyzer/analyzer.rb

@@ -0,0 +1,143 @@
+
+require 'rubygems'
+require 'fileutils'
+require 'yaml'
+require 'pp'
+require 'ruby-debug'
+
+module BleakHouse
+
+  class Analyzer 
+  
+    MAGIC_KEYS = {
+      -1 => 'timestamp',
+      -2 => 'mem usage/swap',
+      -3 => 'mem usage/real',
+      -4 => 'tag',
+      -5 => 'heap/filled',
+      -6 => 'heap/free'
+    }    
+    
+    CLASS_KEYS = eval('[nil, ' + # skip 0
+      open(
+        File.dirname(__FILE__) + '/../../../ext/bleak_house/logger/snapshot.h'
+      ).read[/\{(.*?)\}/m, 1] + ']')
+            
+    # Parses and correlates a BleakHouse::Logger output file.
+    def self.run(logfile)
+      unless File.exists? logfile
+        puts "No data file found: #{logfile}"
+        exit 
+      end 
+            
+      frames = []
+      last_population = []
+      frame = nil
+      ix = nil
+      
+      puts "Examining objects"
+      
+      LightCsv.foreach(logfile) do |row|        
+      
+        # Stupid is fast
+        row[0] = row[0].to_i if row[0].to_i != 0
+        row[1] = row[1].to_i if row[1].to_i != 0
+        
+        if row[0].to_i < 0          
+          # Get frame meta-information
+          if MAGIC_KEYS[row[0]] == 'timestamp'
+
+            # The frame has ended; process the last one            
+            if frame
+              population = frame['objects'].keys
+              births = population - last_population
+              deaths = last_population - population
+              last_population = population
+  
+              # assign births
+              frame['births'] = frame['objects'].slice(births)
+
+              if final = frames[-2]
+                final['deaths'] = final['objects'].slice(deaths)
+                bsize = final['births'].size
+                dsize = final['deaths'].size
+                final['velocity'] = bsize * 100 / dsize / 100.0
+                puts "  Frame #{frames.size - 1} (#{final['meta']['tag]}) finalized: #{bsize} births, #{dsize} deaths, velocity #{final['velocity']}, population #{final['objects'].size}"
+                final.delete 'objects'
+              end
+            end
+
+            # Set up a new frame
+            frame = {}
+            frames << frame
+            frame['objects'] ||= {}
+            frame['meta'] ||= {}
+            
+            #puts "  Frame #{frames.size} opened"
+          end
+          
+          frame['meta'][MAGIC_KEYS[row[0]]] = row[1]
+        else
+          # Assign live objects
+          frame['objects'][row[1]] = row[0]
+        end
+      end
+            
+      # See what objects are still laying around
+      population = frames.last['objects'].reject do |key, value|
+        frames.first['births'][key] == value
+      end
+
+      # Remove bogus frames
+      frames = frames[1..-3]       
+      
+      total_births = frames.inject(0) do |births, frame|
+        births + frame['births'].size
+      end
+      total_deaths = frames.inject(0) do |deaths, frame|
+        deaths + frame['deaths'].size
+      end
+      
+      puts "#{total_births} total meaningful births, #{total_deaths} total meaningful deaths.\n\n"
+      
+      leakers = {}
+      
+      # Find the sources of the leftover objects in the final population
+      population.each do |id, klass|
+        leaker = frames.detect do |frame|
+          frame['births'][id] == klass
+        end
+        if leaker
+          tag = leaker['meta']['tag']
+          klass = CLASS_KEYS[klass] if klass.is_a? Fixnum
+          leakers[tag] ||= Hash.new(0)
+          leakers[tag][klass] += 1
+        end
+      end
+      
+      # Sort
+      leakers = leakers.map do |tag, value| 
+        [tag, value.sort_by do |klass, count| 
+          -count
+        end]
+      end.sort_by do |tag, value|
+        Hash[*value.flatten].values.inject(0) {|i, v| i - v}
+      end
+      
+      puts "Here are your leaks:"
+      leakers.each do |tag, value|
+        puts "  #{tag} leaked per request:"
+        requests = frames.select do |frame|
+          frame['meta']['tag'] == tag
+        end.size
+        value.each do |klass, count|
+          count = count/requests          
+          puts "    #{count} #{klass}" if count > 0
+        end
+      end      
+      puts "\nBye"
+
+    end
+    
+  end
+end

data/lib/bleak_house/analyzer.rb

@@ -0,0 +1,5 @@
+
+require 'vendor/lightcsv'
+
+require 'bleak_house/support/core_extensions'
+require 'bleak_house/analyzer/analyzer'

data/lib/bleak_house/logger/mem_usage.rb

@@ -0,0 +1,13 @@
+
+module BleakHouse
+  
+  class Logger
+   
+    # Returns an array of the running process's real and virtual memory usage, in kilobytes.
+    def mem_usage
+      a = `ps -o vsz,rss -p #{Process.pid}`.split(/\s+/)[-2..-1].map{|el| el.to_i}
+      [a.first - a.last, a.last]
+    end
+
+  end  
+end

data/lib/bleak_house/logger.rb

@@ -0,0 +1,3 @@
+
+require 'bleak_house/logger/snapshot'
+require 'bleak_house/logger/mem_usage'

data/lib/bleak_house/{action_controller.rb → rails/action_controller.rb}

@@ -1,14 +1,15 @@
 
+# Override ActionController::Base.process and process_with_exception to make sure the request tag for the snapshot gets set as a side-effect of request processing.
 class ActionController::Base
   class << self
     def process_with_bleak_house(request, *args) 
-      BleakHouse.set_request_name request
+      BleakHouse::Rails.set_request_name request
       process_without_bleak_house(request, *args)
     end
     alias_method_chain :process, :bleak_house
     
     def process_with_exception_with_bleak_house(request, *args)
-      BleakHouse.set_request_name request, "/error"
+      BleakHouse::Rails.set_request_name request, "/error"
       process_with_exception_without_bleak_house(request, *args)
     end
     alias_method_chain :process_with_exception, :bleak_house    

data/lib/bleak_house/rails/bleak_house.rb

@@ -0,0 +1,58 @@
+
+module BleakHouse
+  module Rails  
+    class << self
+
+      def last_request_name
+        @@last_request_name
+      end
+      
+      def last_request_name=(obj)
+        @@last_request_name = obj
+      end
+      
+      # Avoid making four more strings on each request.
+      CONTROLLER_KEY = 'controller'
+      ACTION_KEY = 'action'
+      GSUB_SEARCH = '/'
+      GSUB_REPLACEMENT = '__'
+    
+      # Sets the request name on the BleakHouse object to match this Rails request. Called from <tt>ActionController::Base.process</tt>. Assign to <tt>last_request_name</tt> yourself if you are not using BleakHouse within Rails.
+      def set_request_name(request, other = nil)
+        self.last_request_name = "#{
+          request.parameters[CONTROLLER_KEY].gsub(GSUB_SEARCH, GSUB_REPLACEMENT) # mangle namespaced controller names
+        }/#{
+          request.parameters[ACTION_KEY]
+        }/#{
+          request.request_method
+        }#{
+          other
+        }"
+      end
+    
+      def debug(s) #:nodoc:
+        s = "** bleak_house: #{s}"
+        RAILS_DEFAULT_LOGGER.debug s if RAILS_DEFAULT_LOGGER
+      end
+        
+      def warn(s) #:nodoc:
+        s = "** bleak_house: #{s}"
+        if RAILS_DEFAULT_LOGGER
+          RAILS_DEFAULT_LOGGER.warn s
+        else
+          $stderr.puts s
+        end
+      end
+    end
+  
+    LOGFILE = "#{RAILS_ROOT}/log/bleak_house_#{RAILS_ENV}.dump"
+    if File.exists?(LOGFILE)
+      File.rename(LOGFILE, "#{LOGFILE}.old") 
+      warn "renamed old logfile"
+    end
+    
+    WITH_SPECIALS = false
+    
+    MEMLOGGER = Logger.new    
+  end
+end

data/lib/bleak_house/rails/dispatcher.rb

@@ -0,0 +1,19 @@
+
+# Override Dispatcher#prepare and Dispatcher#reset_after_dispatch so that each request makes before-and-after usage snapshots.
+class Dispatcher
+  class << self
+
+    def prepare_application_with_bleak_house
+      prepare_application_without_bleak_house
+      BleakHouse::Rails::MEMLOGGER.snapshot(BleakHouse::Rails::LOGFILE, 'core rails', BleakHouse::Rails::WITH_SPECIALS)
+    end
+    alias_method_chain :prepare_application, :bleak_house
+    
+    def reset_after_dispatch_with_bleak_house
+      BleakHouse::Rails::MEMLOGGER.snapshot(BleakHouse::Rails::LOGFILE, BleakHouse::Rails.last_request_name || 'unknown', BleakHouse::Rails::WITH_SPECIALS)
+      reset_after_dispatch_without_bleak_house
+    end
+    alias_method_chain :reset_after_dispatch, :bleak_house
+
+  end
+end

data/lib/bleak_house/rails.rb

@@ -0,0 +1,6 @@
+
+require 'dispatcher'
+
+require 'bleak_house/rails/action_controller.rb'
+require 'bleak_house/rails/bleak_house.rb'
+require 'bleak_house/rails/dispatcher.rb'

data/lib/bleak_house/{support_methods.rb → support/core_extensions.rb}

@@ -1,4 +1,6 @@
 
+require 'set'
+
 class Array
   alias :time :first
   alias :data :last
@@ -13,10 +15,28 @@ class Array
     
 end
 
+class Hash
+
+  # Similar to the ActiveSupport methods in Rails
+  def slice(keys)
+    keys = Set.new(keys)
+    reject do |key,| 
+      !keys.include?(key)
+    end
+  end  
+  
+  def unslice(keys)
+    keys = Set.new(keys)
+    reject do |key,| 
+      keys.include?(key)
+    end
+  end
+end
+
 class Dir
   def self.descend path, &block
     path = path.split("/") unless path.is_a? Array
-    top = path.shift
+    top = (path.shift or ".")
     Dir.mkdir(top) unless File.exists? top
     Dir.chdir(top) do
       if path.any?
@@ -44,4 +64,7 @@ class Symbol
   def =~ regex
     self.to_s =~ regex
   end
+  def [](*args)
+    self.to_s[*args]
+  end
 end

data/lib/bleak_house.rb

@@ -1,13 +1,7 @@
 
-if ENV['BLEAK_HOUSE']
+require 'bleak_house/logger'
 
-  require 'dispatcher' # rails  
-  
-  require 'bleak_house/bleak_house'
-  require 'bleak_house/mem_logger'
-  require 'bleak_house/dispatcher'
-  require 'bleak_house/action_controller'
-
-  BleakHouse.warn "enabled (log/#{RAILS_ENV}_bleak_house.log) (#{BleakHouse.log_interval} requests per frame)"
-    
+if ENV['RAILS_ENV'] and ENV['BLEAK_HOUSE']
+  require 'bleak_house/rails'
+  BleakHouse::Rails.warn "enabled (log/bleak_house_#{RAILS_ENV}.dump)"    
 end

data/lib/vendor/lightcsv.rb

@@ -0,0 +1,168 @@
+# = LightCsv
+# CSV parser
+#
+# $Id: lightcsv.rb 76 2007-04-15 14:34:23Z tommy $
+# Copyright:: 2007 (C) TOMITA Masahiro <tommy@tmtm.org>
+# License:: Ruby's
+# Homepage:: http://tmtm.org/ja/ruby/lightcsv
+
+require "strscan"
+
+# == CSV のパース
+# 各レコードはカラムを要素とする配列である。
+# レコードの区切りは LF,CR,CRLF のいずれか。
+#
+# 以下が csv.rb と異なる。
+# * 空行は [nil] ではなく [] になる。
+# * 「"」で括られていない空カラムは nil ではなく "" になる。
+#
+# == 例
+# * CSVファイルのレコード毎にブロックを繰り返す。
+#     LightCsv.foreach(filename){|row| ...}
+#   次と同じ。
+#     LightCsv.open(filename){|csv| csv.each{|row| ...}}
+#
+# * CSVファイルの全レコードを返す。
+#     LightCsv.readlines(filename)  # => [[col1,col2,...],...]
+#   次と同じ。
+#     LightCsv.open(filename){|csv| csv.map}
+#
+# * CSV文字列のレコード毎にブロックを繰り返す。
+#     LightCsv.parse("a1,a2,..."){|row| ...}
+#   次と同じ。
+#     LightCsv.new("a1,a2,...").each{|row| ...}
+#
+# * CSV文字列の全レコードを返す。
+#     LightCsv.parse("a1,a2,...")  # => [[a1,a2,...],...]
+#   次と同じ。
+#     LightCsv.new("a1,a2,...").map
+#
+class LightCsv
+  include Enumerable
+
+  # == パースできない形式の場合に発生する例外
+  # InvalidFormat#message は処理できなかった位置から 10バイト文の文字列を返す。
+  class InvalidFormat < RuntimeError; end
+
+  # ファイルの各レコード毎にブロックを繰り返す。
+  # ブロック引数はレコードを表す配列。
+  def self.foreach(filename, &block)
+    self.open(filename) do |f|
+      f.each(&block)
+    end
+  end
+
+  # ファイルの全レコードをレコードの配列で返す。
+  def self.readlines(filename)
+    self.open(filename) do |f|
+      return f.map
+    end
+  end
+
+  # CSV文字列の全レコードをレコードの配列で返す。
+  # ブロックが与えられた場合は、レコード毎にブロックを繰り返す。
+  # ブロック引数はレコードを表す配列。
+  def self.parse(string, &block)
+    unless block
+      return self.new(string).map
+    end
+    self.new(string).each do |row|
+      block.call row
+    end
+    return nil
+  end
+
+  # ファイルをオープンして LightCsv オブジェクトを返す。
+  # ブロックを与えた場合は LightCsv オブジェクトを引数としてブロックを実行する。
+  def self.open(filename, &block)
+    f = File.open(filename)
+    csv = self.new(f)
+    if block
+      begin
+        return block.call(csv)
+      ensure
+        csv.close
+      end
+    else
+      return csv
+    end
+  end
+
+  # LightCsv オブジェクトを生成する。
+  # _src_ は String か IO。
+  def initialize(src)
+    if src.kind_of? String
+      @file = nil
+      @ss = StringScanner.new(src)
+    else
+      @file = src
+      @ss = StringScanner.new("")
+    end
+    @buf = ""
+    @bufsize = 64*1024
+  end
+  attr_accessor :bufsize
+
+  # LightCsv オブジェクトに関連したファイルをクローズする。
+  def close()
+    @file.close if @file
+  end
+
+  # 1レコードを返す。データの最後の場合は nil を返す。
+  # 空行の場合は空配列([])を返す。
+  # 空カラムは「"」で括られているか否かにかかわらず空文字列("")になる。
+  def shift()
+    return nil if @ss.eos? and ! read_next_data
+    cols = []
+    while true
+      if @ss.eos? and ! read_next_data
+        cols << ""
+        break
+      end
+      if @ss.scan(/\"/n)
+        until @ss.scan(/(?:\"\"|[^\"])*\"/n)
+          read_next_data or raise InvalidFormat, @ss.rest[0,10]
+        end
+        cols << @ss.matched.chop.gsub(/\"\"/n, '"')
+      else
+        col = @ss.scan(/[^\",\r\n]*/n)
+        while @ss.eos? and read_next_data
+          col << @ss.scan(/[^\",\r\n]*/n)
+        end
+        cols << col
+      end
+      unless @ss.scan(/,/n)
+        break if @ss.scan(/\r\n/n)
+        unless @ss.rest_size < 2 and read_next_data and @ss.scan(/,/n)
+          break if @ss.scan(/\r\n|\n|\r|\z/n)
+          read_next_data
+          raise InvalidFormat, @ss.rest[0,10]
+        end
+      end
+    end
+    cols.clear if cols.size == 1 and cols.first.empty?
+    cols
+  end
+
+  # 各レコード毎にブロックを繰り返す。
+  def each()
+    while row = shift
+      yield row
+    end
+  end
+
+  # 現在位置以降のレコードの配列を返す。
+  def readlines()
+    return map
+  end
+
+  private
+
+  def read_next_data()
+    if @file and @file.read(@bufsize, @buf)
+      @ss.string = @ss.rest + @buf
+    else
+      nil
+    end
+  end
+end

data/patches/gc.c.patch

@@ -0,0 +1,27 @@
+*** gc.old	2007-09-25 00:27:27.000000000 -0400
+--- gc.c	2007-09-25 00:27:28.000000000 -0400
+***************
+*** 309,314 ****
+--- 309,330 ----
+  static int heaps_length = 0;
+  static int heaps_used   = 0;
+  
++ struct heaps_slot *
++ rb_gc_heap_slots()
++ {
++     return heaps;
++ }
++ 
++ int
++ rb_gc_heaps_used() {
++     return heaps_used;
++ }
++ 
++ int
++ rb_gc_heaps_length() {
++     return heaps_length;
++ }
++ 
+  #define HEAP_MIN_SLOTS 10000
+  static int heap_slots = HEAP_MIN_SLOTS;
+  

data/patches/parse.y.patch

@@ -0,0 +1,16 @@
+*** parse.old	2007-09-25 00:21:43.000000000 -0400
+--- parse.y	2007-09-25 00:23:02.000000000 -0400
+***************
+*** 6166,6171 ****
+--- 6166,6176 ----
+   *                                     :wait2, :$>]
+   */
+  
++ struct st_table * 
++ rb_parse_sym_tbl() {
++   return sym_tbl;
++ }
++ 
+  VALUE
+  rb_sym_all_symbols()
+  {

data/test/misc/direct.rb

@@ -0,0 +1,13 @@
+#!/usr/bin/env ruby-bleak-house
+
+require 'rubygems'
+require 'bleak_house/c'
+$memlogger = BleakHouse::Logger.new
+File.delete($logfile = "/tmp/log") rescue nil
+
+puts 0
+$memlogger.snapshot($logfile, "file", true)
+puts 1
+$memlogger.snapshot($logfile, "file/one", true)
+puts 2
+$memlogger.snapshot($logfile, "file/two", true)

data/test/unit/test_bleak_house.rb

@@ -0,0 +1,34 @@
+
+DIR = File.dirname(__FILE__) + "/../../"
+
+require 'rubygems'
+require 'test/unit'
+require 'yaml'
+require 'ruby-debug'
+Debugger.start
+  
+class BleakHouseTest < Test::Unit::TestCase
+  require "#{DIR}lib/bleak_house/logger"
+
+  SNAPSHOT_FILE = "/tmp/bleak_house"
+  SNAPS = {:c => SNAPSHOT_FILE + ".c.yaml",
+    :ruby => SNAPSHOT_FILE + ".rb.yaml"}
+
+  def setup
+  end
+
+  def test_c_snapshot
+    File.delete SNAPS[:c] rescue nil
+    symbol_count = Symbol.all_symbols.size
+    ::BleakHouse::Logger.new.snapshot(SNAPS[:c], "c_test", false)
+    assert_equal symbol_count, Symbol.all_symbols.size
+    assert File.exist?(SNAPS[:c])
+  end
+  
+  def test_c_raises
+    assert_raises(RuntimeError) do
+      ::BleakHouse::Logger.new.snapshot("/", "c_test", false)
+    end    
+  end
+  
+end

data.tar.gz.sig

@@ -0,0 +1,3 @@
+͌�����LԻ�^�t�L���������f?:��x^�Ɛ���e���΋^�E|l��
i;;��
+��h����9�.��Xe>���k�E�+�@�N��C410���je��k��u�(M�Qd.�Lˮ����0l��m�E�>�F�.���7��B�Ӑ���|���Fh��������v��\s�-�uN�pLq}v�m�ƹ�(B�@D�o]-�N���Sp	�;�.��p�^��5����
+��Zj���aؤ�