memory_tracker 1.0.0

data/lib/memory_tracker/gc_stat.rb

@@ -0,0 +1,63 @@
+module MemoryTracker
+  class GcStat
+    attr_reader :stats
+
+    def initialize(rss, vsize)
+      @stats = GC.stat.merge({ :rss => rss, :vsize => vsize})
+    end
+
+    def ordered_keys
+      @stats.keys.sort
+    end
+
+    def ordered_values
+      ordered_keys.inject([]) do |vals, key|
+        vals << @stats[key]
+        vals
+      end
+    end
+
+    def self.gcdiff(before, after)
+      return {} unless (before && before[:total_allocated_object] && before[:total_freed_object])
+      return {} unless (after && after[:total_allocated_object] && after[:total_freed_object])
+      diff = {}
+      b = before.clone
+      a = after.clone
+      diff[:num_alloc] = a[:total_allocated_object] - b[:total_allocated_object]
+      diff[:num_heaps] = a[:heap_used]
+      [ a, b ].each do |x|
+        x.delete(:heap_increment)
+        x.delete(:heap_length)
+        x.delete(:heap_final_num)
+        x[:in_use] = x.delete(:total_allocated_object) - x.delete(:total_freed_object)
+      end
+      b.each_key do |key|
+        diff[key] = a[key] - b[key]
+      end
+      diff
+    end
+  end
+
+  class GcStatDelta
+    attr_reader :stats
+
+    def initialize(before, after)
+      @after = after
+      @stats = after.stats.inject({}) do |h, (k, v)|
+        h[k] = after.stats[k] - before.stats[k]
+        h
+      end
+    end
+
+    def custom
+      return unless stats[:total_allocated_object] && stats[:total_freed_object]
+      h = {}
+      h[:total_allocated_object] = stats[:total_allocated_object]
+      h[:count] = stats[:count]
+      h[:rss] = stats[:rss]
+      h[:heap_used] = @after.stats[:heap_used]
+      h[:in_use]    = @after.stats[:total_allocated_object] - @after.stats[:total_freed_object]
+      h
+    end
+  end
+end

data/lib/memory_tracker/memory_tracker.rb

@@ -0,0 +1,55 @@
+module MemoryTracker
+  class MemoryTracker
+    include Singleton
+    include Enumerable
+
+    attr_accessor :gcstat_logger
+    
+    def stores
+      @stores ||= {}
+    end
+    
+    def add_store(store)
+      stores[store.name] = store
+    end
+    
+    def start_request(env)
+      @request = Request.new(env)
+    end
+
+    def end_request
+      return unless @request
+      @request.close
+      stores.each { |name, store| store.push(@request) }
+
+      @request = nil
+    end
+
+    def stats(store_name)
+      stores[store_name].stats
+    end
+
+    def self.track_block(*args)
+      self.instance.track_block(*args)
+    end
+
+    def track_block(name, &block)
+      raise ArgumentError unless block_given?
+      before = GC.stat
+      ret = yield
+      after = GC.stat
+      gcstat_logger.debug "gcstat diff for #{name}: #{GcStat.gcdiff(before, after)}"
+      ret
+    end
+
+    private
+    
+    def each_store
+      stores.each
+    end
+    
+    def each
+      each_store
+    end
+  end
+end

data/lib/memory_tracker/middleware.rb

@@ -0,0 +1,20 @@
+module MemoryTracker
+  # Middleware responsability is to initialize and close request objects
+  # object at start and end of HTTP query.
+  class Middleware
+    def initialize(app)
+      @app = app
+    end
+
+    def memory_tracker
+      ::MemoryTracker::MemoryTracker.instance
+    end
+
+    def call(env)
+      memory_tracker.start_request(Env.new(env))
+      status, headers, body = @app.call(env)
+    ensure
+      memory_tracker.end_request
+    end
+  end  
+end

data/lib/memory_tracker/request.rb

@@ -0,0 +1,33 @@
+module MemoryTracker
+  class Request
+    include Sys
+
+    attr_reader :start_gcstat, :end_gcstat
+    attr_reader :gcstat_delta
+    attr_reader :rss, :vsize
+
+    extend Forwardable
+    def_delegators :@env, :path, :controller, :action
+
+    def initialize(env)
+      @env          = env
+      @start_gcstat = GcStat.new(self.class.rss, self.class.vsize)
+    end
+
+    def close
+      @end_gcstat   = GcStat.new(self.class.rss, self.class.vsize)
+      @gcstat_delta = GcStatDelta.new(@start_gcstat, @end_gcstat)
+      self
+    end
+
+    private
+
+    def self.rss
+      rss = ProcTable.ps(Process.pid).rss * 0.004096
+    end
+
+    def self.vsize
+      vsize = ProcTable.ps(Process.pid).vsize * 0.000001
+    end
+  end
+end

data/lib/memory_tracker/stores/gcstat_logfile_store.rb

@@ -0,0 +1,39 @@
+module MemoryTracker
+  module Stores
+    class GcstatLogfileStore
+      def initialize(logger_class, logfile_path)
+        @logger = logger_class.new(logfile_path)
+        @num_lines = 0
+      end
+
+      def name
+        :gcstat_logfile
+      end
+
+      def push(request)
+        @request = request
+
+        write_header if @num_lines % 1000 == 0
+        write_request_log
+        @num_lines += 1
+      end
+
+      def stats
+      end
+
+      private
+
+      def write_header
+        @logger.info "##{@request.end_gcstat.ordered_keys.join(',')}"
+      end
+
+      def write_request_log
+        @logger.info logline
+      end
+
+      def logline
+        @request.end_gcstat.ordered_values.join ','
+      end
+    end
+  end
+end

data/lib/memory_tracker/stores/in_memory_store.rb

@@ -0,0 +1,120 @@
+module MemoryTracker
+  module Stores
+    module InMemoryStore
+      class Manager
+        def initialize(window_length = 60*60*4)
+          @length  = window_length
+          @window1 = StatInterval.new(Time.now - @length/2, @length)
+          @window2 = StatInterval.new(Time.now, @length)
+        end
+
+        def name
+          :memory
+        end
+
+        def push(request)
+          rotate_windows
+          @window1.push(request)
+          @window2.push(request)
+        end
+
+        def stats
+          rotate_windows
+          @window1.stats
+        end
+
+        private
+        def rotate_windows
+          if Time.now > @window1.start_time + @length
+            @window1 = @window2
+            @window2 = StatInterval.new(Time.now, @length)
+          end
+        end
+      end
+
+      class StatInterval
+        attr_reader :start_time, :duration, :size, :stats
+
+        extend Forwardable
+        def_delegators :@stats, :fetch, :each
+
+        include Enumerable
+
+        def initialize(start_time, duration_seconds)
+          @start_time = start_time
+          @duration   = duration_seconds
+          @size       = 0
+          @stats = Stats.new
+        end
+
+        def push(request)
+          @size += 1
+          delta = request.gcstat_delta.stats
+          @stats.increment_action_count(request.controller, request.action)
+          delta.each_key do |attr|
+            @stats.increment_action_attribute(request.controller, request.action, attr, delta[attr])
+          end
+        end
+      end
+
+      class Stats
+        def initialize
+          @data = {}
+        end
+
+        def fetch(controller, action, attr)
+          ca = controller_action_data(controller, action)
+          ca[:gcstat][attr]
+        end
+
+        def count(controller, action)
+          ca = controller_action_data(controller, action)
+          ca[:num]
+        end
+
+        def each(&block)
+          @data.each do |controller, h|
+            h.each do |action, stats|
+              yield [controller, action, stats]
+            end
+          end
+        end
+
+        def increment_action_count(controller, action)
+          ca = controller_action_data(controller, action)
+          ca[:num] += 1
+        end
+
+        def increment_action_attribute(controller, action, attr, value)
+          ca = controller_action_data(controller, action)
+          if ca[:gcstat][attr]
+            ca[:gcstat][attr] += value
+          else
+            ca[:gcstat][attr] = value
+          end
+        end
+
+
+        def to_a
+          a = []
+          @data.each do |controller, actions|
+            actions.each do |action, counters|
+              row = { :controller => controller, :action => action, :num => counters[:num] }
+              row.merge!(counters[:gcstat])
+              a << row
+            end
+          end
+          a
+        end
+
+        private
+
+        def controller_action_data(controller, action)
+          @data[controller] = { action => { :num => 0, :gcstat => {} } } unless @data[controller]
+          @data[controller][action] = { :num => 0, :gcstat => {} }       unless @data[controller][action]
+          @data[controller][action]
+        end
+      end
+    end
+  end
+end

data/lib/memory_tracker/stores/url_logfile_store.rb

@@ -0,0 +1,46 @@
+module MemoryTracker
+  module Stores
+    class UrlLogfileStore
+      def initialize(logger_class, logfile_path)
+        @logger = logger_class.new(logfile_path)
+      end
+
+      def name
+        :url_logfile
+      end
+
+      def push(request)
+        @request = request
+
+        write_request_log
+      end
+
+      def stats
+      end
+
+      private
+
+      def write_request_log
+        @logger.info logline
+      end
+
+      def logline
+        pid = Process.pid
+
+        end_gcstats   = @request.end_gcstat.stats
+        start_gcstats = @request.start_gcstat.stats
+        delta_gcstats = @request.gcstat_delta.stats
+
+        log_msg = "#{Time.now.localtime.strftime("%m-%d %H:%M:%S")} pid:#{'%05d' % pid}"
+        log_msg << " rss=#{'%6.2f' % end_gcstats[:rss]}"
+        log_msg << " vsize=#{'%6.2f' % end_gcstats[:vsize]}"
+
+        if (end_gcstats[:rss] / start_gcstats[:rss] > 1.005) || delta_gcstats[:heap_used] > 0
+          log_msg << " *** #{@request.gcstat_delta.custom.inspect}"
+        end
+
+        log_msg << " #{@request.path}"
+      end
+    end
+  end
+end

data/memory_tracker.gemspec

@@ -0,0 +1,86 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = "memory_tracker"
+  s.version = "1.0.0"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Philippe Le Rohellec"]
+  s.date = "2013-11-15"
+  s.description = "Collect and analyze memory usage data for each individual Rails action controller."
+  s.email = "philippe@lerohellec.com"
+  s.extra_rdoc_files = [
+    "LICENSE.txt",
+    "README.rdoc"
+  ]
+  s.files = [
+    "Gemfile",
+    "Gemfile.lock",
+    "LICENSE.txt",
+    "README.rdoc",
+    "Rakefile",
+    "VERSION",
+    "app/controllers/memory_tracker/dashboards_controller.rb",
+    "app/helpers/memory_tracker/dashboards_helper.rb",
+    "app/views/layouts/memory_tracker.html.erb",
+    "app/views/memory_tracker/dashboards/index.html.erb",
+    "config/routes.rb",
+    "docs/design.rb",
+    "lib/memory_tracker.rb",
+    "lib/memory_tracker/engine.rb",
+    "lib/memory_tracker/env.rb",
+    "lib/memory_tracker/gc_stat.rb",
+    "lib/memory_tracker/memory_tracker.rb",
+    "lib/memory_tracker/middleware.rb",
+    "lib/memory_tracker/request.rb",
+    "lib/memory_tracker/stores/gcstat_logfile_store.rb",
+    "lib/memory_tracker/stores/in_memory_store.rb",
+    "lib/memory_tracker/stores/url_logfile_store.rb",
+    "memory_tracker.gemspec",
+    "spec/lib/memory_tracker_spec.rb",
+    "spec/lib/request_spec.rb",
+    "spec/lib/stores/gcstat_logfile_store_spec.rb",
+    "spec/lib/stores/in_memory_store_spec.rb",
+    "spec/lib/stores/url_logfile_store_spec.rb",
+    "spec/spec_helper.rb"
+  ]
+  s.homepage = "http://github.com/plerohellec/memory_tracker"
+  s.licenses = ["MIT"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = "2.0.3"
+  s.summary = "Rails memory allocations tracker"
+
+  if s.respond_to? :specification_version then
+    s.specification_version = 4
+
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<sys-proctable>, [">= 0"])
+      s.add_development_dependency(%q<debugger>, [">= 0"])
+      s.add_development_dependency(%q<rspec>, ["~> 2.14.0"])
+      s.add_development_dependency(%q<rdoc>, ["~> 3.12"])
+      s.add_development_dependency(%q<bundler>, ["~> 1.0"])
+      s.add_development_dependency(%q<jeweler>, ["~> 1.8.7"])
+      s.add_runtime_dependency(%q<sys-proctable>, [">= 0"])
+    else
+      s.add_dependency(%q<sys-proctable>, [">= 0"])
+      s.add_dependency(%q<debugger>, [">= 0"])
+      s.add_dependency(%q<rspec>, ["~> 2.14.0"])
+      s.add_dependency(%q<rdoc>, ["~> 3.12"])
+      s.add_dependency(%q<bundler>, ["~> 1.0"])
+      s.add_dependency(%q<jeweler>, ["~> 1.8.7"])
+      s.add_dependency(%q<sys-proctable>, [">= 0"])
+    end
+  else
+    s.add_dependency(%q<sys-proctable>, [">= 0"])
+    s.add_dependency(%q<debugger>, [">= 0"])
+    s.add_dependency(%q<rspec>, ["~> 2.14.0"])
+    s.add_dependency(%q<rdoc>, ["~> 3.12"])
+    s.add_dependency(%q<bundler>, ["~> 1.0"])
+    s.add_dependency(%q<jeweler>, ["~> 1.8.7"])
+    s.add_dependency(%q<sys-proctable>, [">= 0"])
+  end
+end
+