oink 0.1.2 → 0.9.1

data/lib/oink/reports/request.rb

@@ -0,0 +1,20 @@
+module Oink
+  module Reports
+    class Request
+      include Comparable
+
+      attr_accessor :action, :datetime, :log_lines, :oink_number
+
+      def initialize(action, datetime, log_lines, oink_number)
+        @action = action
+        @datetime = datetime
+        @log_lines = log_lines
+        @oink_number = oink_number
+      end
+
+      def <=>(other)
+        self.oink_number <=> other.oink_number
+      end
+    end
+  end
+end

data/lib/oink/utils/hash_utils.rb

@@ -0,0 +1,9 @@
+module Oink
+  module HashUtils
+
+    def self.to_sorted_array(hsh)
+      hsh.sort{ |a, b| b[1] <=>a [1] }.collect { |k,v| "#{k}: #{v}" }
+    end
+
+  end
+end

data/spec/fakes/fake_application_controller.rb

@@ -0,0 +1,30 @@
+class FakeApplicationController
+
+  def initialize(logger = Logger.new(StringIO.new))
+    @logger = logger
+  end
+
+  class << self
+    attr_accessor :around_filters
+
+    def around_filter method
+      (@around_filters ||= []) << method
+    end
+  end
+
+  def index
+    run_around_filters
+  end
+
+  def logger
+    @logger
+  end
+
+  protected
+  def run_around_filters
+    self.class.around_filters.each { |filter| self.send(filter) { perform_action } }
+  end
+
+  def perform_action
+  end
+end

data/spec/fakes/psuedo_output.rb

@@ -0,0 +1,7 @@
+class PsuedoOutput < Array
+
+  def puts(line)
+    self << line
+  end
+
+end

data/spec/helpers/database.rb

@@ -0,0 +1,20 @@
+require 'active_record'
+
+def setup_memory_database
+  ActiveRecord::Base.establish_connection(
+    :adapter  => 'sqlite3',
+    :database => ':memory:'
+  )
+
+  ActiveRecord::Schema.define(:version => 1) do
+    create_table "pigs", :force => true do |t|
+      t.integer "pen_id"
+      t.string "name"
+      t.boolean "smells"
+    end
+
+    create_table "pens", :force => true do |t|
+      t.string "location"
+    end
+  end
+end

data/spec/{rails → oink/instrumentation}/instance_type_counter_spec.rb

@@ -1,36 +1,38 @@
-require File.expand_path(File.dirname(__FILE__) + "/../spec_helper")
+require File.expand_path(File.dirname(__FILE__) + "/../../spec_helper")
 
-describe Oink::OinkInstanceTypeCounterInstanceMethods do
-  before :each do
+describe Oink::Instrumentation::ActiveRecord do
+  before do
     ActiveRecord::Base.reset_instance_type_count
+    Pig.delete_all
+    Pen.delete_all
   end
-  
+
   describe "hash" do
     it "should not count objects not instantiated" do
       ActiveRecord::Base.instantiated_hash["Pig"].should == nil
     end
-    
+
     it "should include the objects instantiated" do
       Pig.create(:name => "Babe")
       Pig.first
       ActiveRecord::Base.instantiated_hash["Pig"].should == 2
     end
-    
+
     it "should count instantiations for multiple classes" do
       Pig.create(:name => "Babe")
       Pen.create(:location => "Backyard")
       Pig.first
       ActiveRecord::Base.instantiated_hash["Pen"].should == 1
     end
-    
+
     it "should report the total number of objects instantiated" do
       Pig.create(:name => "Babe")
       Pen.create(:location => "Backyard")
       Pig.first
       ActiveRecord::Base.total_objects_instantiated.should == 3
-    end    
+    end
   end
-  
+
   describe "reset" do
     it "should reset the total count" do
       Pig.create(:name => "Babe")

data/spec/oink/instrumentation/memory_snapshot_spec.rb

@@ -0,0 +1,84 @@
+require File.expand_path(File.dirname(__FILE__) + "/../../spec_helper")
+
+module Oink
+  module Instrumentation
+
+    describe StatmMemorySnapshot do
+
+      before do
+        StatmMemorySnapshot.unset_statm_page_size
+      end
+
+      it "returns memory when pagesize is 4096" do
+        pages = 6271
+        statm_file = "#{pages} 1157 411 1 0 763 0\n"
+        File.should_receive(:read).with("/proc/self/statm").and_return(statm_file)
+
+        system_call = mock(SystemCall, :stdout => "4096\n", :success? => true)
+        SystemCall.should_receive(:execute).with('getconf PAGESIZE').and_return(system_call)
+        StatmMemorySnapshot.new.memory.should == (pages * 4)
+      end
+
+      it "falls back to a 4096 if getconf PAGESIZE is not available" do
+        pages = 6271
+        statm_file = "#{pages} 1157 411 1 0 763 0\n"
+        File.should_receive(:read).with("/proc/self/statm").and_return(statm_file)
+        system_call = mock(SystemCall, :stdout => "", :success? => false)
+        SystemCall.should_receive(:execute).with('getconf PAGESIZE').and_return(system_call)
+        StatmMemorySnapshot.new.memory.should == (pages * 4)
+      end
+    end
+
+    describe SmapsMemorySnapshot do
+
+      it "returns a sum of the sizes in the /proc/$$/smaps file" do
+        proc_file = <<-STR
+            Header
+
+            Size: 25
+            Size: 13 trailing
+
+      leading Size: 4
+
+            Footer
+
+            STR
+        File.should_receive(:new).with("/proc/#{$$}/smaps").and_return(proc_file)
+        SmapsMemorySnapshot.new.memory.should == 42
+      end
+
+    end
+
+    describe ProcessStatusMemorySnapshot do
+      it "returns the result of a PS command" do
+        system_call = mock(SystemCall, :stdout => "915")
+        SystemCall.should_receive(:execute).with("ps -o vsz= -p #{$$}").and_return(system_call)
+        ProcessStatusMemorySnapshot.new.memory.should == 915
+      end
+
+      describe "#available?" do
+        it "returns true if ps succeeds" do
+          system_call = mock(SystemCall, :success? => true)
+          SystemCall.should_receive(:execute).with("ps -o vsz= -p #{$$}").and_return(system_call)
+          ProcessStatusMemorySnapshot.available?.should be_true
+        end
+      end
+    end
+
+    describe MemorySnapshot do
+      describe "#memory_snapshot_class" do
+        it "raises an Oink::MemoryDataUnavailableError if not strategies can be found" do
+          [WindowsMemorySnapshot, StatmMemorySnapshot, SmapsMemorySnapshot, ProcessStatusMemorySnapshot].each { |klass| klass.stub(:available? => false) }
+
+          lambda { MemorySnapshot.memory_snapshot_class }.should raise_error(MemoryDataUnavailableError)
+        end
+
+        it "returns the first available memory snapshot strategy" do
+          [WindowsMemorySnapshot, SmapsMemorySnapshot, ProcessStatusMemorySnapshot].each { |klass| klass.stub(:available? => false) }
+          StatmMemorySnapshot.stub(:available? => true)
+          MemorySnapshot.memory_snapshot_class.should == StatmMemorySnapshot
+        end
+      end
+    end
+  end
+end

data/spec/oink/middleware_spec.rb

@@ -0,0 +1,73 @@
+require File.expand_path(File.dirname(__FILE__) + "/../spec_helper")
+require "oink/middleware"
+require 'rack/test'
+
+describe Oink::Middleware do
+  include Rack::Test::Methods
+
+  class SampleApplication
+    def call(env)
+      case env['PATH_INFO']
+      when "/no_pigs"
+      when "/two_pigs"
+        Pig.create(:name => "Babe")
+        Pig.first
+      when "/two_pigs_in_a_pen"
+        Pig.create(:name => "Babe")
+        Pen.create(:location => "Backyard")
+        Pig.first
+      end
+      [200, {}, ""]
+    end
+  end
+
+  let(:app)         { Oink::Middleware.new(SampleApplication.new) }
+
+  before do
+    @log_output = StringIO.new
+    Hodel3000CompliantLogger.stub(:new => Hodel3000CompliantLogger.new(@log_output))
+    Oink::Instrumentation::MemorySnapshot.stub(:memory => 4092)
+    Pig.delete_all
+    Pen.delete_all
+  end
+
+
+  context "support legacy rails log format in transition to oink's own log format" do
+    it "writes rails[pid] to the log even if the app isn't a rails app (for now)" do
+      get "/no_pigs"
+      @log_output.string.should include("rails[#{$$}]")
+    end
+
+    it "writes 'Completed in' after the request has completed" do
+      get "/no_pigs"
+      @log_output.string.should include("Completed in")
+    end
+
+    it "logs the action and controller" do
+      get "/no_pigs", {}, {'action_dispatch.request.parameters' => {'controller' => 'oinkoink', 'action' => 'piggie'}}
+      @log_output.string.should include("Processing oinkoink#piggie")
+    end
+  end
+
+  it "reports 0 totals" do
+    get "/no_pigs"
+    @log_output.string.should include("Instantiation Breakdown: Total: 0")
+  end
+
+  it "reports totals first even if it's a tie" do
+    get "/two_pigs"
+    @log_output.string.should include("Instantiation Breakdown: Total: 2 | Pig: 2")
+  end
+
+  it "reports pigs and pens instantiated" do
+    get "/two_pigs_in_a_pen"
+    @log_output.string.should include("Instantiation Breakdown: Total: 3 | Pig: 2 | Pen: 1")
+  end
+
+  it "logs memory usage" do
+    Oink::Instrumentation::MemorySnapshot.should_receive(:memory).and_return(4092)
+    get "/two_pigs_in_a_pen"
+    @log_output.string.should include("Memory usage: 4092 | PID: #{$$}")
+  end
+
+end

data/spec/oink/rails/instance_type_counter_spec.rb

@@ -0,0 +1,52 @@
+require File.expand_path(File.dirname(__FILE__) + "/../../spec_helper")
+
+class ARCountApplicationController < FakeApplicationController
+  include Oink::InstanceTypeCounter
+
+  def no_pigs
+    run_around_filters
+  end
+
+  def two_pigs_in_a_pen
+    Pig.create!
+    Pig.create!
+    Pen.create!
+    run_around_filters
+  end
+
+  def two_pigs
+    Pig.create!
+    Pig.create!
+    run_around_filters
+  end
+
+end
+
+describe Oink::InstanceTypeCounter do
+
+  before do
+    Pig.delete_all
+    Pen.delete_all
+  end
+
+  let(:log_output)  { StringIO.new }
+  let(:logger)      { Logger.new(log_output) }
+
+  it "reports no AR objects instantiated" do
+    controller = ARCountApplicationController.new(logger)
+    controller.no_pigs
+    log_output.string.should include("Instantiation Breakdown: Total: 0")
+  end
+
+  it "reports AR objects instantiated by type" do
+    controller = ARCountApplicationController.new(logger)
+    controller.two_pigs_in_a_pen
+    log_output.string.should include("Instantiation Breakdown: Total: 3 | Pig: 2 | Pen: 1")
+  end
+
+  it "reports totals first even if its a tie" do
+    controller = ARCountApplicationController.new(logger)
+    controller.two_pigs
+    log_output.string.should include("Instantiation Breakdown: Total: 2 | Pig: 2")
+  end
+end

data/spec/oink/rails/memory_usage_logger_spec.rb

@@ -0,0 +1,23 @@
+require File.expand_path(File.dirname(__FILE__) + "/../../spec_helper")
+
+class MemoryLoggerApplicationController < FakeApplicationController
+  include Oink::MemoryUsageLogger
+end
+
+describe Oink::MemoryUsageLogger do
+  it "should return memory usage info from the snapshot" do
+    Oink::Instrumentation::MemorySnapshot.should_receive("memory").and_return(42)
+    log_output = StringIO.new
+    controller = MemoryLoggerApplicationController.new(Logger.new(log_output))
+    controller.index
+    log_output.string.should include("Memory usage: 42 | PID: #{$$}")
+  end
+
+  it "should log an error message if cannot find a memory snapshot strategy" do
+    Oink::Instrumentation::MemorySnapshot.should_receive("memory").and_raise(Oink::Instrumentation::MemoryDataUnavailableError)
+    controller = MemoryLoggerApplicationController.new
+    lambda {
+      controller.index
+    }.should_not raise_error
+  end
+end

data/spec/oink/reports/active_record_instantiation_report_spec.rb

@@ -0,0 +1,193 @@
+require File.expand_path(File.dirname(__FILE__) + "/../../spec_helper")
+
+module Oink::Reports
+  describe ActiveRecordInstantiationReport do
+
+    describe "short summary with frequent offenders" do
+
+      it "should report actions which exceed the threshold once" do
+        str = <<-STR
+        Feb 01 01:58:29 ey04-s00297 rails[4413]: Processing Users#show (for 92.84.151.171 at 2009-02-01 01:58:29) [GET]
+        Feb 01 01:58:30 ey04-s00297 rails[4413]: Instantiation Breakdown: Total: 51 | User: 51
+        Feb 01 01:58:31 ey04-s00297 rails[4413]: Completed in
+        STR
+
+        io = StringIO.new(str)
+        output = PsuedoOutput.new
+        ActiveRecordInstantiationReport.new(io, 50).print(output)
+        output.should include("1, Users#show")
+      end
+
+      it "should not report actions which do not exceed the threshold" do
+        str = <<-STR
+        Feb 01 01:58:29 ey04-s00297 rails[4413]: Processing Users#show (for 92.84.151.171 at 2009-02-01 01:58:29) [GET]
+        Feb 01 01:58:30 ey04-s00297 rails[4413]: Instantiation Breakdown: Total: 50 | User: 50
+        Feb 01 01:58:31 ey04-s00297 rails[4413]: Completed in
+        STR
+
+        io = StringIO.new(str)
+        output = PsuedoOutput.new
+        ActiveRecordInstantiationReport.new(io, 50).print(output)
+        output.should_not include("1, Users#show")
+      end
+
+      it "should report actions which exceed the threshold multiple times" do
+        str = <<-STR
+        Feb 01 01:58:29 ey04-s00297 rails[4413]: Processing Users#show (for 92.84.151.171 at 2009-02-01 01:58:29) [GET]
+        Feb 01 01:58:30 ey04-s00297 rails[4413]: Instantiation Breakdown: Total: 51 | User: 51
+        Feb 01 01:58:31 ey04-s00297 rails[4413]: Completed in
+        Feb 01 01:58:29 ey04-s00297 rails[4413]: Processing Users#show (for 92.84.151.171 at 2009-02-01 01:58:29) [GET]
+        Feb 01 01:58:30 ey04-s00297 rails[4413]: Instantiation Breakdown: Total: 51 | User: 51
+        Feb 01 01:58:31 ey04-s00297 rails[4413]: Completed in
+        STR
+
+        io = StringIO.new(str)
+        output = PsuedoOutput.new
+        ActiveRecordInstantiationReport.new(io, 50).print(output)
+        output.should include("2, Users#show")
+      end
+
+      it "should order actions by most exceeded" do
+        str = <<-STR
+        Feb 01 01:58:29 ey04-s00297 rails[4413]: Processing Media#show (for 92.84.151.171 at 2009-02-01 01:58:29) [GET]
+        Feb 01 01:58:30 ey04-s00297 rails[4413]: Instantiation Breakdown: Total: 51 | User: 51
+        Feb 01 01:58:31 ey04-s00297 rails[4413]: Completed in
+        Feb 01 01:58:29 ey04-s00297 rails[4413]: Processing Media#show (for 92.84.151.171 at 2009-02-01 01:58:29) [GET]
+        Feb 01 01:58:30 ey04-s00297 rails[4413]: Instantiation Breakdown: Total: 51 | User: 51
+        Feb 01 01:58:31 ey04-s00297 rails[4413]: Completed in
+        Feb 01 01:58:29 ey04-s00297 rails[4413]: Processing Users#show (for 92.84.151.171 at 2009-02-01 01:58:29) [GET]
+        Feb 01 01:58:30 ey04-s00297 rails[4413]: Instantiation Breakdown: Total: 51 | User: 51
+        Feb 01 01:58:31 ey04-s00297 rails[4413]: Completed in
+        STR
+
+        io = StringIO.new(str)
+        output = PsuedoOutput.new
+        ActiveRecordInstantiationReport.new(io, 50).print(output)
+        output[-2].should == "2, Media#show"
+        output[-1].should == "1, Users#show"
+      end
+
+      it "should not be bothered by incomplete requests" do
+        str = <<-STR
+        Feb 01 01:58:29 ey04-s00297 rails[4413]: Processing Media#show (for 92.84.151.171 at 2009-02-01 01:58:29) [GET]
+        Feb 01 01:58:30 ey04-s00297 rails[4413]: Instantiation Breakdown: Total: 24 | User: 24
+        Feb 01 01:58:31 ey04-s00297 rails[4413]: Completed in
+        Feb 01 01:58:29 ey04-s00297 rails[4413]: Processing Media#show (for 92.84.151.171 at 2009-02-01 01:58:29) [GET]
+        Feb 01 01:58:29 ey04-s00297 rails[4413]: Processing Users#show (for 92.84.151.171 at 2009-02-01 01:58:29) [GET]
+        Feb 01 01:58:30 ey04-s00297 rails[4413]: Instantiation Breakdown: Total: 51 | User: 51
+        Feb 01 01:58:31 ey04-s00297 rails[4413]: Completed in
+        STR
+
+        io = StringIO.new(str)
+        output = PsuedoOutput.new
+        ActiveRecordInstantiationReport.new(io, 50).print(output)
+        output.should include("1, Users#show")
+      end
+
+    end
+
+    describe "summary with top 10 offenses" do
+
+      it "should only report requests over threshold" do
+        str = <<-STR
+        Feb 01 01:58:29 ey04-s00297 rails[4413]: Processing Users#show (for 92.84.151.171 at 2009-02-01 01:58:29) [GET]
+        Feb 01 01:58:30 ey04-s00297 rails[4413]: Instantiation Breakdown: Total: 51 | User: 51
+        Feb 01 01:58:31 ey04-s00297 rails[4413]: Completed in
+        STR
+
+        io = StringIO.new(str)
+        output = PsuedoOutput.new
+        ActiveRecordInstantiationReport.new(io, 50).print(output)
+        output.should include("1. Feb 01 01:58:31, 51, Users#show")
+      end
+
+      it "should not include requests which are not over threshold" do
+        str = <<-STR
+        Feb 01 01:58:29 ey04-s00297 rails[4413]: Processing Users#show (for 92.84.151.171 at 2009-02-01 01:58:29) [GET]
+        Feb 01 01:58:30 ey04-s00297 rails[4413]: Instantiation Breakdown: Total: 50 | User: 50
+        Feb 01 01:58:31 ey04-s00297 rails[4413]: Completed in
+        STR
+
+        io = StringIO.new(str)
+        output = PsuedoOutput.new
+        ActiveRecordInstantiationReport.new(io, 50).print(output)
+        output.should_not include("1. Feb 01 01:58:31, 50, Users#show")
+      end
+
+      it "should order offenses from biggest to smallest" do
+        str = <<-STR
+        Feb 01 01:58:29 ey04-s00297 rails[4413]: Processing DetailsController#show (for 92.84.151.171 at 2009-02-01 01:58:29) [GET]
+        Feb 01 01:58:30 ey04-s00297 rails[4413]: Instantiation Breakdown: Total: 75 | User: 75
+        Feb 01 01:58:31 ey04-s00297 rails[4413]: Completed in
+        Feb 01 01:58:32 ey04-s00297 rails[4413]: Processing MediaController#show (for 92.84.151.171 at 2009-02-01 01:58:29) [GET]
+        Feb 01 01:58:33 ey04-s00297 rails[4413]: Instantiation Breakdown: Total: 100 | User: 100
+        Feb 01 01:58:34 ey04-s00297 rails[4413]: Completed in
+        STR
+
+        io = StringIO.new(str)
+        output = PsuedoOutput.new
+        ActiveRecordInstantiationReport.new(io, 50).print(output)
+        output[4].should == "1. Feb 01 01:58:34, 100, MediaController#show"
+        output[5].should == "2. Feb 01 01:58:31, 75, DetailsController#show"
+      end
+
+    end
+
+    describe "verbose format" do
+      it "should print the full lines of actions exceeding the threshold" do
+        str = <<-STR
+        Feb 01 01:58:32 ey04-s00297 rails[4413]: Processing MediaController#show (for 92.84.151.171 at 2009-02-01 01:58:29) [GET]
+        Feb 01 01:58:33 ey04-s00297 rails[4413]: Instantiation Breakdown: Total: 100 | User: 100
+        Feb 01 01:58:34 ey04-s00297 rails[4413]: Completed in
+        STR
+        io = StringIO.new(str)
+        output = PsuedoOutput.new
+        ActiveRecordInstantiationReport.new(io, 50, :format => :verbose).print(output)
+        output[3..5].should == str.split("\n")[0..2].map { |o| o.strip }
+      end
+
+      it "should handle actions which do not complete properly" do
+        str = <<-STR
+        Feb 01 01:58:29 ey04-s00297 rails[4413]: Processing Media#show (for 92.84.151.171 at 2009-02-01 01:58:29) [GET]
+        Feb 01 01:58:30 ey04-s00297 rails[4413]: Instantiation Breakdown: Total: 24 | User: 24
+        Feb 01 01:58:31 ey04-s00297 rails[4413]: Completed in
+        Feb 01 01:58:29 ey04-s00297 rails[4413]: Processing Media#show (for 92.84.151.171 at 2009-02-01 01:58:29) [GET]
+        Feb 01 01:58:29 ey04-s00297 rails[4413]: Processing Users#show (for 92.84.151.171 at 2009-02-01 01:58:29) [GET]
+        Feb 01 01:58:30 ey04-s00297 rails[4413]: Instantiation Breakdown: Total: 51 | User: 51
+        Feb 01 01:58:31 ey04-s00297 rails[4413]: Completed in
+        STR
+
+        io = StringIO.new(str)
+        output = PsuedoOutput.new
+        ActiveRecordInstantiationReport.new(io, 50, :format => :verbose).print(output)
+        output[3..5].should == str.split("\n")[4..6].map { |o| o.strip }
+      end
+    end
+
+    describe "multiple io streams" do
+      it "should accept multiple files" do
+
+        str1 = <<-STR
+        Feb 01 01:58:32 ey04-s00297 rails[4413]: Processing MediaController#show (for 92.84.151.171 at 2009-02-01 01:58:29) [GET]
+        Feb 01 01:58:33 ey04-s00297 rails[4413]: Instantiation Breakdown: Total: 100 | User: 100
+        Feb 01 01:58:34 ey04-s00297 rails[4413]: Completed in
+        STR
+
+        str2 = <<-STR
+        Feb 01 01:58:32 ey04-s00297 rails[4413]: Processing MediaController#show (for 92.84.151.171 at 2009-02-01 01:58:29) [GET]
+        Feb 01 01:58:33 ey04-s00297 rails[4413]: Instantiation Breakdown: Total: 100 | User: 100
+        Feb 01 01:58:34 ey04-s00297 rails[4413]: Completed in
+        STR
+
+        io1 = StringIO.new(str1)
+        io2 = StringIO.new(str2)
+        output = PsuedoOutput.new
+        ActiveRecordInstantiationReport.new([io1, io2], 50).print(output)
+        output.should include("2, MediaController#show")
+      end
+
+    end
+
+
+  end
+end