fiveruns-dash-ruby 0.7.0

data/lib/fiveruns/dash/version.rb

@@ -0,0 +1,86 @@
+require 'yaml'
+
+# (The MIT License)
+# 
+# Copyright (c) 2008 Jamis Buck <jamis@37signals.com>,
+#               with modifications by Bruce Williams <bruce@fiveruns.com>
+# 
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# 'Software'), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+# 
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+module Fiveruns
+  
+  module Dash
+    
+    # A class for describing the current version of a library. The version
+    # consists of three parts: the +major+ number, the +minor+ number, and the
+    # +tiny+ (or +patch+) number.
+    class Version
+      
+      include Comparable
+    
+      # A convenience method for instantiating a new Version instance with the
+      # given +major+, +minor+, and +tiny+ components.
+      def self.[](major, minor, tiny)
+        new(major, minor, tiny)
+      end
+
+      attr_reader :major, :minor, :tiny
+
+      # Create a new Version object with the given components.
+      def initialize(major, minor, tiny)
+        @major, @minor, @tiny = major, minor, tiny
+      end
+
+      # Compare this version to the given +version+ object.
+      def <=>(version)
+        to_i <=> version.to_i
+      end
+
+      # Converts this version object to a string, where each of the three
+      # version components are joined by the '.' character. E.g., 2.0.0.
+      def to_s
+        @to_s ||= [@major, @minor, @tiny].join(".")
+      end
+
+      # Converts this version to a canonical integer that may be compared
+      # against other version objects.
+      def to_i
+        @to_i ||= @major * 1_000_000 + @minor * 1_000 + @tiny
+      end
+      
+      def to_a
+        [@major, @minor, @tiny]
+      end
+      
+      PARSED = YAML.load(File.read(File.dirname(__FILE__) << "/../../../version.yml"))
+
+      MAJOR = PARSED['major']
+      MINOR = PARSED['minor']
+      TINY  = PARSED['patch']
+
+      # The current version as a Version instance
+      CURRENT = new(MAJOR, MINOR, TINY)
+      # The current version as a String
+      STRING = CURRENT.to_s
+            
+    end
+    
+  end
+  
+end

data/recipes/jruby.rb

@@ -0,0 +1,107 @@
+# TODO Error Handling
+# TODO cache VM and/or connection
+# TODO singleton class around vm instance to avoid passing it all over the place
+#      should be able to do vm.connector_avaialble? and vm.load_management_agent
+# TODO cache mbeans
+# TODO more metprogramming in metrics area, pick up names and descriptions 
+#      of attributes direct from mbean and expose all as metrics
+
+if RUBY_PLATFORM[/java/]
+  
+  require 'java'
+  
+  import com.sun.tools.attach.VirtualMachine
+  import javax.management.remote.JMXServiceURL 
+  import javax.management.remote.JMXConnectorFactory
+  import java.lang.management.ManagementFactory
+  import java.util.HashMap
+  
+  module Fiveruns
+    module Dash
+      module JRuby
+        
+        JMX_ADDRESS_PROPERTY = "com.sun.management.jmxremote.localConnectorAddress"
+        
+        def self.connector_address_property( vm )
+          vm.getAgentProperties().getProperty(JMX_ADDRESS_PROPERTY)
+        end
+        
+        def self.connector_available?( vm )
+          !connector_address_property( vm ).nil?
+        end
+        
+        def self.load_management_agent( vm )
+          agent = vm.getSystemProperties().getProperty("java.home") + "/" + "lib" + "/" + "management-agent.jar"
+          vm.loadAgent( agent ) 
+        end
+        
+        def self.connector_address( vm )
+          load_management_agent( vm ) unless connector_available?( vm ) 
+          connector_address_property( vm )
+        end
+        
+        def self.connection #cache somehow ?  instance variable?  Not sure we're in a class ?
+          vm = VirtualMachine::attach( Process.pid.to_s )
+          url = JMXServiceURL.new( connector_address( vm ) )
+          connector = JMXConnectorFactory::connect url, HashMap.new
+          server_connection = connector.mbean_server_connection
+          server_connection
+        end
+        
+        def self.object_name(service_name)
+          "org.jruby:type=Runtime,name=#{::JRuby.runtime.hashCode},service=#{service_name}"
+        end
+        # e.g. mbean( "ClassCache", ClassCacheMBean::java_class)
+        # shold be able to DRY further - How to turn "ClassCache" into ClassCacheMBean::java_class
+        # not sure the pattern holds for everything, but it hold for everything here so far.
+        def self.mbean( service_name, java_class) # cache somehow? map lookup ? only need each bean proxy once.
+          ManagementFactory::newPlatformMXBeanProxy(connection, object_name(service_name), java_class)
+        end
+          
+      end
+    end
+  end
+
+  Fiveruns::Dash.register_recipe :jruby, :url => 'http://dash.fiveruns.com' do |metrics|
+
+    # ############
+    # Class Cache MBean
+    # ############
+  
+    import org.jruby.management.ClassCacheMBean
+  
+    metrics.absolute :live_class_count, "Live Class Count", "Number of active classes" do 
+      Integer(Fiveruns::Dash::JRuby.mbean("ClassCache", ClassCacheMBean::java_class).get_live_class_count)
+    end
+    metrics.absolute :class_load_count, "Class Load Count", "Number of loaded classes" do 
+      Integer(Fiveruns::Dash::JRuby.mbean("ClassCache", ClassCacheMBean::java_class).get_class_load_count)
+    end
+    metrics.absolute :class_reuse_count, "Class Reuse Count", "Number of reused classes" do 
+      Integer(Fiveruns::Dash::JRuby.mbean("ClassCache", ClassCacheMBean::java_class).get_class_reuse_count)
+    end
+
+    # ############
+    # JITCompiler MBean
+    # ############
+  
+    import org.jruby.compiler.JITCompilerMBean
+  
+    metrics.absolute :average_code_size, "Average Code Size", "Average Code Size" do 
+      Integer(Fiveruns::Dash::JRuby.mbean("JITCompiler", JITCompilerMBean::java_class).average_code_size)
+    end
+    metrics.absolute :compile_count, "Compile Count", "Compile Count" do 
+      Integer(Fiveruns::Dash::JRuby.mbean("JITCompiler", JITCompilerMBean::java_class).compile_count)
+    end
+    metrics.absolute :compile_success_count, "Compile Success Count", "Compile Success Count" do 
+      Integer(Fiveruns::Dash::JRuby.mbean("JITCompiler", JITCompilerMBean::java_class).success_count)
+    end
+    metrics.absolute :compile_fail_count, "Compile Fail Count", "Compile Fail Count" do 
+      Integer(Fiveruns::Dash::JRuby.mbean("JITCompiler", JITCompilerMBean::java_class).fail_count)
+    end
+    metrics.absolute :average_compile_time, "Average Compile Time", "Average Compile Time" do 
+      Integer(Fiveruns::Dash::JRuby.mbean("JITCompiler", JITCompilerMBean::java_class).average_compile_time)
+    end
+
+  end
+  
+end

data/recipes/ruby.rb

@@ -0,0 +1,34 @@
+Fiveruns::Dash.register_recipe :ruby, :url => 'http://dash.fiveruns.com' do |metrics|
+  metrics.absolute :vsz, 
+    "Virtual Memory Usage", "The amount of virtual memory used by this process", 
+    :unit => 'kbytes', :scope => :host do 
+    Integer(`ps -o vsz -p #{Process.pid}`[/(\d+)/, 1])
+  end
+  metrics.absolute :rss, "Resident Memory Usage", 
+    "The amount of physical memory used by this process", 
+    :unit => 'kbytes',
+    :scope => :host do
+    Integer(`ps -o rss -p #{Process.pid}`[/(\d+)/, 1])
+  end
+  metrics.percentage :pmem, 
+  "Resident Memory Usage", 
+  "Percentage of Resident Memory Usage", 
+  :scope => :host do
+    Float(`ps -o pmem -p #{Process.pid}`[/(\d+\.\d+)/, 1])
+  end
+
+  if RUBY_PLATFORM == 'java'
+    Fiveruns::Dash.logger.warn "Cannot collect CPU usage data on JRuby"
+  else
+    metrics.percentage :cpu, 
+      'CPU Usage', 
+      'Percentage CPU Usage',
+      :scope => :host do
+      before = Thread.current[:dash_utime] ||= Process.times.utime
+      after = Process.times.utime
+      this_minute = after - before
+      Thread.current[:dash_utime] = after
+      (this_minute / 60) * 100.00
+    end
+  end
+end

data/test/collector_communication_test.rb

@@ -0,0 +1,260 @@
+#require File.dirname(__FILE__) << "/test_helper"
+require 'test/unit'
+require 'rubygems'
+require 'shoulda'
+require 'flexmock/test_unit'
+require 'thread'
+#require 'json'
+require 'thin'
+
+$:.unshift(File.dirname(__FILE__) << '/../lib')
+# Require library files
+require 'fiveruns/dash'
+
+class Test::Unit::TestCase
+  include Fiveruns::Dash
+end
+
+class Integer
+  def intervals
+    self * 5 + 0.1
+  end
+end
+
+module Fiveruns::Dash::Store::HTTP
+  def check_response_of(response)
+    unless response
+      Fiveruns::Dash.logger.debug "Received no response from Dash service"
+      return false
+    end
+    case response.code.to_i
+    when 201
+      true
+    when 400..499
+      Fiveruns::Dash.logger.warn "Could not access Dash service (#{response.code.to_i}, #{response.body.inspect})"
+      false
+    else
+      Fiveruns::Dash.logger.debug "Received unknown response from Dash service (#{response.inspect})"
+      false
+    end
+  rescue JSON::ParserError => e
+    puts response.body
+    Fiveruns::Dash.logger.error "Received non-JSON response (#{response.inspect})"
+    false
+  end
+end
+
+class DummyCollector
+  attr_accessor :sleep_time, :data_payload_count, :info_payload_count, :post_times
+  
+  def initialize(options = {})
+    @startup_delay = options[:startup_delay]
+    @response_delay = options[:response_delay]
+    @info_payload_count = 0
+    @data_payload_count = 0
+    @post_times = []
+  end
+
+  def call(t)
+    sleep(@sleep_time) if @sleep_time
+    puts "sleeping for #{@sleep_time}" if @sleep_time
+    @post_times << Time.now
+    res = nil
+    if t["rack.input"].read =~ /name=\"type\"\r\n\r\ninfo\r\n/
+      @info_payload_count += 1
+      res = info_response
+    else
+     @data_payload_count += 1
+     res = data_response
+    end
+    puts "BOOM! workers: info: #{@info_payload_count} data: #{@data_payload_count} #{Time.now} - #{Time.now.to_f}"
+    return [201, {"Content-Type" => "application/json; charset=utf-8" }, res]
+  end
+
+   
+  def info_response
+    data = {"process_id"=>774736468, "metric_infos"=>[{"name"=>"vsz", "id"=>932254202, "recipe_name"=>"ruby", "recipe_url"=>"http://dash.fiveruns.com"}, {"name"=>"rss", "id"=>932254199, "recipe_name"=>"ruby", "recipe_url"=>"http://dash.fiveruns.com"}, {"name"=>"pmem", "id"=>932254200, "recipe_name"=>"ruby", "recipe_url"=>"http://dash.fiveruns.com"}, {"name"=>"cpu", "id"=>932254201, "recipe_name"=>"ruby", "recipe_url"=>"http://dash.fiveruns.com"}, {"name"=>"activity", "id"=>932254194, "recipe_name"=>"activerecord", "recipe_url"=>"http://dash.fiveruns.com"}, {"name"=>"response_time", "id"=>932254195, "recipe_name"=>"actionpack", "recipe_url"=>"http://dash.fiveruns.com"}, {"name"=>"requests", "id"=>932254196, "recipe_name"=>"actionpack", "recipe_url"=>"http://dash.fiveruns.com"}, {"name"=>"render_time", "id"=>932254197, "recipe_name"=>"actionpack", "recipe_url"=>"http://dash.fiveruns.com"}, {"name"=>"queue_size", "id"=>932254198, "recipe_name"=>"rails", "recipe_url"=>"http://dash.fiveruns.com"}], "traces"=>[]}
+    return data.to_json
+  end
+  def data_response
+   data = {"process_id"=>774736448, "metric_infos"=>[{"name"=>"rss", "id"=>932254199, "recipe_name"=>"ruby", "recipe_url"=>"http://dash.fiveruns.com"}, {"name"=>"pmem", "id"=>932254200, "recipe_name"=>"ruby", "recipe_url"=>"http://dash.fiveruns.com"}, {"name"=>"cpu", "id"=>932254201, "recipe_name"=>"ruby", "recipe_url"=>"http://dash.fiveruns.com"}, {"name"=>"activity", "id"=>932254194, "recipe_name"=>"activerecord", "recipe_url"=>"http://dash.fiveruns.com"}, {"name"=>"response_time", "id"=>932254195, "recipe_name"=>"actionpack", "recipe_url"=>"http://dash.fiveruns.com"}, {"name"=>"requests", "id"=>932254196, "recipe_name"=>"actionpack", "recipe_url"=>"http://dash.fiveruns.com"}, {"name"=>"render_time", "id"=>932254197, "recipe_name"=>"actionpack", "recipe_url"=>"http://dash.fiveruns.com"}, {"name"=>"queue_size", "id"=>932254198, "recipe_name"=>"rails", "recipe_url"=>"http://dash.fiveruns.com"}], "traces"=>[]}
+   return data.to_json
+  end
+end
+
+
+class CollectorCommunicationTest < Test::Unit::TestCase
+  
+  attr_reader :payload
+  context "FiveRuns Dash Gem" do
+
+    setup do
+      ##no_recipe_loading!
+      flexmock(Fiveruns::Dash).should_receive(:load_recipes)
+      
+      ##mock_configuration!
+      @metrics = []
+      @metric_class = Class.new(Metric) do
+        def self.metric_type
+          :test
+        end
+        def info_id
+          1
+        end
+      end
+      3.times do |i|
+        @metrics << @metric_class.new("Metric#{i}") { 1 }
+      end
+      @recipes = []
+      @recipes << Fiveruns::Dash::Recipe.new(:foo, :url => 'http://foo.com')
+      @recipes << Fiveruns::Dash::Recipe.new(:foo2, :url => 'http://foo2.com')
+      @metrics << @metric_class.new("NonCustomMetric") { 2 }
+      @configuration = flexmock(:configuration) do |mock|
+        mock.should_receive(:metrics).and_return(@metrics)
+        mock.should_receive(:recipes).and_return(@recipes)
+      end
+      
+      
+      ##create_session!
+      @session = Session.new(@configuration)
+      
+      # other stuff
+      flexmock(@configuration).should_receive(:options).and_return(:app => '123')
+      flexmock(::Fiveruns::Dash).should_receive(:configuration).and_return(@configuration)
+      flexmock(@session.reporter).should_receive(:update_locations).returns(["http://localhost:9999"])
+      
+      @collector = DummyCollector.new()
+      @thin = Thin::Server.new('127.0.0.1', 9999, @collector)
+      @prev_value = Thread.abort_on_exception
+      Thread.abort_on_exception = true
+    end
+    
+    teardown do
+      Thread.abort_on_exception = @prev_value
+    end
+    
+    should "have no tests in here" do
+      assert true
+    end
+    
+    should_eventually "act properly" do
+      # When the reporter starts, it immediately sends an info packet,
+      # along with a regular payload
+      @t = Thread.new { @thin.start }
+      @session.reporter.interval = 5
+      @session.reporter.start
+      sleep(2.intervals) #enough for 2 cycles
+      assert_equal @collector.info_payload_count, 1
+      assert_equal @collector.data_payload_count, 2
+      @thin.stop     
+      assert_equal @collector.post_times.size, 3 
+      @collector.post_times.size.times do |i|
+        unless i == (@collector.post_times.size - 1)
+          assert_equal (@collector.post_times[i+1].to_i - @collector.post_times[i].to_i), 5
+        end  
+      end
+      @thin.stop!      
+      puts "stopped!"
+    end
+    
+    should_eventually "compensate correctly for delayed post times" do
+      @collector.sleep_time = 2
+      @t = Thread.new { @thin.start }
+      @session.reporter.interval = 5
+      @session.reporter.start
+      sleep(2.intervals + @collector.sleep_time ) #enough for 2 cycles
+      assert_equal @collector.info_payload_count, 1
+      assert_equal @collector.data_payload_count, 2
+      @thin.stop    
+      assert_equal @collector.post_times.size, 3   
+      @collector.post_times.size.times do |i|
+        unless i == (@collector.post_times.size - 1)
+          assert_equal (@collector.post_times[i+1].to_i - @collector.post_times[i].to_i), 5
+        end  
+      end
+      @thin.stop!    
+              
+    end
+    
+    should_eventually "continue to report if the first payload fails" do
+      @session.reporter.interval = 5
+      @session.reporter.start
+      sleep(1.intervals)
+      assert_equal @collector.data_payload_count, 0
+      assert_equal @collector.data_payload_count, 0
+      #puts "STARTING COLLECTOR #{Time.now}"
+      @t = Thread.new { @thin.start }
+      #puts "COLLECTOR STARTED #{Time.now} - #{Time.now.to_f}"
+      #puts "STARTING SLEEP #{Time.now} - #{Time.now.to_f}"
+      sleep(2.intervals)
+      #puts "DONE SLEEPING #{Time.now} - #{Time.now.to_f}"
+      assert_equal @collector.info_payload_count, 1
+      assert_equal @collector.data_payload_count, 1
+      @thin.stop!  
+         
+    end
+  
+    should_eventually "continue to report if collector dies at a random time" do
+      @t = Thread.new { @thin.start }
+      @session.reporter.interval = 5
+      @session.reporter.start
+      sleep(2.intervals)
+      puts "STOPPING!!! #{Time.now}"
+      @thin.stop!
+      @t.kill
+      puts "STOPPED #{Time.now}"
+      assert_equal @collector.info_payload_count, 1
+      assert_equal @collector.data_payload_count, 2
+      sleep(3.intervals) #stop for a while
+      puts "STARTING BACK UP #{Time.now}"
+      @t = Thread.new { @thin.start }
+      puts "STARTED BACK UP #{Time.now}"
+      sleep(3.intervals)
+      puts "STOPPING AGAIN!!! #{Time.now}"
+      @thin.stop
+      puts "STOPPED FINALLY #{Time.now}"
+      assert_equal @collector.data_payload_count, 5
+      
+
+      
+      # check the post time intervals to make sure they match what we expect
+      (0..1).each do |i|
+        assert_equal((@collector.post_times[i+1].to_i - @collector.post_times[i].to_i), 5)
+      end  
+      # because we shut down for 3 intervals, the next post wouldn't happen until the 4th interval
+      # thus, 20 seconds
+      assert_equal (@collector.post_times[3].to_i - @collector.post_times[2].to_i), 20 
+      (3..4).each do |i|
+        assert_equal((@collector.post_times[i+1].to_i - @collector.post_times[i].to_i), 5)
+      end
+
+    end
+    
+  end
+  
+  
+  #######
+  private
+  #######
+  
+  def full_urls(service)
+    full_uris(service).map(&:to_s)
+  end
+  
+  def full_uris(service)
+    @urls.map do |url|
+      uri = URI.parse(url)
+      uri.path = if service == :exceptions
+        '/exceptions.json'
+      else
+        "/apps/123/#{service}.json"
+      end
+      uri
+    end
+  end
+
+  def uris
+    @urls.map { |url| URI.parse(url) }
+  end
+  
+end