scout_rails 0.0.3.pre → 0.0.4.pre

data/CHANGELOG.markdown

@@ -1,7 +1,13 @@
+# 0.0.4.pre
+
+* Transaction Sampling
+
 # 0.0.3.pre
 
 * Removed dynamic ActiveRecord caller instrumentation
 * Fixed issue that prevents the app from loading if ActiveRecord isn't used.
+* Using a metric hash for each request, then merging when complete. Ensures data associated w/requests that overlap a 
+  minute boundary are correctly associated.
 
 # 0.0.2
 

data/lib/scout_rails/agent.rb

@@ -117,7 +117,7 @@ module ScoutRails
     end
     
     def log_version_pid
-      logger.debug "Scout Agent [#{ScoutRails::VERSION}] Initialized"
+      logger.info "Scout Agent [#{ScoutRails::VERSION}] Initialized"
     end
     
     def log_path
@@ -216,19 +216,22 @@ module ScoutRails
             controller_count += stats.call_count
           end
         end        
-        logger.debug "#{config.settings['name']} Delivering metrics for #{controller_count} requests."
+        logger.debug "#{config.settings['name']} Delivering metrics for #{controller_count} requests."        
         response =  post( checkin_uri,
-                           Marshal.dump(metrics),
+                           Marshal.dump(:metrics => metrics, :sample => store.sample),
                            "Content-Type"     => "application/json" )
         if response and response.is_a?(Net::HTTPSuccess)
           directives = Marshal.load(response.body)
           self.metric_lookup.merge!(directives[:metric_lookup])
+          store.transaction_sample_lock.synchronize do
+            store.sample = nil
+          end
           logger.debug "Metric Cache Size: #{metric_lookup.size}"
         end
       end
     rescue
-      logger.debug "Error on checkin to #{checkin_uri.to_s}"
-      logger.debug $!.message
+      logger.info "Error on checkin to #{checkin_uri.to_s}"
+      logger.info $!.message
       logger.debug $!.backtrace
     end
     

data/lib/scout_rails/instruments/active_record_instruments.rb

@@ -15,7 +15,7 @@ module ScoutRails::Instruments
     
     def log_with_scout_instruments(*args, &block)
       sql, name = args
-      self.class.instrument(scout_ar_metric_name(sql,name)) do
+      self.class.instrument(scout_ar_metric_name(sql,name), :desc => scout_sanitize_sql(sql)) do
         log_without_scout_instruments(sql, name, &block)
       end
     end
@@ -35,13 +35,27 @@ module ScoutRails::Instruments
                         end
                       end
         metric = "ActiveRecord/#{model}/#{metric_name}" if metric_name
-        metric = "Database/SQL/other" if metric.nil?
+        metric = "ActiveRecord/SQL/other" if metric.nil?
       else
-        metric = "Database/SQL/Unknown"
+        metric = "ActiveRecord/SQL/Unknown"
       end
       metric
     end
     
+    # Removes actual values from SQL. Used to both obfuscate the SQL and group 
+    # similar queries in the UI.
+    def scout_sanitize_sql(sql)
+      return nil if sql.length > 1000 # safeguard - don't sanitize large SQL statements
+      sql = sql.dup
+      sql.gsub!(/\\"/, '') # removing escaping double quotes
+      sql.gsub!(/\\'/, '') # removing escaping single quotes
+      sql.gsub!(/'(?:[^']|'')*'/, '?') # removing strings (single quote)
+      sql.gsub!(/"(?:[^"]|"")*"/, '?') # removing strings (double quote)
+      sql.gsub!(/\b\d+\b/, '?') # removing integers
+      sql.gsub!(/\?(,\?)+/,'?') # replace multiple ? w/a single ?
+      sql
+    end
+    
   end # module ActiveRecordInstruments
 end # module Instruments
 
@@ -55,6 +69,8 @@ def add_instruments
        include ::ScoutRails::Tracer
     end
   end
+rescue
+  ScoutRails::Agent.instance.logger.warn "ActiveRecord instrumentation exception: #{$!.message}"
 end
 
 if defined?(::Rails) && ::Rails::VERSION::MAJOR.to_i == 3

data/lib/scout_rails/instruments/rails/action_controller_instruments.rb

@@ -16,10 +16,8 @@ module ScoutRails::Instruments
     # specific controller actions.
     def perform_action_with_scout_instruments(*args, &block)
       scout_controller_action = "Controller/#{controller_path}/#{action_name}"
-      self.class.instrument(scout_controller_action) do
-        Thread::current[:scout_scope_name] = scout_controller_action
+      self.class.trace(scout_controller_action, :uri => request.request_uri) do
         perform_action_without_scout_instruments(*args, &block)
-        Thread::current[:scout_scope_name] = nil
       end
     end
   end
@@ -42,6 +40,6 @@ if defined?(ActionController) && defined?(ActionController::Base)
   ScoutRails::Agent.instance.logger.debug "Instrumenting ActionView::Template"
   ActionView::Template.class_eval do
     include ::ScoutRails::Tracer
-    instrument_method :render, 'View/#{path[%r{^(/.*/)?(.*)$},2]}/Rendering'
+    instrument_method :render, :metric_name => 'View/#{path[%r{^(/.*/)?(.*)$},2]}/Rendering', :scope => true
   end
 end

data/lib/scout_rails/instruments/rails3/action_controller_instruments.rb

@@ -1,10 +1,10 @@
+# Rails 3
 module ScoutRails::Instruments
   module ActionControllerInstruments
     # Instruments the action and tracks errors.
     def process_action(*args)
       scout_controller_action = "Controller/#{controller_path}/#{action_name}"
-      self.class.instrument(scout_controller_action) do
-        Thread::current[:scout_scope_name] = scout_controller_action
+      self.class.trace(scout_controller_action, :uri => request.fullpath) do
         begin
           super
         rescue Exception => e
@@ -30,6 +30,6 @@ if defined?(ActionView) && defined?(ActionView::PartialRenderer)
   ScoutRails::Agent.instance.logger.debug "Instrumenting ActionView::PartialRenderer"
   ActionView::PartialRenderer.class_eval do
     include ScoutRails::Tracer
-    instrument_method :render_partial, 'View/#{@template.virtual_path}/Rendering'
+    instrument_method :render_partial, :metric_name => 'View/#{@template.virtual_path}/Rendering', :scope => true
   end
 end

data/lib/scout_rails/instruments/sinatra_instruments.rb

@@ -15,8 +15,7 @@ module ScoutRails::Instruments
       name = 'root' if name.empty?
       name = @request.request_method + ' ' + name if @request && @request.respond_to?(:request_method)      
       scout_controller_action = "Controller/Sinatra/#{name}"
-      self.class.instrument(scout_controller_action) do
-        Thread::current[:scout_scope_name] = scout_controller_action  
+      self.class.trace(scout_controller_action, :uri => @request.path_info) do
         route_eval_without_scout_instruments(&blockarg)    
       end
     end # route_eval_with_scout_instrumentss

data/lib/scout_rails/layaway.rb

@@ -61,7 +61,6 @@ class ScoutRails::Layaway
       ScoutRails::Agent.instance.logger.debug "Local Storage is stale (#{Time.at(most_recent).strftime("%m/%d/%y %H:%M:%S %z")}). Not sending data."
       {}
     else
-      #ScoutRails::Agent.instance.logger.debug "Validated time slot: #{Time.at(most_recent).strftime("%m/%d/%y %H:%M:%S %z")}"
       data.first.last
     end
   rescue

data/lib/scout_rails/metric_meta.rb

@@ -1,13 +1,16 @@
 # Contains the meta information associated with a metric. Used to lookup Metrics in to Store's metric_hash.
 class ScoutRails::MetricMeta
-  def initialize(metric_name)
+  def initialize(metric_name, options = {})
     @metric_name = metric_name
     @metric_id = nil
-    @scope = Thread::current[:scout_scope_name]
+    @scope = Thread::current[:scout_sub_scope] || Thread::current[:scout_scope_name]
+    @desc = options[:desc]
+    @extra = {}
   end
   attr_accessor :metric_id, :metric_name
   attr_accessor :scope
   attr_accessor :client_id
+  attr_accessor :desc, :extra
   
   # To avoid conflicts with different JSON libaries
   def to_json(*a)
@@ -21,16 +24,19 @@ class ScoutRails::MetricMeta
   def hash
      h = metric_name.hash
      h ^= scope.hash unless scope.nil?
+     h ^= desc.hash unless desc.nil?
      h
    end
    
    def <=>(o)
      namecmp = self.name <=> o.name
      return namecmp if namecmp != 0
-     return (self.scope || '') <=> (o.scope || '')
+     scopecmp = (self.scope || '') <=> (o.scope || '')
+     return scopecmp if scopecmp != 0
+     (self.desc || '') <=> (o.desc || '') 
    end
 
   def eql?(o)
-   self.class == o.class && metric_name.eql?(o.metric_name) && scope == o.scope && client_id == o.client_id
+   self.class == o.class && metric_name.eql?(o.metric_name) && scope == o.scope && client_id == o.client_id && desc == o.desc
   end
 end # class MetricMeta

data/lib/scout_rails/metric_stats.rb

@@ -7,7 +7,8 @@ class ScoutRails::MetricStats
   attr_accessor :total_exclusive_time
   attr_accessor :sum_of_squares
   
-  def initialize
+  def initialize(scoped = false)
+    @scoped = scoped
     self.call_count = 0
     self.total_call_time = 0.0
     self.total_exclusive_time = 0.0
@@ -17,12 +18,15 @@ class ScoutRails::MetricStats
   end
   
   def update!(call_time,exclusive_time)
-    self.min_call_time = call_time if self.call_count == 0 or call_time < min_call_time
-    self.max_call_time = call_time if self.call_count == 0 or call_time > max_call_time   
+    # If this metric is scoped inside another, use exclusive time for min/max and sum_of_squares. Non-scoped metrics
+    # (like controller actions) track the total call time.
+    t = (@scoped ? exclusive_time : call_time)
+    self.min_call_time = t if self.call_count == 0 or t < min_call_time
+    self.max_call_time = t if self.call_count == 0 or t > max_call_time   
     self.call_count +=1
     self.total_call_time += call_time
     self.total_exclusive_time += exclusive_time
-    self.sum_of_squares += (call_time * call_time)
+    self.sum_of_squares += (t * t)
     self
   end
   
@@ -31,7 +35,7 @@ class ScoutRails::MetricStats
     self.call_count += other.call_count
     self.total_call_time += other.total_call_time
     self.total_exclusive_time += other.total_exclusive_time
-    self.min_call_time = other.min_call_time if other.min_call_time < self.min_call_time
+    self.min_call_time = other.min_call_time if self.min_call_time.zero? or other.min_call_time < self.min_call_time
     self.max_call_time = other.max_call_time if other.max_call_time > self.max_call_time
     self.sum_of_squares += other.sum_of_squares
     self

data/lib/scout_rails/store.rb

@@ -2,11 +2,26 @@
 # of instrumented methods that are being called. It's accessed via +ScoutRails::Agent.instance.store+. 
 class ScoutRails::Store
   attr_accessor :metric_hash
+  attr_accessor :transaction_hash
   attr_accessor :stack
+  attr_accessor :sample
+  attr_reader :transaction_sample_lock
   
   def initialize
     @metric_hash = Hash.new
+    # Stores aggregate metrics for the current transaction. When the transaction is finished, metrics
+    # are merged with the +metric_hash+.
+    @transaction_hash = Hash.new
     @stack = Array.new
+    # ensure background thread doesn't manipulate transaction sample while the store is.
+    @transaction_sample_lock = Mutex.new
+  end
+  
+  # Called when the last stack item completes for the current transaction to clear
+  # for the next run.
+  def reset_transaction!
+    Thread::current[:scout_scope_name] = nil
+    @transaction_hash = Hash.new
   end
   
   # Called at the start of Tracer#instrument:
@@ -15,31 +30,68 @@ class ScoutRails::Store
   # (2) Adds a StackItem to the stack. This StackItem is returned and later used to validate the item popped off the stack
   # when an instrumented code block completes.
   def record(metric_name)
-    #ScoutRails::Agent.instance.logger.debug "recording #{metric_name}"
     item = ScoutRails::StackItem.new(metric_name)
     stack << item
     item
   end
   
-  def stop_recording(sanity_check_item)
+  def stop_recording(sanity_check_item, options={})
     item = stack.pop
+    stack_empty = stack.empty?
     raise "items not equal: #{item.inspect} / #{sanity_check_item.inspect}" if item != sanity_check_item
     duration = Time.now - item.start_time
     if last=stack.last
-      #ScoutRails::Agent.instance.logger.debug "found an element on stack [#{last.inspect}]. adding duration #{duration} to children time [#{last.children_time}]"
       last.children_time += duration
     end
-    #ScoutRails::Agent.instance.logger.debug "popped #{item.inspect} off stack. duration: #{duration}s"
-    if stack.empty? # this is the last item on the stack. it shouldn't have a scope.
-      Thread::current[:scout_scope_name] = nil
+    meta = ScoutRails::MetricMeta.new(item.metric_name, :desc => options[:desc])
+    meta.scope = nil if stack_empty
+    
+    # add backtrace for slow calls ... how is exclusive time handled?
+    if duration > 0.5 and !stack_empty
+      meta.extra = {:backtrace => caller.find_all { |c| c =~ /\/app\//}}
     end
-    meta = ScoutRails::MetricMeta.new(item.metric_name)
-    #ScoutRails::Agent.instance.logger.debug "meta: #{meta.inspect}"
-    stat = metric_hash[meta] || ScoutRails::MetricStats.new
-    #ScoutRails::Agent.instance.logger.debug "found existing stat w/ky: #{meta}" if !stat.call_count.zero?
+    stat = transaction_hash[meta] || ScoutRails::MetricStats.new(!stack_empty)
+    
     stat.update!(duration,duration-item.children_time)
-    metric_hash[meta] = stat   
-    #ScoutRails::Agent.instance.logger.debug "metric hash has #{metric_hash.size} items"
+    transaction_hash[meta] = stat   
+    
+    if stack_empty
+      aggs=aggregate_calls(transaction_hash.dup,meta)
+      store_sample(options[:uri],transaction_hash.dup.merge(aggs),meta,stat)  
+      # ugly attempt to see if deep dup is the issue  
+      duplicate = aggs.dup
+      duplicate.each_pair do |k,v|
+        duplicate[k.dup] = v.dup
+      end  
+      merge_data(duplicate.merge({meta.dup => stat.dup})) # aggregrates + controller 
+    end
+  end
+  
+  # Takes a metric_hash of calls and generates aggregates for ActiveRecord and View calls.
+  def aggregate_calls(metrics,parent_meta)
+    categories = %w(ActiveRecord View)
+    aggregates = {}
+    categories.each do |cat|
+      agg_meta=ScoutRails::MetricMeta.new("#{cat}/all")
+      agg_meta.scope = parent_meta.metric_name
+      agg_stats = ScoutRails::MetricStats.new
+      metrics.each do |meta,stats|
+        if meta.metric_name =~ /\A#{cat}\//
+          agg_stats.combine!(stats) 
+        end
+      end # metrics.each
+      aggregates[agg_meta] = agg_stats unless agg_stats.call_count.zero?
+    end # categories.each    
+    aggregates
+  end
+  
+  # Stores the slowest transaction. This will be sent to the server.
+  def store_sample(uri,transaction_hash,parent_meta,parent_stat,options = {})    
+    @transaction_sample_lock.synchronize do
+      if parent_stat.total_call_time > 1 and (@sample.nil? or (@sample and parent_stat.total_call_time > @sample.total_call_time))
+        @sample = ScoutRails::TransactionSample.new(uri,parent_meta.metric_name,parent_stat.total_call_time,transaction_hash.dup)
+      end
+    end
   end
   
   # Finds or creates the metric w/the given name in the metric_hash, and updates the time. Primarily used to 

data/lib/scout_rails/tracer.rb

@@ -17,35 +17,45 @@ module ScoutRails::Tracer
       ScoutRails::Agent.instance.store
     end
     
-    def instrument(metric_name, &block)
-      #ScoutRails::Agent.instance.logger.debug "Instrumenting: #{metric_name} w/scope: #{Thread::current[:scout_scope_name]}"
+    # Use to trace a method call, possibly reporting slow transaction traces to Scout. 
+    def trace(metric_name, options = {}, &block)
+      ScoutRails::Agent.instance.store.reset_transaction!      
+      instrument(metric_name, options) do
+        Thread::current[:scout_scope_name] = metric_name
+        yield
+        Thread::current[:scout_scope_name] = nil
+      end
+    end
+    
+    def instrument(metric_name, options={}, &block)
+      if options.delete(:scope)
+        Thread::current[:scout_sub_scope] = metric_name 
+      end
       stack_item = store.record(metric_name)
-      #ScoutRails::Agent.instance.logger.debug "Stack contains: #{store.stack.size} methods"
       begin
         yield
       ensure
-        store.stop_recording(stack_item)
+        Thread::current[:scout_sub_scope] = nil if Thread::current[:scout_sub_scope] == metric_name
+        store.stop_recording(stack_item,options)
       end
     end
     
-    def instrument_method(method,metric_name = nil)
-      #ScoutRails::Agent.instance.logger.debug "Instrumenting method name: #{name} w/metric name: #{metric_name}"
-      metric_name = metric_name || default_metric_name(method)
+    def instrument_method(method,options = {})
+      metric_name = options[:metric_name] || default_metric_name(method)
       return if !instrumentable?(method) or instrumented?(method,metric_name)
-      class_eval instrumented_method_string(method, metric_name), __FILE__, __LINE__
+      class_eval instrumented_method_string(method, {:metric_name => metric_name, :scope => options[:scope]}), __FILE__, __LINE__
       
       alias_method _uninstrumented_method_name(method, metric_name), method
       alias_method method, _instrumented_method_name(method, metric_name)
-      #ScoutRails::Agent.instance.logger.debug "Instrumented #{self.name}##{method} w/metric name: #{metric_name}"
     end
     
     private
     
-    def instrumented_method_string(method, metric_name)
+    def instrumented_method_string(method, options)
       klass = (self === Module) ? "self" : "self.class"
-      "def #{_instrumented_method_name(method, metric_name)}(*args, &block)
-        result = #{klass}.instrument(\"#{metric_name}\") do
-          #{_uninstrumented_method_name(method, metric_name)}(*args, &block)
+      "def #{_instrumented_method_name(method, options[:metric_name])}(*args, &block)
+        result = #{klass}.instrument(\"#{options[:metric_name]}\",{:scope => #{options[:scope] || false}}) do
+          #{_uninstrumented_method_name(method, options[:metric_name])}(*args, &block)
         end
         result
       end"

data/lib/scout_rails/transaction_sample.rb

@@ -0,0 +1,10 @@
+class ScoutRails::TransactionSample
+  attr_reader :metric_name, :total_call_time, :metrics, :meta, :uri
+  
+  def initialize(uri,metric_name,total_call_time,metrics)
+    @uri = uri
+    @metric_name = metric_name
+    @total_call_time = total_call_time
+    @metrics = metrics
+  end
+end

data/lib/scout_rails/version.rb

@@ -1,3 +1,3 @@
 module ScoutRails
-  VERSION = "0.0.3.pre"
+  VERSION = "0.0.4.pre"
 end

data/lib/scout_rails.rb

@@ -14,6 +14,7 @@ require File.expand_path('../scout_rails/metric_stats.rb', __FILE__)
 require File.expand_path('../scout_rails/stack_item.rb', __FILE__)
 require File.expand_path('../scout_rails/store.rb', __FILE__)
 require File.expand_path('../scout_rails/tracer.rb', __FILE__)
+require File.expand_path('../scout_rails/transaction_sample.rb', __FILE__)
 require File.expand_path('../scout_rails/instruments/process/process_cpu.rb', __FILE__)
 require File.expand_path('../scout_rails/instruments/process/process_memory.rb', __FILE__)
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: scout_rails
 version: !ruby/object:Gem::Version
-  version: 0.0.3.pre
+  version: 0.0.4.pre
   prerelease: 6
 platform: ruby
 authors:
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-05-16 00:00:00.000000000 Z
+date: 2012-06-12 00:00:00.000000000 Z
 dependencies: []
 description: Monitors a Ruby on Rails application and reports detailed metrics on
   performance to Scout, a hosted monitoring service.
@@ -43,6 +43,7 @@ files:
 - lib/scout_rails/stack_item.rb
 - lib/scout_rails/store.rb
 - lib/scout_rails/tracer.rb
+- lib/scout_rails/transaction_sample.rb
 - lib/scout_rails/version.rb
 - scout_rails.gemspec
 homepage: https://github.com/scoutapp/scout_rails