scout_rails_proxy_proxy 1.0.5

data/lib/scout_rails_proxy/layaway.rb

@@ -0,0 +1,76 @@
+# Stores metrics in a file before sending them to the server. Two uses:
+# 1. A centralized store for multiple Agent processes. This way, only 1 checkin is sent to Scout rather than 1 per-process.
+# 2. Bundling up reports from multiple timeslices to make updates more efficent server-side.
+# 
+# Metrics are stored in a Hash, where the keys are Time.to_i on the minute. When depositing data, 
+# metrics are either merged with an existing time or placed in a new key.
+class ScoutRailsProxy::Layaway
+  attr_accessor :file
+  def initialize
+    @file = ScoutRailsProxy::LayawayFile.new
+  end
+  
+  def deposit_and_deliver
+    new_data = ScoutRailsProxy::Agent.instance.store.metric_hash
+    controller_count = 0
+    new_data.each do |meta,stats|
+      if meta.metric_name =~ /\AController/
+        controller_count += stats.call_count
+      end
+    end
+    ScoutRailsProxy::Agent.instance.logger.debug "Depositing #{controller_count} requests into #{Time.at(slot).strftime("%m/%d/%y %H:%M:%S %z")} slot."
+    
+    to_deliver = {}
+    file.read_and_write do |old_data|
+      old_data ||= Hash.new
+      # merge data
+      # if the previous minute has ended, its time to send those metrics
+      if old_data.any? and old_data[slot].nil?
+        to_deliver = old_data
+        old_data = Hash.new
+      elsif old_data.any?
+        ScoutRailsProxy::Agent.instance.logger.debug "Not yet time to deliver metrics for slot [#{Time.at(old_data.keys.sort.last).strftime("%m/%d/%y %H:%M:%S %z")}]"
+      else
+        ScoutRailsProxy::Agent.instance.logger.debug "There is no data in the layaway file to deliver."
+      end
+      old_data[slot]=ScoutRailsProxy::Agent.instance.store.merge_data_and_clear(old_data[slot] || Hash.new)
+      ScoutRailsProxy::Agent.instance.logger.debug "Saving the following #{old_data.size} time slots locally:"
+      old_data.each do |k,v|
+        controller_count = 0
+        new_data.each do |meta,stats|
+          if meta.metric_name =~ /\AController/
+            controller_count += stats.call_count
+          end
+        end
+        ScoutRailsProxy::Agent.instance.logger.debug "#{Time.at(k).strftime("%m/%d/%y %H:%M:%S %z")} => #{controller_count} requests"
+      end
+      old_data
+    end
+    to_deliver.any? ? validate_data(to_deliver) : {}
+  end
+  
+  # Ensures the data we're sending to the server isn't stale. 
+  # This can occur if the agent is collecting data, and app server goes down w/data in the local storage. 
+  # When it is restarted later data will remain in local storage but it won't be for the current reporting interval.
+  # 
+  # If the data is stale, an empty Hash is returned. Otherwise, the data from the most recent slot is returned.
+  def validate_data(data)
+    data = data.to_a.sort
+    now = Time.now
+    if (most_recent = data.first.first) < now.to_i - 2*60
+      ScoutRailsProxy::Agent.instance.logger.debug "Local Storage is stale (#{Time.at(most_recent).strftime("%m/%d/%y %H:%M:%S %z")}). Not sending data."
+      {}
+    else
+      data.first.last
+    end
+  rescue
+    ScoutRailsProxy::Agent.instance.logger.debug $!.message
+    ScoutRailsProxy::Agent.instance.logger.debug $!.backtrace
+  end
+  
+  def slot
+    t = Time.now
+    t -= t.sec
+    t.to_i
+  end
+end

data/lib/scout_rails_proxy/layaway_file.rb

@@ -0,0 +1,70 @@
+# Logic for the serialized file access
+class ScoutRailsProxy::LayawayFile
+  def path
+    "#{ScoutRailsProxy::Agent.instance.log_path}/scout_rails_proxy.db"
+  end
+
+  def dump(object)
+    Marshal.dump(object)
+  end
+
+  def load(dump)
+    if dump.size == 0
+      ScoutRailsProxy::Agent.instance.logger.debug("No data in layaway file.")
+      return nil
+    end
+    Marshal.load(dump)
+  rescue ArgumentError, TypeError => e
+    ScoutRailsProxy::Agent.instance.logger.debug("Error loading data from layaway file: #{e.inspect}")
+    ScoutRailsProxy::Agent.instance.logger.debug(e.backtrace.inspect)
+    nil
+  end
+
+  def read_and_write
+    File.open(path, File::RDWR | File::CREAT) do |f|
+      f.flock(File::LOCK_EX)
+      begin
+        result = (yield get_data(f))
+        f.rewind
+        f.truncate(0)
+        if result
+          write(f, dump(result))
+        end
+      ensure
+        f.flock(File::LOCK_UN)
+      end
+    end
+  rescue Errno::ENOENT, Exception  => e
+    ScoutRailsProxy::Agent.instance.logger.error(e.message)
+    ScoutRailsProxy::Agent.instance.logger.debug(e.backtrace.split("\n"))
+  end
+
+  def get_data(f)
+    data = read_until_end(f)
+    result = load(data)
+    f.truncate(0)
+    result
+  end
+
+  def write(f, string)
+    result = 0
+    while (result < string.length)
+      result += f.write_nonblock(string)
+    end
+  rescue Errno::EAGAIN, Errno::EINTR
+    IO.select(nil, [f])
+    retry
+  end
+
+  def read_until_end(f)
+    contents = ""
+    while true
+      contents << f.read_nonblock(10_000)
+    end
+  rescue Errno::EAGAIN, Errno::EINTR
+    IO.select([f])
+    retry
+  rescue EOFError
+    contents
+  end
+end

data/lib/scout_rails_proxy/metric_meta.rb

@@ -0,0 +1,34 @@
+# Contains the meta information associated with a metric. Used to lookup Metrics in to Store's metric_hash.
+class ScoutRailsProxy::MetricMeta
+  def initialize(metric_name, options = {})
+    @metric_name = metric_name
+    @metric_id = nil
+    @scope = Thread::current[:scout_sub_scope] || Thread::current[:scout_scope_name]
+    @desc = options[:desc]
+    @extra = {}
+  end
+  attr_accessor :metric_id, :metric_name
+  attr_accessor :scope
+  attr_accessor :client_id
+  attr_accessor :desc, :extra
+  
+  # To avoid conflicts with different JSON libaries
+  def to_json(*a)
+     %Q[{"metric_id":#{metric_id || 'null'},"metric_name":#{metric_name.to_json},"scope":#{scope.to_json || 'null'}}]
+  end
+  
+  def ==(o)
+    self.eql?(o)
+  end
+  
+  def hash
+     h = metric_name.downcase.hash
+     h ^= scope.downcase.hash unless scope.nil?
+     h ^= desc.downcase.hash unless desc.nil?
+     h
+   end
+
+  def eql?(o)
+   self.class == o.class && metric_name.downcase.eql?(o.metric_name.downcase) && scope == o.scope && client_id == o.client_id && desc == o.desc
+  end
+end # class MetricMeta

data/lib/scout_rails_proxy/metric_stats.rb

@@ -0,0 +1,49 @@
+# Stats that are associated with each instrumented method. 
+class ScoutRailsProxy::MetricStats
+  attr_accessor :call_count
+  attr_accessor :min_call_time
+  attr_accessor :max_call_time
+  attr_accessor :total_call_time
+  attr_accessor :total_exclusive_time
+  attr_accessor :sum_of_squares
+  
+  def initialize(scoped = false)
+    @scoped = scoped
+    self.call_count = 0
+    self.total_call_time = 0.0
+    self.total_exclusive_time = 0.0
+    self.min_call_time = 0.0
+    self.max_call_time = 0.0
+    self.sum_of_squares = 0.0
+  end
+  
+  def update!(call_time,exclusive_time)
+    # If this metric is scoped inside another, use exclusive time for min/max and sum_of_squares. Non-scoped metrics
+    # (like controller actions) track the total call time.
+    t = (@scoped ? exclusive_time : call_time)
+    self.min_call_time = t if self.call_count == 0 or t < min_call_time
+    self.max_call_time = t if self.call_count == 0 or t > max_call_time   
+    self.call_count +=1
+    self.total_call_time += call_time
+    self.total_exclusive_time += exclusive_time
+    self.sum_of_squares += (t * t)
+    self
+  end
+  
+  # combines data from another MetricStats object
+  def combine!(other)
+    self.call_count += other.call_count
+    self.total_call_time += other.total_call_time
+    self.total_exclusive_time += other.total_exclusive_time
+    self.min_call_time = other.min_call_time if self.min_call_time.zero? or other.min_call_time < self.min_call_time
+    self.max_call_time = other.max_call_time if other.max_call_time > self.max_call_time
+    self.sum_of_squares += other.sum_of_squares
+    self
+  end
+  
+  # To avoid conflicts with different JSON libaries handle JSON ourselves. 
+  # Time-based metrics are converted to milliseconds from seconds.
+  def to_json(*a)
+     %Q[{"total_exclusive_time":#{total_exclusive_time*1000},"min_call_time":#{min_call_time*1000},"call_count":#{call_count},"sum_of_squares":#{sum_of_squares*1000},"total_call_time":#{total_call_time*1000},"max_call_time":#{max_call_time*1000}}]
+  end
+end # class MetricStats

data/lib/scout_rails_proxy/stack_item.rb

@@ -0,0 +1,18 @@
+class ScoutRailsProxy::StackItem
+  attr_accessor :children_time
+  attr_reader :metric_name, :start_time 
+  
+  def initialize(metric_name)
+    @metric_name = metric_name
+    @start_time = Time.now
+    @children_time = 0
+  end
+  
+  def ==(o)
+    self.eql?(o)
+  end
+
+  def eql?(o)
+    self.class == o.class && metric_name.eql?(o.metric_name)
+  end
+end # class StackItem

data/lib/scout_rails_proxy/store.rb

@@ -0,0 +1,159 @@
+# The store encapsolutes the logic that (1) saves instrumented data by Metric name to memory and (2) maintains a stack (just an Array)
+# of instrumented methods that are being called. It's accessed via +ScoutRailsProxy::Agent.instance.store+. 
+class ScoutRailsProxy::Store
+  attr_accessor :metric_hash
+  attr_accessor :transaction_hash
+  attr_accessor :stack
+  attr_accessor :sample
+  attr_reader :transaction_sample_lock
+  
+  def initialize
+    @metric_hash = Hash.new
+    # Stores aggregate metrics for the current transaction. When the transaction is finished, metrics
+    # are merged with the +metric_hash+.
+    @transaction_hash = Hash.new
+    @stack = Array.new
+    # ensure background thread doesn't manipulate transaction sample while the store is.
+    @transaction_sample_lock = Mutex.new
+  end
+  
+  # Called when the last stack item completes for the current transaction to clear
+  # for the next run.
+  def reset_transaction!
+    Thread::current[:ignore_transaction] = nil
+    Thread::current[:scout_scope_name] = nil
+    @transaction_hash = Hash.new
+    @stack = Array.new
+  end
+  
+  def ignore_transaction!
+    Thread::current[:ignore_transaction] = true
+  end
+  
+  # Called at the start of Tracer#instrument:
+  # (1) Either finds an existing MetricStats object in the metric_hash or 
+  # initialize a new one. An existing MetricStats object is present if this +metric_name+ has already been instrumented.
+  # (2) Adds a StackItem to the stack. This StackItem is returned and later used to validate the item popped off the stack
+  # when an instrumented code block completes.
+  def record(metric_name)
+    item = ScoutRailsProxy::StackItem.new(metric_name)
+    stack << item
+    item
+  end
+  
+  def stop_recording(sanity_check_item, options={})
+    item = stack.pop
+    stack_empty = stack.empty?
+    # if ignoring the transaction, the item is popped but nothing happens. 
+    if Thread::current[:ignore_transaction]
+      return
+    end
+    # unbalanced stack check - unreproducable cases have seen this occur. when it does, sets a Thread variable 
+    # so we ignore further recordings. +Store#reset_transaction!+ resets this. 
+    if item != sanity_check_item
+      ScoutRailsProxy::Agent.instance.logger.warn "Scope [#{Thread::current[:scout_scope_name]}] Popped off stack: #{item.inspect} Expected: #{sanity_check_item.inspect}. Aborting."
+      ignore_transaction!
+      return
+    end
+    duration = Time.now - item.start_time
+    if last=stack.last
+      last.children_time += duration
+    end
+    meta = ScoutRailsProxy::MetricMeta.new(item.metric_name, :desc => options[:desc])
+    meta.scope = nil if stack_empty
+    
+    # add backtrace for slow calls ... how is exclusive time handled?
+    if duration > 0.5 and !stack_empty
+      meta.extra = {:backtrace => caller.find_all { |c| c =~ /\/app\//}}
+    end
+    stat = transaction_hash[meta] || ScoutRailsProxy::MetricStats.new(!stack_empty)
+    
+    stat.update!(duration,duration-item.children_time)
+    transaction_hash[meta] = stat   
+    
+    # Uses controllers as the entry point for a transaction. Otherwise, stats are ignored.
+    if stack_empty and meta.metric_name.match(/\AController\//)
+      aggs=aggregate_calls(transaction_hash.dup,meta)
+      store_sample(options[:uri],transaction_hash.dup.merge(aggs),meta,stat)  
+      # deep duplicate  
+      duplicate = aggs.dup
+      duplicate.each_pair do |k,v|
+        duplicate[k.dup] = v.dup
+      end  
+      merge_data(duplicate.merge({meta.dup => stat.dup})) # aggregrates + controller 
+    end
+  end
+  
+  # Returns the top-level category names used in the +metrics+ hash.
+  def categories(metrics)
+    cats = Set.new
+    metrics.keys.each do |meta|
+      next if meta.scope.nil? # ignore controller
+      if match=meta.metric_name.match(/\A([\w|\d]+)\//)
+        cats << match[1]
+      end
+    end # metrics.each
+    cats
+  end
+  
+  # Takes a metric_hash of calls and generates aggregates for ActiveRecord and View calls.
+  def aggregate_calls(metrics,parent_meta)
+    categories = categories(metrics)
+    aggregates = {}
+    categories.each do |cat|
+      agg_meta=ScoutRailsProxy::MetricMeta.new("#{cat}/all")
+      agg_meta.scope = parent_meta.metric_name
+      agg_stats = ScoutRailsProxy::MetricStats.new
+      metrics.each do |meta,stats|
+        if meta.metric_name =~ /\A#{cat}\//
+          agg_stats.combine!(stats) 
+        end
+      end # metrics.each
+      aggregates[agg_meta] = agg_stats unless agg_stats.call_count.zero?
+    end # categories.each    
+    aggregates
+  end
+  
+  # Stores the slowest transaction. This will be sent to the server.
+  def store_sample(uri,transaction_hash,parent_meta,parent_stat,options = {})    
+    @transaction_sample_lock.synchronize do
+      if parent_stat.total_call_time >= 2 and (@sample.nil? or (@sample and parent_stat.total_call_time > @sample.total_call_time))
+        @sample = ScoutRailsProxy::TransactionSample.new(uri,parent_meta.metric_name,parent_stat.total_call_time,transaction_hash.dup)
+      end
+    end
+  end
+  
+  # Finds or creates the metric w/the given name in the metric_hash, and updates the time. Primarily used to 
+  # record sampled metrics. For instrumented methods, #record and #stop_recording are used.
+  #
+  # Options:
+  # :scope => If provided, overrides the default scope. 
+  # :exclusive_time => Sets the exclusive time for the method. If not provided, uses +call_time+.
+  def track!(metric_name,call_time,options = {})
+     meta = ScoutRailsProxy::MetricMeta.new(metric_name)
+     meta.scope = options[:scope] if options.has_key?(:scope)
+     stat = metric_hash[meta] || ScoutRailsProxy::MetricStats.new
+     stat.update!(call_time,options[:exclusive_time] || call_time)
+     metric_hash[meta] = stat
+  end
+  
+  # Combines old and current data
+  def merge_data(old_data)
+    old_data.each do |old_meta,old_stats|
+      if stats = metric_hash[old_meta]
+        metric_hash[old_meta] = stats.combine!(old_stats)
+      else
+        metric_hash[old_meta] = old_stats
+      end
+    end
+    metric_hash
+  end
+  
+  # Merges old and current data, clears the current in-memory metric hash, and returns
+  # the merged data
+  def merge_data_and_clear(old_data)
+    merged = merge_data(old_data)
+    self.metric_hash =  {}
+    merged
+  end
+end # class Store

data/lib/scout_rails_proxy/tracer.rb

@@ -0,0 +1,105 @@
+# Contains the methods that instrument blocks of code. 
+# 
+# When a code block is wrapped inside #instrument(metric_name):
+# * The #instrument method pushes a StackItem onto Store#stack
+# * When a code block is finished, #instrument pops the last item off the stack and verifies it's the StackItem
+#   we created earlier. 
+# * Once verified, the metrics for the recording session are merged into the in-memory Store#metric_hash. The current scope
+#   is also set for the metric (if Thread::current[:scout_scope_name] isn't nil).
+module ScoutRailsProxy::Tracer
+  def self.included(klass)
+    klass.extend ClassMethods
+  end
+  
+  module ClassMethods
+    
+    # Use to trace a method call, possibly reporting slow transaction traces to Scout. 
+    def trace(metric_name, options = {}, &block)
+      ScoutRailsProxy::Agent.instance.store.reset_transaction!      
+      instrument(metric_name, options) do
+        Thread::current[:scout_scope_name] = metric_name
+        yield
+        Thread::current[:scout_scope_name] = nil
+      end
+    end
+    
+    # Options:
+    # - :scope => If specified, sets the sub-scope for the metric. We allow additional scope level. This is used
+    # when rendering the transaction tree in the UI. 
+    def instrument(metric_name, options={}, &block)
+      # don't instrument if (1) NOT inside a transaction and (2) NOT a Controller metric.
+      if !Thread::current[:scout_scope_name] and metric_name !~ /\AController\//
+        ScoutRailsProxy::Agent.instance.logger.debug "Not instrumenting [#{metric_name}] - no scope."
+        return yield
+      end
+      if options.delete(:scope)
+        Thread::current[:scout_sub_scope] = metric_name 
+      end
+      stack_item = ScoutRailsProxy::Agent.instance.store.record(metric_name)
+      begin
+        yield
+      ensure
+        Thread::current[:scout_sub_scope] = nil if Thread::current[:scout_sub_scope] == metric_name
+        ScoutRailsProxy::Agent.instance.store.stop_recording(stack_item,options)
+      end
+    end
+    
+    def instrument_method(method,options = {})
+      ScoutRailsProxy::Agent.instance.logger.info "Instrumenting #{method}"
+      metric_name = options[:metric_name] || default_metric_name(method)
+      return if !instrumentable?(method) or instrumented?(method,metric_name)
+      class_eval instrumented_method_string(method, {:metric_name => metric_name, :scope => options[:scope]}), __FILE__, __LINE__
+      
+      alias_method _uninstrumented_method_name(method, metric_name), method
+      alias_method method, _instrumented_method_name(method, metric_name)
+    end
+    
+    private
+    
+    def instrumented_method_string(method, options)
+      klass = (self === Module) ? "self" : "self.class"
+      "def #{_instrumented_method_name(method, options[:metric_name])}(*args, &block)
+        result = #{klass}.instrument(\"#{options[:metric_name]}\",{:scope => #{options[:scope] || false}}) do
+          #{_uninstrumented_method_name(method, options[:metric_name])}(*args, &block)
+        end
+        result
+      end"
+    end
+    
+    # The method must exist to be instrumented.
+    def instrumentable?(method)
+      exists = method_defined?(method) || private_method_defined?(method)
+      ScoutRailsProxy::Agent.instance.logger.warn "The method [#{self.name}##{method}] does not exist and will not be instrumented" unless exists
+      exists
+    end
+    
+    # +True+ if the method is already instrumented. 
+    def instrumented?(method,metric_name)
+      instrumented = method_defined?(_instrumented_method_name(method, metric_name))
+      ScoutRailsProxy::Agent.instance.logger.warn "The method [#{self.name}##{method}] has already been instrumented" if instrumented
+      instrumented
+    end
+    
+    def default_metric_name(method)
+      "Custom/#{self.name}/#{method.to_s}"
+    end
+    
+    # given a method and a metric, this method returns the
+    # untraced alias of the method name
+    def _uninstrumented_method_name(method, metric_name)
+      "#{_sanitize_name(method)}_without_scout_instrument_#{_sanitize_name(metric_name)}"
+    end
+    
+    # given a method and a metric, this method returns the traced
+    # alias of the method name
+    def _instrumented_method_name(method, metric_name)
+      name = "#{_sanitize_name(method)}_with_scout_instrument_#{_sanitize_name(metric_name)}"
+    end
+    
+    # Method names like +any?+ or +replace!+ contain a trailing character that would break when
+    # eval'd as ? and ! aren't allowed inside method names.
+    def _sanitize_name(name)
+      name.to_s.tr_s('^a-zA-Z0-9', '_')
+    end
+  end # ClassMethods
+end # module Tracer