scout_apm 2.2.0.pre3 → 2.3.0.pre

data/lib/scout_apm/db_query_metric_set.rb

@@ -0,0 +1,80 @@
+module ScoutApm
+  class DbQueryMetricSet
+    include Enumerable
+
+    attr_reader :metrics # the raw metrics. You probably want #metrics_to_report
+    attr_reader :config # A ScoutApm::Config instance
+
+    def initialize(config=ScoutApm::Agent.instance.config)
+      # A hash of DbQueryMetricStats values, keyed by DbQueryMetricStats.key
+      @metrics = Hash.new
+      @config = config
+    end
+
+    def each
+      metrics.each do |_key, db_query_metric_stat|
+        yield db_query_metric_stat
+      end
+    end
+
+    # Looks up a DbQueryMetricStats instance in the +@metrics+ hash. Sets the value to +other+ if no key
+    # Returns a DbQueryMetricStats instance
+    def lookup(other)
+      metrics[other.key] ||= other
+    end
+
+    # Take another set, and merge it with this one
+    def combine!(other)
+      other.each do |metric|
+        self << metric
+      end
+      self
+    end
+
+    # Add a single DbQueryMetricStats object to this set.
+    #
+    # Looks up an existing one under this key and merges, or just saves a new
+    # one under the key
+    def <<(stat)
+      existing_stat = metrics[stat.key]
+      if existing_stat
+        existing_stat.combine!(stat)
+      elsif at_limit?
+        # We're full up, can't add any more.
+        # Should I log this? It may get super noisy?
+      else
+        metrics[stat.key] = stat
+      end
+    end
+
+    def increment_transaction_count!
+      metrics.each do |_key, db_query_metric_stat|
+        db_query_metric_stat.increment_transaction_count!
+      end
+    end
+
+    def metrics_to_report
+      report_limit = config.value('database_metric_report_limit')
+      if metrics.size > report_limit
+        metrics.
+          values.
+          sort_by {|stat| stat.call_time }.
+          reverse.
+          take(report_limit)
+      else
+        metrics.values
+      end
+    end
+
+    def inspect
+      metrics.map {|key, metric|
+        "#{key.inspect} - Count: #{metric.call_count}, Total Time: #{"%.2f" % metric.call_time}"
+      }.join("\n")
+    end
+
+    def at_limit?
+      @limit ||= config.value('database_metric_limit')
+      metrics.size >= @limit
+    end
+  end
+end

data/lib/scout_apm/db_query_metric_stats.rb

@@ -0,0 +1,102 @@
+module ScoutApm
+  class DbQueryMetricStats
+
+    DEFAULT_HISTOGRAM_SIZE = 50
+
+    attr_reader :model_name
+    attr_reader :operation
+    attr_reader :scope
+
+    attr_reader :transaction_count
+
+    attr_reader :call_count
+    attr_reader :call_time
+    attr_reader :rows_returned
+
+    attr_reader :min_call_time
+    attr_reader :max_call_time
+
+    attr_reader :min_rows_returned
+    attr_reader :max_rows_returned
+
+    attr_reader :histogram
+
+    def initialize(model_name, operation, scope, call_count, call_time, rows_returned)
+      @model_name = model_name
+      @operation = operation
+
+      @call_count = call_count
+
+      @call_time = call_time
+      @min_call_time = call_time
+      @max_call_time = call_time
+
+      @rows_returned = rows_returned
+      @min_rows_returned = rows_returned
+      @max_rows_returned = rows_returned
+
+      # Should we have a histogram for timing, and one for rows_returned?
+      # This histogram is for call_time
+      @histogram = NumericHistogram.new(DEFAULT_HISTOGRAM_SIZE)
+      @histogram.add(call_time)
+
+      @transaction_count = 0
+
+      @scope = scope
+    end
+
+    # Merge data in this scope. Used in DbQueryMetricSet
+    def key
+      @key ||= [model_name, operation, scope]
+    end
+
+    # Combine data from another DbQueryMetricStats into +self+. Modifies and returns +self+
+    def combine!(other)
+      return self if other == self
+
+      @transaction_count += other.transaction_count
+      @call_count += other.call_count
+      @rows_returned += other.rows_returned
+      @call_time += other.call_time
+
+      @min_call_time = other.min_call_time if @min_call_time.zero? or other.min_call_time < @min_call_time
+      @max_call_time = other.max_call_time if other.max_call_time > @max_call_time
+
+      @min_rows_returned = other.min_rows_returned if @min_rows_returned.zero? or other.min_rows_returned < @min_rows_returned
+      @max_rows_returned = other.max_rows_returned if other.max_rows_returned > @max_rows_returned
+
+      @histogram.combine!(other.histogram)
+      self
+    end
+
+    def as_json
+      json_attributes = [
+        :model_name,
+        :operation,
+        :scope,
+
+        :transaction_count,
+        :call_count,
+
+        :histogram,
+        :call_time,
+        :max_call_time,
+        :min_call_time,
+
+        :max_rows_returned,
+        :min_rows_returned,
+        :rows_returned,
+      ]
+
+      ScoutApm::AttributeArranger.call(self, json_attributes)
+    end
+
+    # Called by the Set on each DbQueryMetricStats object that it holds, only
+    # once during the recording of a transaction.
+    #
+    # Don't call elsewhere, and don't set to 1 in the initializer.
+    def increment_transaction_count!
+      @transaction_count += 1
+    end
+  end
+end

data/lib/scout_apm/debug.rb

@@ -0,0 +1,37 @@
+module ScoutApm
+  class Debug
+    # see self.instance
+    @@instance = nil
+
+    def self.instance
+      @@instance ||= new
+    end
+
+    def register_periodic_hook(&hook)
+      @periodic_hooks << hook
+    end
+
+    def call_periodic_hooks
+      @periodic_hooks.each do |hook|
+        begin
+          hook.call
+        rescue => e
+          logger.info("Periodic debug hook failed to run: #{e}\n\t#{e.backtrace.join("\n\t")}")
+        end
+      end
+    rescue
+      # Something went super wrong for the inner rescue to not catch this. Just
+      # swallow the error. The debug tool should never crash the app.
+    end
+
+    private
+
+    def initialize
+      @periodic_hooks = []
+    end
+
+    def logger
+      ScoutApm::Agent.instance.logger
+    end
+  end
+end

data/lib/scout_apm/environment.rb

@@ -24,8 +24,9 @@ module ScoutApm
     ]
 
     BACKGROUND_JOB_INTEGRATIONS = [
+      ScoutApm::BackgroundJobIntegrations::Resque.new,
       ScoutApm::BackgroundJobIntegrations::Sidekiq.new,
-      # ScoutApm::BackgroundJobIntegrations::DelayedJob.new
+      ScoutApm::BackgroundJobIntegrations::DelayedJob.new,
     ]
 
     FRAMEWORK_INTEGRATIONS = [
@@ -41,13 +42,8 @@ module ScoutApm
       ScoutApm::PlatformIntegrations::Server.new,
     ]
 
-    DEPLOY_INTEGRATIONS = [
-      ScoutApm::DeployIntegrations::Capistrano3.new(STDOUT_LOGGER),
-      # ScoutApm::DeployIntegrations::Capistrano2.new(STDOUT_LOGGER),
-    ]
-
     def env
-      @env ||= deploy_integration? ? deploy_integration.env : framework_integration.env
+      @env ||= framework_integration.env
     end
 
     def framework
@@ -63,7 +59,9 @@ module ScoutApm
     end
 
     def application_name
-      Agent.instance.config.value("name") || framework_integration.application_name
+      Agent.instance.config.value("name") ||
+        framework_integration.application_name ||
+        "App"
     end
 
     def database_engine
@@ -87,8 +85,16 @@ module ScoutApm
                       end
     end
 
+    def scm_subdirectory
+      @scm_subdirectory ||= if Agent.instance.config.value('scm_subdirectory').empty?
+        ''
+      else
+        Agent.instance.config.value('scm_subdirectory').sub(/^\//, '') # Trim any leading slash
+      end
+    end
+
     def root
-      @root ||= deploy_integration? ? deploy_integration.root : framework_root
+      @root ||= framework_root
     end
 
     def framework_root
@@ -110,6 +116,10 @@ module ScoutApm
       @hostname ||= Agent.instance.config.value("hostname") || platform_integration.hostname
     end
 
+    def git_revision
+      @git_revision ||= ScoutApm::GitRevision.new
+    end
+
     # Returns the whole integration object
     # This needs to be improved. Frequently, multiple app servers gem are present and which
     # ever is checked first becomes the designated app server.
@@ -143,12 +153,9 @@ module ScoutApm
       background_job_integration && background_job_integration.name
     end
 
-    def deploy_integration
-      @deploy_integration ||= DEPLOY_INTEGRATIONS.detect{ |integration| integration.present? }
-    end
-
-    def deploy_integration?
-      !@deploy_integration.nil?
+    # If both stdin & stdout are interactive and the Rails::Console constant is defined
+    def interactive?
+      defined?(::Rails::Console) && $stdout.isatty && $stdin.isatty
     end
 
     ### ruby checks

data/lib/scout_apm/git_revision.rb

@@ -0,0 +1,51 @@
+module ScoutApm
+  class GitRevision
+
+    attr_accessor :sha
+
+    def initialize
+      @sha = detect
+      ScoutApm::Agent.instance.logger.debug "Detected Git Revision [#{@sha}]"
+    end
+
+    private
+
+    def detect
+      detect_from_env_var    ||
+      detect_from_heroku     ||
+      detect_from_capistrano ||
+      detect_from_git
+    end
+
+    def detect_from_heroku
+      ENV['HEROKU_SLUG_COMMIT']
+    end
+
+    def detect_from_env_var
+      ENV['SCOUT_REVISION_SHA']
+    end
+
+    def detect_from_capistrano
+      version = File.read(File.join(app_root, 'REVISION')).strip
+      # Capistrano 3.0 - 3.1.x
+      version || File.open(File.join(app_root, '..', 'revisions.log')).to_a.last.strip.sub(/.*as release ([0-9]+).*/, '\1')
+    rescue
+      ScoutApm::Agent.instance.logger.debug "Unable to detect Git Revision from Capistrano: #{$!.message}"
+      nil
+    end
+
+    def detect_from_git
+      if File.directory?(".git")
+        `git rev-parse --short HEAD`.strip 
+      end
+    rescue
+      ScoutApm::Agent.instance.logger.debug "Unable to detect Git Revision from Git: #{$!.message}"
+      nil
+    end
+
+    def app_root
+      ScoutApm::Environment.instance.root
+    end
+
+  end
+end

data/lib/scout_apm/histogram.rb

@@ -90,7 +90,11 @@ module ScoutApm
     def combine!(other)
       mutex.synchronize do
         other.mutex.synchronize do
-          @bins = (other.bins + @bins).sort_by {|b| b.value }
+          @bins = (other.bins + @bins).
+            group_by {|b| b.value }.
+            map {|val, bs| [val, bs.inject(0) {|sum, b| sum + b.count }] }.
+            map {|val, sum| HistogramBin.new(val,sum) }.
+            sort_by { |b| b.value }
           @total += other.total
           trim
           self
@@ -100,7 +104,12 @@ module ScoutApm
 
     def as_json
       mutex.synchronize do
-        bins.map{|b| [b.value, b.count]}
+        bins.map{ |b|
+          [
+            ScoutApm::Utils::Numbers.round(b.value, 4),
+            b.count
+          ]
+        }
       end
     end
 

data/lib/scout_apm/instant/assets/xmlhttp_instrumentation.html

@@ -3,7 +3,7 @@
   (function(){var open=window.XMLHttpRequest.prototype.open;var send=window.XMLHttpRequest.prototype.send;function openReplacement(method,url,async,user,password){this._url=url;return open.apply(this,arguments);}
     function sendReplacement(data){if(this.onload){this._onload=this.onload;}
       this.onload=onLoadReplacement;return send.apply(this,arguments);}
-    function onLoadReplacement(){if(this._url.startsWith(window.location.protocol+"//"+window.location.host)||!this._url.startsWith("http")){try{traceText=this.getResponseHeader("X-scoutapminstant");if(traceText){setTimeout(function(){window.scoutInstant("addTrace",traceText)},0);}}catch(e){console.debug("Problem getting X-scoutapminstant header");}}
+    function onLoadReplacement(){if(this._url.startsWith(window.location.protocol+"//"+window.location.host)||!this._url.startsWith("http")){try{var traceText=this.getResponseHeader("X-scoutapminstant");if(traceText){setTimeout(function(){window.scoutInstant("addTrace",traceText)},0);}}catch(e){console.debug("Problem getting X-scoutapminstant header");}}
       if(this._onload){return this._onload.apply(this,arguments);}}
     window.XMLHttpRequest.prototype.open=openReplacement;window.XMLHttpRequest.prototype.send=sendReplacement;})();
-</script>
+</script>

data/lib/scout_apm/instant/middleware.rb

@@ -5,6 +5,11 @@ module ScoutApm
     class Page
       def initialize(html)
         @html = html
+
+        if html.is_a?(Array)
+          @html = html.inject("") { |memo, str| memo + str }
+        end
+
         @to_add_to_head = []
         @to_add_to_body = []
       end
@@ -45,63 +50,200 @@ module ScoutApm
       end
 
       def call(env)
-        status, headers, response = @app.call(env)
-        path, content_type = env['PATH_INFO'], headers['Content-Type']
-        if ScoutApm::Agent.instance.config.value('dev_trace')
-          if response.respond_to?(:body)
-            req = ScoutApm::RequestManager.lookup
-            slow_converter = LayerConverters::SlowRequestConverter.new(req)
-            trace = slow_converter.call
-            if trace
-              metadata = {
-                  :app_root      => ScoutApm::Environment.instance.root.to_s,
-                  :unique_id     => env['action_dispatch.request_id'], # note, this is a different unique_id than what "normal" payloads use
-                  :agent_version => ScoutApm::VERSION,
-                  :platform      => "ruby",
-              }
-              hash = ScoutApm::Serializers::PayloadSerializerToJson.rearrange_slow_transaction(trace)
-              hash.merge!(:metadata => metadata)
-              payload = ScoutApm::Serializers::PayloadSerializerToJson.jsonify_hash(hash)
-
-              if env['HTTP_X_REQUESTED_WITH'] == 'XMLHttpRequest' || content_type.include?("application/json")
-                ScoutApm::Agent.instance.logger.debug("DevTrace: in middleware, dev_trace is active, and response has a body. This is either AJAX or JSON. Path=#{path}; ContentType=#{content_type}")
-                # Add the payload as a header if it's an AJAX call or JSON
-                headers['X-scoutapminstant'] = payload
-                [status, headers, response]
-              else
-                # otherwise, attempt to add it inline in the page, along with the appropriate JS & CSS. Note, if page doesn't have a head or body,
-                #duration = (req.root_layer.total_call_time*1000).to_i
-                apm_host=ScoutApm::Agent.instance.config.value("direct_host")
-                page = ScoutApm::Instant::Page.new(response.body)
-                page.add_to_head(ScoutApm::Instant::Util.read_asset("xmlhttp_instrumentation.html")) # This monkey-patches XMLHttpRequest. It could possibly be part of the main scout_instant.js too. Putting it here so it runs as soon as possible.
-                page.add_to_head("<link href='#{apm_host}/instant/scout_instant.css?cachebust=#{Time.now.to_i}' media='all' rel='stylesheet' />")
-                page.add_to_body("<script src='#{apm_host}/instant/scout_instant.js?cachebust=#{Time.now.to_i}'></script>")
-                page.add_to_body("<script>var scoutInstantPageTrace=#{payload};window.scoutInstant=window.scoutInstant('#{apm_host}', scoutInstantPageTrace)</script>")
-
-                if response.is_a?(ActionDispatch::Response)
-                  ScoutApm::Agent.instance.logger.debug("DevTrace: in middleware, dev_trace is active, and response has a body. This appears to be an HTML page and an ActionDispatch::Response. Path=#{path}; ContentType=#{content_type}")
-                  # preserve the ActionDispatch::Response when applicable
-                  response.body=[page.res]
-                  [status, headers, response]
-                else
-                  ScoutApm::Agent.instance.logger.debug("DevTrace: in middleware, dev_trace is active, and response has a body. This appears to be an HTML page but not an ActionDispatch::Response. Path=#{path}; ContentType=#{content_type}")
-                  # otherwise, just return an array
-                  [status, headers, [page.res]]
-                end
-              end
-            else
-              ScoutApm::Agent.instance.logger.debug("DevTrace: in middleware, dev_trace is active, and response has a body, but no trace was found. Path=#{path}; ContentType=#{content_type}")
-              [status, headers, response]
-            end
-          else
-            # don't log anything here - this is the path for all assets served in development, and the log would get noisy
-            [status, headers, response]
-          end
+        rack_response = @app.call(env)
+        begin
+          DevTraceResponseManipulator.new(env, rack_response).call
+        rescue Exception => e
+          # If anything went wrong at all, just bail out and return the unmodified response.
+          ScoutApm::Agent.instance.logger.debug("DevTrace: Raised an exception: #{e.message}, #{e.backtrace}")
+          rack_response
+        end
+      end
+    end
+
+    class DevTraceResponseManipulator
+      attr_reader :rack_response
+      attr_reader :rack_status, :rack_headers, :rack_body
+      attr_reader :env
+
+      def initialize(env, rack_response)
+        @env = env
+        @rack_response = rack_response
+
+        @rack_status = rack_response[0]
+        @rack_headers = rack_response[1]
+        @rack_body = rack_response[2]
+      end
+
+      def call
+        return rack_response unless preconditions_met?
+
+        if ajax_request?
+          ScoutApm::Agent.instance.logger.debug("DevTrace: in middleware, dev_trace is active, and response has a body. This is either AJAX or JSON. Path=#{path}; ContentType=#{content_type}")
+          adjust_ajax_header
+        else
+          adjust_html_response
+        end
+
+        rebuild_rack_response
+      end
+
+      ###########################
+      #  Precondition checking  #
+      ###########################
+
+      def preconditions_met?
+        if dev_trace_disabled?
+          logger.debug("DevTrace: isn't activated via config. Try: SCOUT_DEV_TRACE=true rails server")
+          return false
+        end
+
+        # Don't attempt to instrument assets.
+        # Don't log this case, since it would be very noisy
+        logger.debug("DevTrace: dev asset ignored") and return false if development_asset?
+
+        # If we didn't have a tracked_request object, or we explicitly ignored
+        # this request, don't do any work.
+        logger.debug("DevTrace: no tracked request") and return false if tracked_request.nil? || tracked_request.ignoring_request?
+
+        # If we didn't get a trace, we can't show a trace...
+        if trace.nil?
+          logger.debug("DevTrace: in middleware, dev_trace is active, and response has a body, but no trace was found. Path=#{path}; ContentType=#{content_type}")
+          return false
+        end
+
+        true
+      end
+
+      def dev_trace_disabled?
+        ! ScoutApm::Agent.instance.config.value('dev_trace')
+      end
+
+      ########################
+      #  Response Injection  #
+      ########################
+
+      def rebuild_rack_response
+        [rack_status, rack_headers, rack_body]
+      end
+
+      def adjust_ajax_header
+        rack_headers['X-scoutapminstant'] = payload
+      end
+
+      def adjust_html_response
+        case true
+        when older_rails_response? then adjust_older_rails_response
+        when newer_rails_response? then adjust_newer_rails_response
+        when rack_proxy_response? then  adjust_rack_proxy_response
         else
-          ScoutApm::Agent.instance.logger.debug("DevTrace: isn't activated via config. Try: SCOUT_DEV_TRACE=true rails server")
-          [status, headers, response]
+          # No action taken, we only adjust if we know exactly what we have.
+        end
+      end
+
+      def older_rails_response?
+        if defined?(ActionDispatch::Response)
+          return true if rack_body.is_a?(ActionDispatch::Response)
+        end
+      end
+
+      def newer_rails_response?
+        if defined?(ActionDispatch::Response::RackBody)
+          return true if rack_body.is_a?(ActionDispatch::Response::RackBody)
         end
       end
+
+      def rack_proxy_response?
+        rack_body.is_a?(Rack::BodyProxy)
+      end
+
+      def adjust_older_rails_response
+        logger.debug("DevTrace: in middleware, dev_trace is active, and response has a (older) body. This appears to be an HTML page and an ActionDispatch::Response. Path=#{path}; ContentType=#{content_type}")
+        rack_body.body = [ html_manipulator.res ]
+      end
+
+      # Preserve the ActionDispatch::Response object we're working with
+      def adjust_newer_rails_response
+        logger.debug("DevTrace: in middleware, dev_trace is active, and response has a (newer) body. This appears to be an HTML page and an ActionDispatch::Response. Path=#{path}; ContentType=#{content_type}")
+        @rack_body = [ html_manipulator.res ]
+      end
+
+      def adjust_rack_proxy_response
+        logger.debug("DevTrace: in middleware, dev_trace is active, and response has a body. This appears to be an HTML page and an Rack::BodyProxy. Path=#{path}; ContentType=#{content_type}")
+        @rack_body = [ html_manipulator.res ]
+        @rack_headers.delete("Content-Length")
+      end
+
+      def html_manipulator
+        @html_manipulator ||=
+          begin
+            page = ScoutApm::Instant::Page.new(rack_body.body)
+
+            # This monkey-patches XMLHttpRequest. It could possibly be part of the main scout_instant.js too. Putting it here so it runs as soon as possible.
+            page.add_to_head(ScoutApm::Instant::Util.read_asset("xmlhttp_instrumentation.html"))
+
+            # Add a link to CSS, then JS
+            page.add_to_head("<link href='#{apm_host}/instant/scout_instant.css?cachebust=#{Time.now.to_i}' media='all' rel='stylesheet' />")
+            page.add_to_body("<script src='#{apm_host}/instant/scout_instant.js?cachebust=#{Time.now.to_i}'></script>")
+            page.add_to_body("<script>var scoutInstantPageTrace=#{payload};window.scoutInstant=window.scoutInstant('#{apm_host}', scoutInstantPageTrace)</script>")
+
+            page
+          end
+      end
+
+      def ajax_request?
+        env['HTTP_X_REQUESTED_WITH'] == 'XMLHttpRequest' || content_type.include?("application/json")
+      end
+
+      def development_asset?
+        !rack_body.respond_to?(:body)
+      end
+
+      def path
+        env['PATH_INFO']
+      end
+
+      def content_type
+        rack_headers['Content-Type']
+      end
+
+      ##############################
+      #  APM Helpers & Shorthands  #
+      ##############################
+
+      def logger
+        ScoutApm::Agent.instance.logger
+      end
+
+      def tracked_request
+        @tracked_request ||= ScoutApm::RequestManager.lookup
+      end
+
+      def apm_host
+        ScoutApm::Agent.instance.config.value("direct_host")
+      end
+
+      def trace
+        @trace ||= LayerConverters::SlowRequestConverter.new(tracked_request).call
+      end
+
+      def payload
+        @payload ||=
+          begin
+            metadata = {
+              :app_root      => ScoutApm::Environment.instance.root.to_s,
+              :unique_id     => env['action_dispatch.request_id'], # note, this is a different unique_id than what "normal" payloads use
+              :agent_version => ScoutApm::VERSION,
+              :platform      => "ruby",
+            }
+
+            hash = ScoutApm::Serializers::PayloadSerializerToJson.
+              rearrange_slow_transaction(trace).
+              merge!(:metadata => metadata)
+            ScoutApm::Serializers::PayloadSerializerToJson.jsonify_hash(hash)
+          end
+      end
+
     end
   end
 end