scout_apm 3.0.0.pre8 → 3.0.0.pre9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 63c9b4319eecbefb59fdb10fb0fbc0d264dfb7a0
-  data.tar.gz: 821271be9ef95416e72b659a825895513d80017d
+  metadata.gz: ed6963870425b875415255f3aa5bb51144b5fabd
+  data.tar.gz: f31e6e8780992531d3a87733b3f4ef5bc3e11ab8
 SHA512:
-  metadata.gz: 330d8721be955d8676285665d7db211919b1867da0720b1b24c663119f434cb7aaffa9a55cec27d8286633d40d924b337283fea0c45195d872900f54a3c6c041
-  data.tar.gz: 0eef9b024471d03e689aaceffd56402c32a5f0904e6c9124209bb60e4d06173f281fc25b3562225303b5c8292be7daab2f5233618a098454c646da597d4913fa
+  metadata.gz: bb4b61f29fc72f0c1eeec3c9bc2ac86b57a66605a5d8dba43612dc686d95b75c6658665d8c2106ae637c794f2a7af689796163aa31e4c8e561610a16f2064778
+  data.tar.gz: d7a9137fcdd64016a75cb74f1c4f02c653c38dff5621b6d23baaefa1c4a5657fbd13dc8d7ccd25844ee0c7ab44771becb4982754e6a48589c92624cfad012919

data/CHANGELOG.markdown

@@ -2,6 +2,23 @@
 
 * ScoutProf BETA
 
+# 2.1.22
+
+* Add DevTrace support for newest 4.2.x and 5.x versions of Rails
+
+# 2.1.21
+
+* Fix edge case, causing DevTrace to fail
+* Add debug tooling, allowing custom functions to be inserted into the agent at
+  key points.
+
+# 2.1.20
+
+* Add a `detailed_middleware` boolean configuration option to capture
+  per-middleware data, as opposed to the default of aggregating all middleware
+  together.  This has a small amount of additional overhead, approximately
+  10-15ms per request.
+
 # 2.1.19
 
 * Log all configuration settings at start when log level is debug

data/lib/scout_apm/agent/reporting.rb

@@ -72,7 +72,7 @@ module ScoutApm
         log_deliver(metrics, slow_transactions, metadata, slow_jobs, histograms)
 
         payload = ScoutApm::Serializers::PayloadSerializer.serialize(metadata, metrics, slow_transactions, jobs, slow_jobs, histograms)
-        # logger.debug("Payload: #{payload}")
+        logger.debug("Sending payload w/ Headers: #{headers.inspect}")
 
         reporter.report(payload, headers)
       rescue => e

data/lib/scout_apm/agent.rb

@@ -265,6 +265,7 @@ module ScoutApm
       @background_worker = ScoutApm::BackgroundWorker.new
       @background_worker_thread = Thread.new do
         @background_worker.start {
+          ScoutApm::Debug.instance.call_periodic_hooks
           ScoutApm::Agent.instance.process_metrics
           clean_old_percentiles
         }
@@ -293,8 +294,13 @@ module ScoutApm
         install_instrument(ScoutApm::Instruments::ActionControllerRails2)
       when :rails3_or_4 then
         install_instrument(ScoutApm::Instruments::ActionControllerRails3Rails4)
-        install_instrument(ScoutApm::Instruments::MiddlewareSummary)
         install_instrument(ScoutApm::Instruments::RailsRouter)
+
+        if config.value("detailed_middleware")
+          install_instrument(ScoutApm::Instruments::MiddlewareDetailed)
+        else
+          install_instrument(ScoutApm::Instruments::MiddlewareSummary)
+        end
       end
 
       install_instrument(ScoutApm::Instruments::ActiveRecord)

data/lib/scout_apm/config.rb

@@ -37,6 +37,7 @@ module ScoutApm
         'compress_payload',
         'config_file',
         'data_file',
+        'detailed_middleware',
         'dev_trace',
         'direct_host',
         'disabled_instruments',
@@ -45,8 +46,8 @@ module ScoutApm
         'hostname',
         'ignore',
         'key',
-        'log_level',
         'log_file_path',
+        'log_level',
         'monitor',
         'name',
         'profile',
@@ -129,6 +130,7 @@ module ScoutApm
       "monitor"                => BooleanCoercion.new,
       "enable_background_jobs" => BooleanCoercion.new,
       "dev_trace"              => BooleanCoercion.new,
+      "detailed_middleware"    => BooleanCoercion.new,
       "ignore"                 => JsonCoercion.new,
     }
 
@@ -203,6 +205,7 @@ module ScoutApm
     class ConfigDefaults
       DEFAULTS = {
         'compress_payload'       => true,
+        'detailed_middleware'    => false,
         'dev_trace'              => false,
         'direct_host'            => 'https://apm.scoutapp.com',
         'disabled_instruments'   => [],

data/lib/scout_apm/debug.rb

@@ -0,0 +1,37 @@
+module ScoutApm
+  class Debug
+    # see self.instance
+    @@instance = nil
+
+    def self.instance
+      @@instance ||= new
+    end
+
+    def register_periodic_hook(&hook)
+      @periodic_hooks << hook
+    end
+
+    def call_periodic_hooks
+      @periodic_hooks.each do |hook|
+        begin
+          hook.call
+        rescue => e
+          logger.info("Periodic debug hook failed to run: #{e}\n\t#{e.backtrace.join("\n\t")}")
+        end
+      end
+    rescue
+      # Something went super wrong for the inner rescue to not catch this. Just
+      # swallow the error. The debug tool should never crash the app.
+    end
+
+    private
+
+    def initialize
+      @periodic_hooks = []
+    end
+
+    def logger
+      ScoutApm::Agent.instance.logger
+    end
+  end
+end

data/lib/scout_apm/instant/assets/xmlhttp_instrumentation.html

@@ -3,7 +3,7 @@
   (function(){var open=window.XMLHttpRequest.prototype.open;var send=window.XMLHttpRequest.prototype.send;function openReplacement(method,url,async,user,password){this._url=url;return open.apply(this,arguments);}
     function sendReplacement(data){if(this.onload){this._onload=this.onload;}
       this.onload=onLoadReplacement;return send.apply(this,arguments);}
-    function onLoadReplacement(){if(this._url.startsWith(window.location.protocol+"//"+window.location.host)||!this._url.startsWith("http")){try{traceText=this.getResponseHeader("X-scoutapminstant");if(traceText){setTimeout(function(){window.scoutInstant("addTrace",traceText)},0);}}catch(e){console.debug("Problem getting X-scoutapminstant header");}}
+    function onLoadReplacement(){if(this._url.startsWith(window.location.protocol+"//"+window.location.host)||!this._url.startsWith("http")){try{var traceText=this.getResponseHeader("X-scoutapminstant");if(traceText){setTimeout(function(){window.scoutInstant("addTrace",traceText)},0);}}catch(e){console.debug("Problem getting X-scoutapminstant header");}}
       if(this._onload){return this._onload.apply(this,arguments);}}
     window.XMLHttpRequest.prototype.open=openReplacement;window.XMLHttpRequest.prototype.send=sendReplacement;})();
-</script>
+</script>

data/lib/scout_apm/instant/middleware.rb

@@ -5,6 +5,11 @@ module ScoutApm
     class Page
       def initialize(html)
         @html = html
+
+        if html.is_a?(Array)
+          @html = html.inject("") { |memo, str| memo + str }
+        end
+
         @to_add_to_head = []
         @to_add_to_body = []
       end
@@ -45,66 +50,200 @@ module ScoutApm
       end
 
       def call(env)
-        status, headers, response = @app.call(env)
-        path, content_type = env['PATH_INFO'], headers['Content-Type']
-        if ScoutApm::Agent.instance.config.value('dev_trace')
-          if response.respond_to?(:body)
-            req = ScoutApm::RequestManager.lookup
-
-            return [status, headers, response] if req.ignoring_request?
-
-            slow_converter = LayerConverters::SlowRequestConverter.new(req)
-            trace = slow_converter.call
-            if trace
-              metadata = {
-                  :app_root      => ScoutApm::Environment.instance.root.to_s,
-                  :unique_id     => env['action_dispatch.request_id'], # note, this is a different unique_id than what "normal" payloads use
-                  :agent_version => ScoutApm::VERSION,
-                  :platform      => "ruby",
-              }
-              hash = ScoutApm::Serializers::PayloadSerializerToJson.rearrange_slow_transaction(trace)
-              hash.merge!(:metadata => metadata)
-              payload = ScoutApm::Serializers::PayloadSerializerToJson.jsonify_hash(hash)
-
-              if env['HTTP_X_REQUESTED_WITH'] == 'XMLHttpRequest' || content_type.include?("application/json")
-                ScoutApm::Agent.instance.logger.debug("DevTrace: in middleware, dev_trace is active, and response has a body. This is either AJAX or JSON. Path=#{path}; ContentType=#{content_type}")
-                # Add the payload as a header if it's an AJAX call or JSON
-                headers['X-scoutapminstant'] = payload
-                [status, headers, response]
-              else
-                # otherwise, attempt to add it inline in the page, along with the appropriate JS & CSS. Note, if page doesn't have a head or body,
-                #duration = (req.root_layer.total_call_time*1000).to_i
-                apm_host=ScoutApm::Agent.instance.config.value("direct_host")
-                page = ScoutApm::Instant::Page.new(response.body)
-                page.add_to_head(ScoutApm::Instant::Util.read_asset("xmlhttp_instrumentation.html")) # This monkey-patches XMLHttpRequest. It could possibly be part of the main scout_instant.js too. Putting it here so it runs as soon as possible.
-                page.add_to_head("<link href='#{apm_host}/instant/scout_instant.css?cachebust=#{Time.now.to_i}' media='all' rel='stylesheet' />")
-                page.add_to_body("<script src='#{apm_host}/instant/scout_instant.js?cachebust=#{Time.now.to_i}'></script>")
-                page.add_to_body("<script>var scoutInstantPageTrace=#{payload};window.scoutInstant=window.scoutInstant('#{apm_host}', scoutInstantPageTrace)</script>")
-
-                if response.is_a?(ActionDispatch::Response)
-                  ScoutApm::Agent.instance.logger.debug("DevTrace: in middleware, dev_trace is active, and response has a body. This appears to be an HTML page and an ActionDispatch::Response. Path=#{path}; ContentType=#{content_type}")
-                  # preserve the ActionDispatch::Response when applicable
-                  response.body=[page.res]
-                  [status, headers, response]
-                else
-                  ScoutApm::Agent.instance.logger.debug("DevTrace: in middleware, dev_trace is active, and response has a body. This appears to be an HTML page but not an ActionDispatch::Response. Path=#{path}; ContentType=#{content_type}")
-                  # otherwise, just return an array
-                  [status, headers, [page.res]]
-                end
-              end
-            else
-              ScoutApm::Agent.instance.logger.debug("DevTrace: in middleware, dev_trace is active, and response has a body, but no trace was found. Path=#{path}; ContentType=#{content_type}")
-              [status, headers, response]
-            end
-          else
-            # don't log anything here - this is the path for all assets served in development, and the log would get noisy
-            [status, headers, response]
-          end
+        rack_response = @app.call(env)
+        begin
+          DevTraceResponseManipulator.new(env, rack_response).call
+        rescue Exception => e
+          # If anything went wrong at all, just bail out and return the unmodified response.
+          ScoutApm::Agent.instance.logger.debug("DevTrace: Raised an exception: #{e.message}, #{e.backtrace}")
+          rack_response
+        end
+      end
+    end
+
+    class DevTraceResponseManipulator
+      attr_reader :rack_response
+      attr_reader :rack_status, :rack_headers, :rack_body
+      attr_reader :env
+
+      def initialize(env, rack_response)
+        @env = env
+        @rack_response = rack_response
+
+        @rack_status = rack_response[0]
+        @rack_headers = rack_response[1]
+        @rack_body = rack_response[2]
+      end
+
+      def call
+        return rack_response unless preconditions_met?
+
+        if ajax_request?
+          ScoutApm::Agent.instance.logger.debug("DevTrace: in middleware, dev_trace is active, and response has a body. This is either AJAX or JSON. Path=#{path}; ContentType=#{content_type}")
+          adjust_ajax_header
+        else
+          adjust_html_response
+        end
+
+        rebuild_rack_response
+      end
+
+      ###########################
+      #  Precondition checking  #
+      ###########################
+
+      def preconditions_met?
+        if dev_trace_disabled?
+          logger.debug("DevTrace: isn't activated via config. Try: SCOUT_DEV_TRACE=true rails server")
+          return false
+        end
+
+        # Don't attempt to instrument assets.
+        # Don't log this case, since it would be very noisy
+        logger.debug("DevTrace: dev asset ignored") and return false if development_asset?
+
+        # If we didn't have a tracked_request object, or we explicitly ignored
+        # this request, don't do any work.
+        logger.debug("DevTrace: no tracked request") and return false if tracked_request.nil? || tracked_request.ignoring_request?
+
+        # If we didn't get a trace, we can't show a trace...
+        if trace.nil?
+          logger.debug("DevTrace: in middleware, dev_trace is active, and response has a body, but no trace was found. Path=#{path}; ContentType=#{content_type}")
+          return false
+        end
+
+        true
+      end
+
+      def dev_trace_disabled?
+        ! ScoutApm::Agent.instance.config.value('dev_trace')
+      end
+
+      ########################
+      #  Response Injection  #
+      ########################
+
+      def rebuild_rack_response
+        [rack_status, rack_headers, rack_body]
+      end
+
+      def adjust_ajax_header
+        rack_headers['X-scoutapminstant'] = payload
+      end
+
+      def adjust_html_response
+        case true
+        when older_rails_response? then adjust_older_rails_response
+        when newer_rails_response? then adjust_newer_rails_response
+        when rack_proxy_response? then  adjust_rack_proxy_response
         else
-          ScoutApm::Agent.instance.logger.debug("DevTrace: isn't activated via config. Try: SCOUT_DEV_TRACE=true rails server")
-          [status, headers, response]
+          # No action taken, we only adjust if we know exactly what we have.
+        end
+      end
+
+      def older_rails_response?
+        if defined?(ActionDispatch::Response)
+          return true if rack_body.is_a?(ActionDispatch::Response)
+        end
+      end
+
+      def newer_rails_response?
+        if defined?(ActionDispatch::Response::RackBody)
+          return true if rack_body.is_a?(ActionDispatch::Response::RackBody)
         end
       end
+
+      def rack_proxy_response?
+        rack_body.is_a?(Rack::BodyProxy)
+      end
+
+      def adjust_older_rails_response
+        logger.debug("DevTrace: in middleware, dev_trace is active, and response has a (older) body. This appears to be an HTML page and an ActionDispatch::Response. Path=#{path}; ContentType=#{content_type}")
+        rack_body.body = [ html_manipulator.res ]
+      end
+
+      # Preserve the ActionDispatch::Response object we're working with
+      def adjust_newer_rails_response
+        logger.debug("DevTrace: in middleware, dev_trace is active, and response has a (newer) body. This appears to be an HTML page and an ActionDispatch::Response. Path=#{path}; ContentType=#{content_type}")
+        @rack_body = [ html_manipulator.res ]
+      end
+
+      def adjust_rack_proxy_response
+        logger.debug("DevTrace: in middleware, dev_trace is active, and response has a body. This appears to be an HTML page and an Rack::BodyProxy. Path=#{path}; ContentType=#{content_type}")
+        @rack_body = [ html_manipulator.res ]
+        @rack_headers.delete("Content-Length")
+      end
+
+      def html_manipulator
+        @html_manipulator ||=
+          begin
+            page = ScoutApm::Instant::Page.new(rack_body.body)
+
+            # This monkey-patches XMLHttpRequest. It could possibly be part of the main scout_instant.js too. Putting it here so it runs as soon as possible.
+            page.add_to_head(ScoutApm::Instant::Util.read_asset("xmlhttp_instrumentation.html"))
+
+            # Add a link to CSS, then JS
+            page.add_to_head("<link href='#{apm_host}/instant/scout_instant.css?cachebust=#{Time.now.to_i}' media='all' rel='stylesheet' />")
+            page.add_to_body("<script src='#{apm_host}/instant/scout_instant.js?cachebust=#{Time.now.to_i}'></script>")
+            page.add_to_body("<script>var scoutInstantPageTrace=#{payload};window.scoutInstant=window.scoutInstant('#{apm_host}', scoutInstantPageTrace)</script>")
+
+            page
+          end
+      end
+
+      def ajax_request?
+        env['HTTP_X_REQUESTED_WITH'] == 'XMLHttpRequest' || content_type.include?("application/json")
+      end
+
+      def development_asset?
+        !rack_body.respond_to?(:body)
+      end
+
+      def path
+        env['PATH_INFO']
+      end
+
+      def content_type
+        rack_headers['Content-Type']
+      end
+
+      ##############################
+      #  APM Helpers & Shorthands  #
+      ##############################
+
+      def logger
+        ScoutApm::Agent.instance.logger
+      end
+
+      def tracked_request
+        @tracked_request ||= ScoutApm::RequestManager.lookup
+      end
+
+      def apm_host
+        ScoutApm::Agent.instance.config.value("direct_host")
+      end
+
+      def trace
+        @trace ||= LayerConverters::SlowRequestConverter.new(tracked_request).call
+      end
+
+      def payload
+        @payload ||=
+          begin
+            metadata = {
+              :app_root      => ScoutApm::Environment.instance.root.to_s,
+              :unique_id     => env['action_dispatch.request_id'], # note, this is a different unique_id than what "normal" payloads use
+              :agent_version => ScoutApm::VERSION,
+              :platform      => "ruby",
+            }
+
+            hash = ScoutApm::Serializers::PayloadSerializerToJson.
+              rearrange_slow_transaction(trace).
+              merge!(:metadata => metadata)
+            ScoutApm::Serializers::PayloadSerializerToJson.jsonify_hash(hash)
+          end
+      end
+
     end
   end
 end

data/lib/scout_apm/instruments/middleware_detailed.rb

@@ -1,13 +1,11 @@
 # Inserts a new middleware between each actual middleware in the application,
 # so as to trace the time for each one.
 #
-# Currently disabled due to the overhead of this approach (~10-15ms per request
-# in practice).  Instead, middleware as a whole are instrumented via the
-# MiddlewareSummary class.
+# Currently disabled by default due to the overhead of this approach (~10-15ms
+# per request in practice).  Instead, middleware as a whole are instrumented
+# via the MiddlewareSummary class.
 #
-# There will likely be a configuration flag to turn this on in favor of the
-# summary tracing in a future version of the agent, but this is not yet
-# implemented.
+# Turn this on with the configuration setting `detailed_middleware` set to true
 module ScoutApm
   module Instruments
     class MiddlewareDetailed

data/lib/scout_apm/version.rb

@@ -1,3 +1,3 @@
 module ScoutApm
-  VERSION = "3.0.0.pre8"
+  VERSION = "3.0.0.pre9"
 end

data/lib/scout_apm.rb

@@ -26,6 +26,7 @@ require 'rusage'
 #####################################
 require 'scout_apm/version'
 
+require 'scout_apm/debug'
 require 'scout_apm/tracked_request'
 require 'scout_apm/layer'
 require 'scout_apm/limited_layer'
@@ -76,7 +77,7 @@ require 'scout_apm/instruments/active_record'
 require 'scout_apm/instruments/action_controller_rails_2'
 require 'scout_apm/instruments/action_controller_rails_3_rails4'
 require 'scout_apm/instruments/middleware_summary'
-# require 'scout_apm/instruments/middleware_detailed' # Currently disabled functionality, see the file for details.
+require 'scout_apm/instruments/middleware_detailed' # Disabled by default, see the file for more details
 require 'scout_apm/instruments/rails_router'
 require 'scout_apm/instruments/grape'
 require 'scout_apm/instruments/sinatra'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: scout_apm
 version: !ruby/object:Gem::Version
-  version: 3.0.0.pre8
+  version: 3.0.0.pre9
 platform: ruby
 authors:
 - Derek Haynes
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-12-29 00:00:00.000000000 Z
+date: 2017-02-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: minitest
@@ -148,6 +148,7 @@ files:
 - lib/scout_apm/call_set.rb
 - lib/scout_apm/config.rb
 - lib/scout_apm/context.rb
+- lib/scout_apm/debug.rb
 - lib/scout_apm/environment.rb
 - lib/scout_apm/fake_store.rb
 - lib/scout_apm/framework_integrations/rails_2.rb
@@ -292,34 +293,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: 1.3.1
 requirements: []
 rubyforge_project: scout_apm
-rubygems_version: 2.5.2
+rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
 summary: Ruby application performance monitoring
-test_files:
-- test/data/config_test_1.yml
-- test/test_helper.rb
-- test/unit/agent_test.rb
-- test/unit/background_job_integrations/sidekiq_test.rb
-- test/unit/config_test.rb
-- test/unit/context_test.rb
-- test/unit/environment_test.rb
-- test/unit/git_revision_test.rb
-- test/unit/histogram_test.rb
-- test/unit/ignored_uris_test.rb
-- test/unit/instruments/active_record_instruments_test.rb
-- test/unit/instruments/net_http_test.rb
-- test/unit/instruments/percentile_sampler_test.rb
-- test/unit/layaway_test.rb
-- test/unit/layer_children_set_test.rb
-- test/unit/limited_layer_test.rb
-- test/unit/metric_set_test.rb
-- test/unit/scored_item_set_test.rb
-- test/unit/serializers/payload_serializer_test.rb
-- test/unit/slow_job_policy_test.rb
-- test/unit/slow_request_policy_test.rb
-- test/unit/sql_sanitizer_test.rb
-- test/unit/store_test.rb
-- test/unit/utils/active_record_metric_name_test.rb
-- test/unit/utils/backtrace_parser_test.rb
-- test/unit/utils/numbers_test.rb
+test_files: []