scout_apm 2.2.0.pre2 → 2.2.0.pre3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9b1ea1128aa9aa0aa0b5086860810f324ffe732c
-  data.tar.gz: 00e3f0e517bfb8b580205bd7709a040fd13d611a
+  metadata.gz: b4fa1f78b691cca73f89e5f79902cdc097633773
+  data.tar.gz: da4c781988cd5996d6d565b42d5ce25e90e4608a
 SHA512:
-  metadata.gz: bea83409eb9f21cccbfc9aeeeeae3f4a22113cfdd13fa1f51ede6df3422ed5a0d2b7a15d922c7d132632c486430372c68616482b9d2356fd80689ba0cf9d74e8
-  data.tar.gz: 43c32ce10c3e01134904e19a1d2068c1b4a06ac928d44dbb3f1fd46e781ab98414737f9c004858f3ced00fe29534f91af0804f9b2559364ac3e15ad856020918
+  metadata.gz: dc8f935276423d6cb568af1dded73c38af744747b07fb8ed390a0f3f968c6ff7188e912da1acd1c3599d683a1ee8160196ff7756c2b55c384b4584342e677ca9
+  data.tar.gz: e5709ffa058e1ffed9d16bc37733f38c0d777f1d33d597447f2d38c7a168635ecbca9d13540be7baa3a8e84a8f082b68ce23b2a62b3d92b7cb0c103010443bc5

data/.rubocop.yml

@@ -0,0 +1,8 @@
+# 80 is stifling, especially with a few levels of nesting before we even start.
+# So bump it to 100 to keep really long lines from creeping in.
+Metrics/LineLength:
+  Max: 100
+
+Style/UseHashRocketsWithSymbolValues: false
+
+

data/CHANGELOG.markdown

@@ -2,6 +2,26 @@
 
 * ScoutProf BETA
 
+# 2.1.4
+
+* Enhance regular expression that determines if a backtrace line is "inside"
+  the application
+    * Avoids labeling vendor/ as part of the monitored app
+
+# 2.1.3
+
+* Less noisy output on errors with Context
+  * Not logging errors w/nil keys or values
+  * Bumping log level down from WARN => INFO on errors
+* Fix error with complicated AR queries
+  * Caused high log noise
+* Sidekiq instrumentation changes to handle a variety of edge cases
+
+# 2.1.2
+
+* Applies `Rails.application.config.filter_parameters` settings to reported transaction trace uris
+* Fix incompatibility with ResqueWeb and middleware instrumentation
+
 # 2.1.1
 
 * Fix an issue with AR instrumentation and complex queries

data/lib/scout_apm/background_job_integrations/sidekiq.rb

@@ -8,7 +8,7 @@ module ScoutApm
       end
 
       def present?
-        defined?(::Sidekiq) && (File.basename($0) =~ /\Asidekiq/)
+        defined?(::Sidekiq) && File.basename($PROGRAM_NAME).start_with?('sidekiq')
       end
 
       def forking?
@@ -16,22 +16,33 @@ module ScoutApm
       end
 
       def install
+        install_tracer
+        add_middleware
+        install_processor
+      end
+
+      def install_tracer
         # ScoutApm::Tracer is not available when this class is defined
         SidekiqMiddleware.class_eval do
           include ScoutApm::Tracer
         end
+      end
 
+      def add_middleware
         ::Sidekiq.configure_server do |config|
           config.server_middleware do |chain|
             chain.add SidekiqMiddleware
           end
         end
+      end
 
+      def install_processor
         require 'sidekiq/processor' # sidekiq v4 has not loaded this file by this point
 
         ::Sidekiq::Processor.class_eval do
           def initialize_with_scout(boss)
-            ::ScoutApm::Agent.instance.start_background_worker unless ::ScoutApm::Agent.instance.background_worker_running?
+            agent = ::ScoutApm::Agent.instance
+            agent.start_background_worker
             initialize_without_scout(boss)
           end
 
@@ -41,21 +52,18 @@ module ScoutApm
       end
     end
 
+    # We insert this middleware into the Sidekiq stack, to capture each job,
+    # and time them.
     class SidekiqMiddleware
-      def call(worker, msg, queue)
-        job_class = msg["class"] # TODO: Validate this across different versions of Sidekiq
-        if job_class == "ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper" && msg.has_key?("wrapped")
-          job_class = msg["wrapped"]
-        end
-
-        latency = (Time.now.to_f - (msg['enqueued_at'] || msg['created_at']))
+      ACTIVE_JOB_KLASS = 'ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper'.freeze
 
+      def call(_worker, msg, queue)
         req = ScoutApm::RequestManager.lookup
         req.job!
-        req.annotate_request(:queue_latency => latency)
+        req.annotate_request(:queue_latency => latency(msg))
 
-        queue_layer = ScoutApm::Layer.new("Queue", queue)
-        job_layer = ScoutApm::Layer.new("Job", job_class)
+        queue_layer = ScoutApm::Layer.new('Queue', queue)
+        job_layer = ScoutApm::Layer.new('Job', job_class(msg))
 
         if ScoutApm::Agent.instance.config.value('profile') && SidekiqMiddleware.version_supports_profiling?
           # Capture ScoutProf if we can
@@ -64,18 +72,43 @@ module ScoutApm
           #job_layer.traced!
         end
 
-        req.start_layer(queue_layer)
-        req.start_layer(job_layer)
-
         begin
+          req.start_layer(queue_layer)
+          started_queue = true
+          req.start_layer(job_layer)
+          started_job = true
+
           yield
         rescue
           req.error!
           raise
+        ensure
+          req.stop_layer if started_job
+          req.stop_layer if started_queue
+        end
+      end
+
+      UNKNOWN_CLASS_PLACEHOLDER = 'UnknownJob'.freeze
+
+      def job_class(msg)
+        job_class = msg.fetch('class', UNKNOWN_CLASS_PLACEHOLDER)
+        if job_class == ACTIVE_JOB_KLASS && msg.key?('wrapped')
+          job_class = msg['wrapped']
+        end
+        job_class
+      rescue
+        UNKNOWN_CLASS_PLACEHOLDER
+      end
+
+      def latency(msg, time = Time.now.to_f)
+        created_at = msg['enqueued_at'] || msg['created_at']
+        if created_at
+          (time - created_at)
+        else
+          0
         end
-      ensure
-        req.stop_layer # Job
-        req.stop_layer # Queue
+      rescue
+        0
       end
 
       def self.version_supports_profiling?

data/lib/scout_apm/context.rb

@@ -75,8 +75,10 @@ module ScoutApm
     def value_valid?(key_value)
       # ensure one of our accepted types.
       value = key_value.values.last
-      if !valid_type?([String, Symbol, Numeric, Time, Date, TrueClass, FalseClass],value)
-        ScoutApm::Agent.instance.logger.warn "The value for [#{key_value.keys.first}] is not a valid type [#{value.class}]."
+      if value.nil?
+        false # don't log this ... easy to happen
+      elsif !valid_type?([String, Symbol, Numeric, Time, Date, TrueClass, FalseClass],value)
+        ScoutApm::Agent.instance.logger.info "The value for [#{key_value.keys.first}] is not a valid type [#{value.class}]."
         false
       else
         true
@@ -86,14 +88,16 @@ module ScoutApm
     # for consistently with #value_valid?, takes a hash eventhough the value isn't yet used.
     def key_valid?(key_value)
       key = key_value.keys.first
+      if key.nil?
+        false # don't log this ... easy to happen
       # ensure a string or a symbol
-      if !valid_type?([String, Symbol],key)
-        ScoutApm::Agent.instance.logger.warn "The key [#{key}] is not a valid type [#{key.class}]."
+      elsif !valid_type?([String, Symbol],key)
+        ScoutApm::Agent.instance.logger.info "The key [#{key}] is not a valid type [#{key.class}]."
         return false
       end
       # only alphanumeric, dash, and underscore allowed.
       if key.to_s.match(/[^\w-]/)
-        ScoutApm::Agent.instance.logger.warn "They key name [#{key}] is not valid."
+        ScoutApm::Agent.instance.logger.info "They key name [#{key}] is not valid."
         return false
       end
       true

data/lib/scout_apm/instruments/action_controller_rails_2.rb

@@ -48,8 +48,7 @@ module ScoutApm
       # specific controller actions.
       def perform_action_with_scout_instruments(*args, &block)
         req = ScoutApm::RequestManager.lookup
-        path = ScoutApm::Agent.instance.config.value("uri_reporting") == 'path' ? request.path : request.fullpath
-        req.annotate_request(:uri => path)
+        req.annotate_request(:uri => request.path) # for security by-default, we don't use request.fullpath which could reveal filtered params.
         req.context.add_user(:ip => request.remote_ip)
         req.set_headers(request.headers)
         req.start_layer( ScoutApm::Layer.new("Controller", "#{controller_path}/#{action_name}") )

data/lib/scout_apm/instruments/action_controller_rails_3_rails4.rb

@@ -60,8 +60,7 @@ module ScoutApm
     module ActionControllerRails3Rails4Instruments
       def process_action(*args)
         req = ScoutApm::RequestManager.lookup
-        path = ScoutApm::Agent.instance.config.value("uri_reporting") == 'path' ? request.path : request.fullpath
-        req.annotate_request(:uri => path)
+        req.annotate_request(:uri => scout_transaction_uri(request))
 
         # IP Spoofing Protection can throw an exception, just move on w/o remote ip
         req.context.add_user(:ip => request.remote_ip) rescue nil
@@ -95,8 +94,19 @@ module ScoutApm
         ensure
           req.stop_layer
         end
+      end # process_action
+
+      # Given an +ActionDispatch::Request+, formats the uri based on config settings.
+      def scout_transaction_uri(request)
+        case ScoutApm::Agent.instance.config.value("uri_reporting")
+        when 'path'
+          request.path # strips off the query string for more security
+        else # default handles filtered params
+          request.filtered_path
+        end
       end
-    end
+
+    end # ActionControllerRails3Rails4Instruments
   end
 end
 

data/lib/scout_apm/instruments/middleware_summary.rb

@@ -23,7 +23,15 @@ module ScoutApm
           ActionDispatch::MiddlewareStack.class_eval do
             def build_with_scout_instruments(app = nil, &block)
               mw_stack = build_without_scout_instruments(app) { block.call if block }
-              MiddlewareSummaryWrapper.new(mw_stack)
+              if app == mw_stack
+                # Return the raw middleware stack if it equaled app. No
+                # middlewares were created, so nothing to wrap & test.
+                #
+                # Avoids instrumentation of something that doesn't exist
+                mw_stack
+              else
+                MiddlewareSummaryWrapper.new(mw_stack)
+              end
             end
 
             alias_method :build_without_scout_instruments, :build
@@ -45,6 +53,36 @@ module ScoutApm
         ensure
           req.stop_layer
         end
+
+        # Some code (found in resque_web initially) attempts to call methods
+        # directly on `MyApplication.app`, which is the middleware stack.
+        # If it hits our middleware instead of the object at the root of the
+        # app that it expected, then a method it expects will not be there, and an
+        # error thrown.
+        #
+        # Specifically, resque_web assumes `ResqueWeb::Engine.app.url_helpers`
+        # is a method call on rails router for its own Engine, when in fact,
+        # we've added a middleware before it.
+        #
+        # So method_missing just proxies anything to the nested @app object
+        #
+        # While method_missing is not very performant, this is only here to
+        # handle edge-cases in other code, and should not be regularly called
+        def method_missing(sym, *arguments, &block)
+          if @app.respond_to?(sym)
+            @app.send(sym, *arguments, &block)
+          else
+            super
+          end
+        end
+
+        def respond_to?(sym, include_private = false)
+          if @app.respond_to?(sym, include_private)
+            true
+          else
+            super
+          end
+        end
       end
     end
   end

data/lib/scout_apm/tracked_request.rb

@@ -47,7 +47,7 @@ module ScoutApm
       @layers = []
       @call_set = Hash.new { |h, k| h[k] = CallSet.new }
       @annotations = {}
-      @ignoring_children = false
+      @ignoring_children = 0
       @context = Context.new
       @root_layer = nil
       @error = false
@@ -323,27 +323,41 @@ module ScoutApm
     # specific, and useful than the fact that InfluxDB happens to use Net::HTTP
     # internally
     #
-    # When enabled, new layers won't be added to the current Request.
+    # When enabled, new layers won't be added to the current Request, and calls
+    # to stop_layer will be ignored.
     #
     # Do not forget to turn if off when leaving a layer, it is the
     # instrumentation's task to do that.
+    #
+    # When you use this in code, be sure to use it in this order:
+    #
+    # start_layer
+    # ignore_children
+    #  -> call
+    # acknowledge_children
+    # stop_layer
+    #
+    # If you don't call it in this order, it's possible to get out of sync, and
+    # have an ignored start and an actually-executed stop, causing layers to
+    # get out of sync
 
     def ignore_children!
-      @ignoring_children = true
+      @ignoring_children += 1
     end
 
     def acknowledge_children!
-      @ignoring_children = false
+      if @ignoring_children > 0
+        @ignoring_children -= 1
+      end
     end
 
     def ignoring_children?
-      @ignoring_children
+      @ignoring_children > 0
     end
 
     # Grab backtraces more aggressively when running in dev trace mode
     def backtrace_threshold
       dev_trace ? 0.05 : 0.5 # the minimum threshold in seconds to record the backtrace for a metric.
     end
-
   end
 end

data/lib/scout_apm/utils/backtrace_parser.rb

@@ -14,7 +14,10 @@ module ScoutApm
       def initialize(call_stack, root=ScoutApm::Environment.instance.root)
         @call_stack = call_stack
         # We can't use a constant as it'd be too early to fetch environment info
-        @@app_dir_regex = %r|#{root}/(.*)|
+        #
+        # This regex looks for files under the app root, inside lib/, app/, and
+        # config/ dirs, and captures the path under root.
+        @@app_dir_regex = %r[#{root}/((?:lib/|app/|config/).*)]
       end
 
       def call
@@ -27,7 +30,6 @@ module ScoutApm
         end
         stack
       end
-
     end
   end
 end

data/lib/scout_apm/version.rb

@@ -1,4 +1,4 @@
 module ScoutApm
-  VERSION = "2.2.0.pre2"
+  VERSION = "2.2.0.pre3"
 end
 

data/scout_apm.gemspec

@@ -24,6 +24,7 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency "rusage", '~> 0.2.0'
 
   s.add_development_dependency "minitest"
+  s.add_development_dependency 'mocha'
   s.add_development_dependency "pry"
   s.add_development_dependency "m"
   s.add_development_dependency "simplecov"

data/test/test_helper.rb

@@ -5,6 +5,7 @@ SimpleCov.start
 require 'minitest/autorun'
 require 'minitest/unit'
 require 'minitest/pride'
+require 'mocha/mini_test'
 require 'pry'
 
 
@@ -82,4 +83,3 @@ class Minitest::Test
   DATA_FILE_PATH = "#{DATA_FILE_DIR}/scout_apm.db"
 end
 
-

data/test/unit/background_job_integrations/sidekiq_test.rb

@@ -0,0 +1,104 @@
+require 'test_helper'
+require 'scout_apm/request_manager'
+require 'scout_apm/background_job_integrations/sidekiq'
+
+class SidekiqTest < Minitest::Test
+  SidekiqMiddleware = ScoutApm::BackgroundJobIntegrations::SidekiqMiddleware
+
+  ########################################
+  # Middleware
+  ########################################
+  def test_middleware_call_happy_path
+    fake_request = mock
+    fake_request.expects(:annotate_request)
+    fake_request.expects(:job!)
+    fake_request.expects(:start_layer).twice
+    fake_request.expects(:stop_layer).twice
+    fake_request.expects(:error!).never
+
+    ScoutApm::RequestManager.stubs(:lookup).returns(fake_request)
+
+    block_called = false
+    msg = { 'class' => 'MyJobClass',
+            'created_at' => Time.now }
+
+    SidekiqMiddleware.new.call(nil, msg, "defaultqueue") { block_called = true }
+    assert_equal true, block_called
+  end
+
+  def test_middleware_call_job_exception
+    fake_request = mock
+    fake_request.expects(:annotate_request)
+    fake_request.expects(:job!)
+    fake_request.expects(:start_layer).twice
+    fake_request.expects(:stop_layer).twice
+    fake_request.expects(:error!)
+
+    ScoutApm::RequestManager.stubs(:lookup).returns(fake_request)
+
+    msg = { 'class' => 'MyJobClass',
+            'created_at' => Time.now }
+
+    assert_raises RuntimeError do
+      SidekiqMiddleware.new.call(nil, msg, "defaultqueue") { raise "TheJobFailed" }
+    end
+  end
+
+  def test_middleware_call_edge_cases
+    fake_request = mock
+    fake_request.expects(:annotate_request)
+    fake_request.expects(:job!)
+    fake_request.expects(:start_layer).twice
+    fake_request.expects(:stop_layer).twice
+    fake_request.expects(:error!)
+
+    ScoutApm::RequestManager.stubs(:lookup).returns(fake_request)
+
+    # msg doesn't have anything
+    msg = { }
+
+    assert_raises RuntimeError do
+      SidekiqMiddleware.new.call(nil, msg, "defaultqueue") { raise "TheJobFailed" }
+    end
+  end
+
+  ########################################
+  # Job Class Determination
+  ########################################
+  def test_job_class_name_normally
+    msg = { 'class' => 'AGreatJob' }
+    assert_equal 'AGreatJob', SidekiqMiddleware.new.job_class(msg)
+  end
+
+  def test_job_class_name_activejob
+    msg = { 'class' => 'ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper',
+            'wrapped' => 'RealJobClass' }
+    assert_equal 'RealJobClass', SidekiqMiddleware.new.job_class(msg)
+  end
+
+  def test_job_class_name_error_default
+    msg = {}
+    assert_equal 'UnknownJob', SidekiqMiddleware.new.job_class(msg)
+  end
+
+  ########################################
+  # Latency Calculation
+  ########################################
+  def test_latency_from_created_at
+    # Created at time 80, but now it is 200. Latency was 120
+    msg = { 'created_at' => 80 }
+    assert_equal 120, SidekiqMiddleware.new.latency(msg, 200)
+  end
+
+  def test_latency_from_enqueued_at
+    # Created at time 80, but now it is 200. Latency was 120
+    msg = { 'enqueued_at' => 80 }
+    assert_equal 120, SidekiqMiddleware.new.latency(msg, 200)
+  end
+
+  def test_latency_fallback
+    # No created at time, so fall back to 0
+    msg = {}
+    assert_equal 0, SidekiqMiddleware.new.latency(msg, 200)
+  end
+end

data/test/unit/context_test.rb

@@ -0,0 +1,30 @@
+require 'test_helper'
+
+require 'scout_apm/context'
+
+class ContextText < Minitest::Test
+  def test_ignore_nil_value
+    context = ScoutApm::Context.new
+    assert hash = context.add(:nil_key => nil, :org => 'org')
+    assert_equal ({:org => 'org'}), hash
+  end
+
+  def test_ignore_nil_key
+    context = ScoutApm::Context.new
+    assert hash = context.add(nil => nil, :org => 'org')
+    assert_equal ({:org => 'org'}), hash
+  end
+
+  def test_ignore_unsupported_value_type
+    context = ScoutApm::Context.new
+    assert hash = context.add(:array => [1,2,3,4], :org => 'org')
+    assert_equal ({:org => 'org'}), hash
+  end
+
+  def test_ignore_unsupported_key_type
+    context = ScoutApm::Context.new
+    assert hash = context.add([1,2,3,4] => 'hey', :org => 'org')
+    assert_equal ({:org => 'org'}), hash
+  end
+end
+

data/test/unit/utils/backtrace_parser_test.rb

@@ -49,4 +49,18 @@ class BacktraceParserTest < Minitest::Test
     result = ScoutApm::Utils::BacktraceParser.new(raw_backtrace, "/Users/scout/different-secrets").call
     assert_equal 0, result.length
   end
+
+  def test_excludes_vendor_paths
+    raw_backtrace = [
+      "#{root}/vendor/ruby/thing.rb",
+      "#{root}/app/controllers/users_controller.rb",
+      "#{root}/vendor/ruby/thing.rb",
+      "#{root}/config/initializers/inject_something.rb",
+    ]
+    result = ScoutApm::Utils::BacktraceParser.new(raw_backtrace, root).call
+
+    assert_equal 2, result.length
+    assert_equal false, (result[0] =~ %r|app/controllers/users_controller.rb|).nil?
+    assert_equal false, (result[1] =~ %r|config/initializers/inject_something.rb|).nil?
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: scout_apm
 version: !ruby/object:Gem::Version
-  version: 2.2.0.pre2
+  version: 2.2.0.pre3
 platform: ruby
 authors:
 - Derek Haynes
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-08-22 00:00:00.000000000 Z
+date: 2016-08-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rusage
@@ -39,6 +39,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: mocha
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: pry
   requirement: !ruby/object:Gem::Requirement
@@ -105,6 +119,7 @@ extensions:
 extra_rdoc_files: []
 files:
 - ".gitignore"
+- ".rubocop.yml"
 - CHANGELOG.markdown
 - Gemfile
 - LICENSE.md
@@ -229,7 +244,9 @@ files:
 - test/data/config_test_1.yml
 - test/test_helper.rb
 - test/unit/agent_test.rb
+- test/unit/background_job_integrations/sidekiq_test.rb
 - test/unit/config_test.rb
+- test/unit/context_test.rb
 - test/unit/environment_test.rb
 - test/unit/histogram_test.rb
 - test/unit/ignored_uris_test.rb