scout_apm 2.4.0.pre3 → 2.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 29cede8ad70dd0d7cb0be6a777da5d346309ff24
-  data.tar.gz: 17f98116601cc9b75c3d0da0e9f315f2250af174
+  metadata.gz: 35153897d05053c188035cea90a9c9821abcd4cf
+  data.tar.gz: 94fe5aee4fa214256fe31ffe643973274f2d66fb
 SHA512:
-  metadata.gz: 4b5717805b91c5f13ab00e22c5739b4bba32ab7f4bf1eae18118607019937cfba7ad53dfd76086ddc1c5d8fcd2f2350686b594cbb5b6b3a44a61b4759305cad3
-  data.tar.gz: 60b82a60a84001d6a480f2ebdbf16cb5c964e729b30bcbc82c47b0cd526aea71802e3b753bb568176d5a4beca02a1a2d5bdf9b012a0bb48318fc6fb6b2908307
+  metadata.gz: efef5649e203008c653a8fba69e1c28de35f510d831ff3e47a4483fda8e92116cd8aa5e1e64d3ab7ffe63cef9f03113987f953cc78f65a05af076be04fa1fb7b
+  data.tar.gz: 37966ab847e9c50ce354c9f8a2fe3c4ecfda397e091fc2d89f02b7816b2b14bfdd8e4bf56528a1f6572e9bb92b26ab6438b7a4bb83709969c887a8f904a810a0

data/CHANGELOG.markdown

@@ -1,10 +1,27 @@
 # 2.4.0
 
 * Rework agent startup sequence
+* Install all background job instrumentations if you're running more than one
+* Capture longer individual SQL statements
+* Capture multiple SQL statements if multiple are run during a single AR call.
+
+# 2.3.5
+
+* More robust recovery from stale layaway files
+* Quiet logging when hitting unusual layaway file limits
+* Better naming for Sidekiq delayed method jobs
+* Webrick is only required if actually needed
+
+# 2.3.4
+
+* Capture 300 characters of a url from net/http and httpclient instruments (up from 100).
 
 # 2.3.3
 
 * Capture ActiveRecord calls that generate more complex queries
+* More aggressively determine names of complex queries (to determine "User/find", "Account/create" and similar)
+* Increases the maximum size of SQL queries that are sanitized to 16KB from 4 KB
+* Captures all SQL individual queries generated in a given AR call (previous only a single query was captured)
 
 # 2.3.2
 

data/README.markdown

@@ -1,14 +1,21 @@
-# ScoutApm
+# ScoutApm Ruby Agent
 
-A Ruby gem for detailed Rails application performance analysis. Metrics are reported to [Scout](https://scoutapp.com), a hosted application monitoring service. 
+A Ruby gem for detailed Rails application performance analysis. Metrics are
+reported to [Scout](https://scoutapp.com), a hosted application monitoring
+service.
 
 ## Getting Started
 
-Install the gem:
+Add the gem to your Gemfile
 
-    gem install scout_apm
-    
-Signup for a [Scout](https://apm.scoutapp.com) account and copy the config file to `RAILS_ROOT/config/scout_apm.yml`.
+    gem 'scout_apm'
+
+Update your Gemfile
+
+    bundle install
+
+Signup for a [Scout](https://apm.scoutapp.com) account and put the provided
+config file at `RAILS_ROOT/config/scout_apm.yml`.
 
 Your config file should look like:
 
@@ -22,7 +29,8 @@ Your config file should look like:
 
 ## Docs
 
-For the complete list of supported frameworks, Rubies, etc, see our [help site](http://help.apm.scoutapp.com/).
+For the complete list of supported frameworks, Rubies, configuration options
+and more, see our [help site](http://help.apm.scoutapp.com/).
 
 ## Help
 

data/lib/scout_apm.rb

@@ -14,7 +14,6 @@ require 'socket'
 require 'thread'
 require 'time'
 require 'yaml'
-require 'webrick'
 
 #####################################
 # Gem Requires

data/lib/scout_apm/agent.rb

@@ -43,18 +43,18 @@ module ScoutApm
       install_background_job_integrations
       install_app_server_integration
 
-      # XXX: Should this happen at application start?
-      # Should this ever happen after fork?
-      # We start a thread in this, which can screw stuff up when we then fork.
-      #
-      # Save it into a variable to prevent it from ever running twice
-      @app_server_load ||= AppServerLoad.new(context).run
-
       logger.info "Scout Agent [#{ScoutApm::VERSION}] installed"
 
       context.installed!
 
       if ScoutApm::Agent::Preconditions.check?(context) || force
+        # XXX: Should this happen at application start?
+        # Should this ever happen after fork?
+        # We start a thread in this, which can screw stuff up when we then fork.
+        #
+        # Save it into a variable to prevent it from ever running twice
+        @app_server_load ||= AppServerLoad.new(context).run
+
         start
       end
     end
@@ -124,9 +124,8 @@ module ScoutApm
 
     def should_load_instruments?
       return true if context.config.value('dev_trace')
-      # XXX: If monitor is true, we want to install, right?
-      # return false if context.config.value('monitor')
-      context.environment.app_server_integration.found? || context.environment.background_job_integration
+      return false unless context.config.value('monitor')
+      context.environment.app_server_integration.found? || context.environment.background_job_integrations.any?
     end
 
     #################################

data/lib/scout_apm/background_job_integrations/delayed_job.rb

@@ -51,7 +51,6 @@ module ScoutApm
               queue = job.queue || "default"
 
               req = ScoutApm::RequestManager.lookup
-              req.job!
 
               begin
                 latency = Time.now - job.created_at

data/lib/scout_apm/background_job_integrations/sidekiq.rb

@@ -57,7 +57,6 @@ module ScoutApm
     class SidekiqMiddleware
       def call(_worker, msg, queue)
         req = ScoutApm::RequestManager.lookup
-        req.job!
         req.annotate_request(:queue_latency => latency(msg))
 
         begin
@@ -92,10 +91,16 @@ module ScoutApm
           end
         elsif job_class == DELAYED_WRAPPER_KLASS
           begin
+            # Extract the info out of the wrapper
             yml = msg['args'].first
             deserialized_args = YAML.load(yml)
             klass, method, *rest = deserialized_args
-            job_class = [klass,method].map(&:to_s).join(".")
+
+            # If this is an instance of a class, get the class itself
+            # Prevents instances from coming through named like "#<Foo:0x007ffd7a9dd8a0>"
+            klass = klass.class unless klass.is_a? Module
+
+            job_class = [klass, method].map(&:to_s).join(".")
           rescue
             DELAYED_WRAPPER_KLASS
           end

data/lib/scout_apm/config.rb

@@ -66,6 +66,7 @@ module ScoutApm
         'report_format',
         'scm_subdirectory',
         'uri_reporting',
+        'instrument_http_url_length',
     ]
 
     ################################################################################
@@ -153,6 +154,7 @@ module ScoutApm
       "monitor"                => BooleanCoercion.new,
       'database_metric_limit'  => IntegerCoercion.new,
       'database_metric_report_limit' => IntegerCoercion.new,
+      'instrument_http_url_length' => IntegerCoercion.new,
     }
 
 
@@ -256,6 +258,7 @@ module ScoutApm
         'remote_agent_port'      => 7721, # picked at random
         'database_metric_limit'  => 5000, # The hard limit on db metrics
         'database_metric_report_limit' => 1000,
+        'instrument_http_url_length' => 300,
       }.freeze
 
       def value(key)

data/lib/scout_apm/instruments/action_controller_rails_2.rb

@@ -57,7 +57,6 @@ module ScoutApm
         req.context.add_user(:ip => request.remote_ip)
         req.set_headers(request.headers)
         req.start_layer( ScoutApm::Layer.new("Controller", "#{controller_path}/#{action_name}") )
-        req.web!
 
         begin
           perform_action_without_scout_instruments(*args, &block)

data/lib/scout_apm/instruments/action_controller_rails_3_rails4.rb

@@ -77,8 +77,6 @@ module ScoutApm
               req.context.add_user(:ip => request.remote_ip) rescue nil
               req.set_headers(request.headers)
 
-              req.web!
-
               resolved_name = scout_action_name(*args)
               req.start_layer( ScoutApm::Layer.new("Controller", "#{controller_path}/#{resolved_name}") )
               begin

data/lib/scout_apm/instruments/grape.rb

@@ -44,7 +44,6 @@ module ScoutApm
         req.context.add_user(:ip => request.ip) rescue nil
 
         req.set_headers(request.headers)
-        req.web!
 
         begin
           name = ["Grape",

data/lib/scout_apm/instruments/http_client.rb

@@ -26,9 +26,12 @@ module ScoutApm
             include ScoutApm::Tracer
 
             def request_with_scout_instruments(*args, &block)
+
               method = args[0].to_s
               url = args[1]
-              url = url && url.to_s[0..99]
+
+              max_length = ScoutApm::Agent.instance.context.config.value('instrument_http_url_length')
+              url = url && url.to_s[0..(max_length - 1)]
 
               self.class.instrument("HTTP", method, :desc => url) do
                 request_without_scout_instruments(*args, &block)

data/lib/scout_apm/instruments/net_http.rb

@@ -34,7 +34,9 @@ module ScoutApm
             def request_scout_description(req)
               path = req.path
               path = path.path if path.respond_to?(:path)
-              (@address + path.split('?').first)[0..99]
+
+              max_length = ScoutApm::Agent.instance.context.config.value('instrument_http_url_length')
+              (@address + path.split('?').first)[0..(max_length - 1)]
             end
 
             alias request_without_scout_instruments request

data/lib/scout_apm/instruments/resque.rb

@@ -11,7 +11,6 @@ module ScoutApm
         end
 
         req = ScoutApm::RequestManager.lookup
-        req.job!
 
         begin
           req.start_layer(ScoutApm::Layer.new('Queue', queue))

data/lib/scout_apm/layaway.rb

@@ -51,7 +51,8 @@ module ScoutApm
 
     def write_reporting_period(reporting_period, files_limit = MAX_FILES_LIMIT)
       if at_layaway_file_limit?(files_limit)
-        logger.error("Hit layaway file limit. Not writing to layaway file")
+        # This will happen constantly once we hit this case, so only log the first time
+        @wrote_layaway_limit_error_message ||= logger.error("Hit layaway file limit. Not writing to layaway file")
         return false
       end
       filename = file_for(reporting_period.timestamp)
@@ -59,9 +60,22 @@ module ScoutApm
       layaway_file.write(reporting_period)
     end
 
-    # Claims a given timestamp (getting a lock on a particular filename),
-    # then yields ReportingPeriods collected up from all the files.
-    # If the yield returns truthy, delete the layaway files that made it up.
+    # Claims a given timestamp by getting an exclusive lock on a timestamped
+    # coordinator file. The coordinator file never contains data, it's just a
+    # syncronization mechanism.
+    #
+    # Once the 'claim' is obtained:
+    #   * load and yield each ReportingPeriod from the layaway files.
+    #   * if there are reporting periods:
+    #     * yields any ReportingPeriods collected up from all the files.
+    #     * deletes all of the layaway files (including the coordinator) for the timestamp
+    #   * if not
+    #     * delete the coordinator
+    #   * remove any stale layaway files that may be hanging around.
+    #   * Finally unlock and ensure the coordinator file is cleared.
+    #
+    # If a claim file can't be obtained, return false without doing any work
+    # Another process is handling the reporting.
     def with_claim(timestamp)
       coordinator_file = glob_pattern(timestamp, :coordinator)
 
@@ -86,14 +100,14 @@ module ScoutApm
 
               logger.debug("Deleting the now-reported layaway files for #{timestamp.to_s}")
               delete_files_for(timestamp) # also removes the coodinator_file
-
-              logger.debug("Checking for any Stale layaway files")
-              delete_stale_files(timestamp.to_time - STALE_AGE)
             else
               File.unlink(coordinator_file)
               logger.debug("No layaway files to report")
             end
 
+            logger.debug("Checking for any Stale layaway files")
+            delete_stale_files(timestamp.to_time - STALE_AGE)
+
             true
           rescue Exception => e
             logger.debug("Caught an exception in with_claim, with the coordination file locked: #{e.message}, #{e.backtrace.inspect}")

data/lib/scout_apm/layer_converters/find_layer_by_type.rb

@@ -16,6 +16,10 @@ module ScoutApm
         @scope ||= call(["Controller", "Job"])
       end
 
+      def controller
+        @controller ||= call(["Controller"])
+      end
+
       def job
         @job ||= call(["Job"])
       end

data/lib/scout_apm/rack.rb

@@ -10,8 +10,8 @@ module ScoutApm
       req.annotate_request(:uri => env["REQUEST_PATH"]) rescue nil
       req.context.add_user(:ip => env["REMOTE_ADDR"]) rescue nil
 
-      req.web!
-      req.start_layer(ScoutApm::Layer.new('Controller', endpoint_name))
+      layer = ScoutApm::Layer.new('Controller', endpoint_name)
+      req.start_layer(layer)
 
       begin
         yield

data/lib/scout_apm/remote/server.rb

@@ -17,6 +17,8 @@ module ScoutApm
       end
 
       def start
+        require 'webrick'
+
         @server = WEBrick::HTTPServer.new(
           :BindAddress => bind,
           :Port => port,

data/lib/scout_apm/tracked_request.rb

@@ -26,10 +26,6 @@ module ScoutApm
     # Can be nil if we never reach a Rails Controller
     attr_reader :headers
 
-    # What kind of request is this? A trace of a web request, or a background job?
-    # Use job! and web! to set, and job? and web? to query
-    attr_reader :request_type
-
     # This maintains a lookup hash of Layer names and call counts. It's used to trigger fetching a backtrace on n+1 calls.
     # Note that layer names might not be Strings - can alse be Utils::ActiveRecordMetricName. Also, this would fail for layers
     # with same names across multiple types.
@@ -222,20 +218,14 @@ module ScoutApm
       @headers = headers
     end
 
-    def job!
-      @request_type = "job"
-    end
-
+    # This request is a job transaction iff it has a 'Job' layer
     def job?
-      request_type == "job"
-    end
-
-    def web!
-      @request_type = "web"
+      layer_finder.job != nil
     end
 
+    # This request is a web transaction iff it has a 'Controller' layer
     def web?
-      request_type == "web"
+      layer_finder.controller != nil
     end
 
     def instant?
@@ -279,7 +269,6 @@ module ScoutApm
         LayerConverters::SlowRequestConverter,
       ]
 
-      layer_finder = LayerConverters::FindLayerByType.new(self)
       walker = LayerConverters::DepthFirstWalker.new(self.root_layer)
       converters = converters.map do |klass|
         instance = klass.new(@agent_context, self, layer_finder, @store)
@@ -301,6 +290,10 @@ module ScoutApm
       end
     end
 
+    def layer_finder
+      @layer_finder ||= LayerConverters::FindLayerByType.new(self)
+    end
+
     # Ensure the background worker thread is up & running - a fallback if other
     # detection doesn't achieve this at boot.
     def ensure_background_worker

data/lib/scout_apm/version.rb

@@ -1,4 +1,4 @@
 module ScoutApm
-  VERSION = "2.4.0.pre3"
+  VERSION = "2.4.0"
 end
 

data/test/unit/background_job_integrations/sidekiq_test.rb

@@ -11,7 +11,6 @@ class SidekiqTest < Minitest::Test
   def test_middleware_call_happy_path
     fake_request = mock
     fake_request.expects(:annotate_request)
-    fake_request.expects(:job!)
     fake_request.expects(:start_layer).twice
     fake_request.expects(:stop_layer).twice
     fake_request.expects(:error!).never
@@ -29,7 +28,6 @@ class SidekiqTest < Minitest::Test
   def test_middleware_call_job_exception
     fake_request = mock
     fake_request.expects(:annotate_request)
-    fake_request.expects(:job!)
     fake_request.expects(:start_layer).twice
     fake_request.expects(:stop_layer).twice
     fake_request.expects(:error!)
@@ -47,7 +45,6 @@ class SidekiqTest < Minitest::Test
   def test_middleware_call_edge_cases
     fake_request = mock
     fake_request.expects(:annotate_request)
-    fake_request.expects(:job!)
     fake_request.expects(:start_layer).twice
     fake_request.expects(:stop_layer).twice
     fake_request.expects(:error!)

data/test/unit/test_tracked_request.rb

@@ -3,7 +3,7 @@ require 'test_helper'
 class TrackedRequestDumpAndLoadTest < Minitest::Test
   # TrackedRequest must be marshalable
   def test_marshal_dump_load
-    tr = ScoutApm::TrackedRequest.new(ScoutApm::FakeStore.new)
+    tr = ScoutApm::TrackedRequest.new(ScoutApm::AgentContext.new, ScoutApm::FakeStore.new)
     tr.prepare_to_dump!
 
     dumped = Marshal.dump(tr)
@@ -12,58 +12,22 @@ class TrackedRequestDumpAndLoadTest < Minitest::Test
   end
 
   def test_restore_store
-    faux = ScoutApm::FakeStore.new
-    tr = ScoutApm::TrackedRequest.new(faux)
-    assert_equal faux, tr.instance_variable_get("@store")
+    faux_store = ScoutApm::FakeStore.new
+    context = ScoutApm::AgentContext.new
+    tr = ScoutApm::TrackedRequest.new(context, faux_store)
+    assert_equal faux_store, tr.instance_variable_get("@store")
 
     tr.prepare_to_dump!
     assert_nil tr.instance_variable_get("@store")
 
     tr.restore_store
-    assert_equal ScoutApm::Agent.instance.store, tr.instance_variable_get("@store")
-  end
-end
-
-class TrackedRequestFlagsTest < Minitest::Test
-  def test_set_web
-    tr = ScoutApm::TrackedRequest.new(ScoutApm::FakeStore.new)
-    assert_false tr.web?
-    tr.web!
-    assert tr.web?
-  end
-
-  def test_set_job
-    tr = ScoutApm::TrackedRequest.new(ScoutApm::FakeStore.new)
-    assert ! tr.job?
-    tr.job!
-    assert tr.job?
-  end
-
-  def test_set_error
-    tr = ScoutApm::TrackedRequest.new(ScoutApm::FakeStore.new)
-    assert_false tr.error?
-    tr.error!
-    assert tr.error?
-  end
-
-  def test_set_error_and_web
-    tr = ScoutApm::TrackedRequest.new(ScoutApm::FakeStore.new)
-    assert_false tr.error?
-    assert_false tr.web?
-
-    tr.web!
-    assert_false tr.error?
-    assert tr.web?
-
-    tr.error!
-    assert tr.error?
-    assert tr.web?
+    assert_equal context.store, tr.instance_variable_get("@store")
   end
 end
 
 class TrackedRequestLayerManipulationTest < Minitest::Test
   def test_start_layer
-    tr = ScoutApm::TrackedRequest.new(ScoutApm::FakeStore.new)
+    tr = ScoutApm::TrackedRequest.new(ScoutApm::AgentContext.new, ScoutApm::FakeStore.new)
     tr.start_layer(ScoutApm::Layer.new("Foo", "Bar"))
 
     assert_equal "Foo", tr.current_layer.type
@@ -74,7 +38,7 @@ class TrackedRequestLayerManipulationTest < Minitest::Test
     controller_layer = ScoutApm::Layer.new("Controller", "users/index")
     ar_layer = ScoutApm::Layer.new("ActiveRecord", "Users#find")
 
-    tr = ScoutApm::TrackedRequest.new(ScoutApm::FakeStore.new)
+    tr = ScoutApm::TrackedRequest.new(ScoutApm::AgentContext.new, ScoutApm::FakeStore.new)
     tr.start_layer(controller_layer)
     tr.start_layer(ar_layer)
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: scout_apm
 version: !ruby/object:Gem::Version
-  version: 2.4.0.pre3
+  version: 2.4.0
 platform: ruby
 authors:
 - Derek Haynes
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-12-21 00:00:00.000000000 Z
+date: 2018-01-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: minitest
@@ -360,9 +360,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
 rubyforge_project: scout_apm
 rubygems_version: 2.4.5.2